From 3be0479b8cbb802b04e759015fcbf3532829ae9c Mon Sep 17 00:00:00 2001 From: egg12138 Date: Sat, 11 Oct 2025 20:47:28 +0800 Subject: [PATCH 1/5] refactor: fix inconsistent feature name fetching * kconfig features and _list_features share the same feature list Signed-off-by: egg12138 --- src/oebuild/app/plugins/generate/generate.py | 60 ++++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index 3b0caa2..e25780e 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -367,26 +367,42 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") table.add_row([platform.replace('.yaml', '')]) print(table) - def _list_feature(self, ): + def _list_feature(self): yocto_dir = self.configure.source_yocto_dir() - yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") - list_feature = os.listdir(os.path.join(yocto_oebuild_dir, 'features')) + yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") + feature_triples = self.get_feature_triple(yocto_oebuild_dir) print("the feature list is:") table = PrettyTable(['feature name', 'support arch']) table.align = "l" - for feature in list_feature: - feature_name = feature.replace('.yml', '') - if feature.endswith('.yaml'): - feature_name = feature.replace('.yaml', '') - feat = oebuild_util.read_yaml( - pathlib.Path( - os.path.join(yocto_oebuild_dir, 'features', feature))) - if "support" in feat: - table.add_row([feature_name, feat.get('support')]) - else: - table.add_row([feature_name, "all"]) + for feature_name, feature_path, feature_data in feature_triples: + table.add_row([feature_name, feature_data.get('support') or "all"]) print(table) + @staticmethod + def get_feature_triple(oebuild_dir): + feature_dir = pathlib.Path(oebuild_dir, 'features') + # feature_dir = os.path.join(oebuild_dir, 'features') + if not os.path.exists(feature_dir): + logger.error("features directory does not exist under oebuild meta directory ") + sys.exit(-1) + feature_files = os.listdir(feature_dir) + features = [] + for ft_filename in feature_files: + if not (ft_filename.endswith('.yml') or ft_filename.endswith('.yaml')): + continue + feature_path = pathlib.Path(feature_dir, ft_filename) + try: + feature_data = oebuild_util.read_yaml(feature_path) + feature_name = os.path.splitext(ft_filename)[0] + features.append((feature_name, feature_path, feature_data)) + except Exception as e: + logger.warning(""" + parse feature file '%s' failed, skipped. + Please check the file content. + """, feature_path) + continue + return features + def check_support_oebuild(self, yocto_dir): ''' Determine if OeBuild is supported by checking if .oebuild @@ -513,21 +529,15 @@ endif Returns: """ - feature_path = os.path.join(yocto_oebuild_dir, 'features') - if os.path.exists(feature_path): - feature_list = os.listdir(feature_path) - else: - logger.error('feature dir is not exists') - sys.exit(-1) + feature_start = """ if IMAGE comment "this is choose OS features" """ - for feature in feature_list: + + feature_triples = self.get_feature_triple(yocto_oebuild_dir) + for feature_name, _, feature_data in feature_triples: support_str = "" - feature_path = pathlib.Path(yocto_oebuild_dir, 'features', feature) - feature_data = oebuild_util.read_yaml(feature_path) - feature_name = os.path.splitext(feature)[0].strip("\n") if 'support' in feature_data: support_str = ("if PLATFORM_" + feature_data['support'].upper().replace( @@ -989,3 +999,5 @@ def get_sdk_docker_image(yocto_dir): if docker_image is None: docker_image = oebuild_const.DEFAULT_SDK_DOCKER return docker_image + + -- Gitee From 42300d19bbd2c65e6c823ce791f02c644d677834 Mon Sep 17 00:00:00 2001 From: egg12138 Date: Sat, 11 Oct 2025 22:19:16 +0800 Subject: [PATCH 2/5] refactor: generate.py duplicate feat case and styles * migrate file operations to pathlib for more pythonic taste * add resolution with .yaml/.yml priority handling, prevent duplicate feature case (e.g. feat1.yaml and feat1.yml) * add validation for platform support lists in kconfig generation. Signed-off-by: egg12138 --- src/oebuild/app/plugins/generate/generate.py | 168 +++++++++++-------- 1 file changed, 102 insertions(+), 66 deletions(-) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index e25780e..a6abda9 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -272,12 +272,12 @@ oebuild toolchain def _add_platform_template(self, args, yocto_oebuild_dir, parser_template: ParseTemplate): - if args.platform + '.yaml' in os.listdir( - os.path.join(yocto_oebuild_dir, 'platform')): + platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') + platform_files = [f.name for f in platform_path.iterdir() if f.is_file()] + if args.platform + '.yaml' in platform_files: try: - parser_template.add_template( - os.path.join(yocto_oebuild_dir, 'platform', - args.platform + '.yaml')) + platform_file = platform_path / (args.platform + '.yaml') + parser_template.add_template(platform_file) except BaseParseTemplate as e_p: raise e_p else: @@ -288,13 +288,13 @@ wrong platform, please run `oebuild generate -l` to view support platform""") def _add_features_template(self, args, yocto_oebuild_dir, parser_template: ParseTemplate): if args.features: + features_path = pathlib.Path(yocto_oebuild_dir, 'features') + feature_files = [f.name for f in features_path.iterdir() if f.is_file()] for feature in args.features: - if feature + '.yaml' in os.listdir( - os.path.join(yocto_oebuild_dir, 'features')): + if feature + '.yaml' in feature_files: try: - parser_template.add_template( - os.path.join(yocto_oebuild_dir, 'features', - feature + '.yaml')) + feature_file = features_path / (feature + '.yaml') + parser_template.add_template(feature_file) except BaseParseTemplate as b_t: raise b_t else: @@ -355,53 +355,74 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") def _list_platform(self): logger.info("=============================================") yocto_dir = self.configure.source_yocto_dir() - yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") - list_platform = os.listdir(os.path.join(yocto_oebuild_dir, 'platform')) + yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") + platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') + list_platform = [f for f in platform_path.iterdir() if f.is_file()] print("the platform list is:") table = PrettyTable(['platform name']) table.align = "l" for platform in list_platform: - if platform.endswith('.yml'): - table.add_row([platform.replace('.yml', '')]) - if platform.endswith('.yaml'): - table.add_row([platform.replace('.yaml', '')]) + if platform.suffix in ['.yml', '.yaml']: + table.add_row([platform.stem]) print(table) def _list_feature(self): yocto_dir = self.configure.source_yocto_dir() yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") - feature_triples = self.get_feature_triple(yocto_oebuild_dir) + feature_triples = self.resolve_feature_files(yocto_oebuild_dir) print("the feature list is:") table = PrettyTable(['feature name', 'support arch']) table.align = "l" - for feature_name, feature_path, feature_data in feature_triples: + for feature_name, _, feature_data in feature_triples: table.add_row([feature_name, feature_data.get('support') or "all"]) print(table) @staticmethod - def get_feature_triple(oebuild_dir): + def resolve_feature_files(oebuild_dir): + """ + both yaml and yml are supported, but yaml is preferred + """ + feature_dir = pathlib.Path(oebuild_dir, 'features') - # feature_dir = os.path.join(oebuild_dir, 'features') - if not os.path.exists(feature_dir): + if not feature_dir.exists(): logger.error("features directory does not exist under oebuild meta directory ") sys.exit(-1) - feature_files = os.listdir(feature_dir) + + # Group files by base name to handle priority + file_groups = {} + + for ft_path in feature_dir.iterdir(): + if ft_path.is_file(): + if ft_path.suffix == '.yaml': + feature_name = ft_path.stem + if feature_name not in file_groups: + file_groups[feature_name] = {'yaml': None, 'yml': None} + file_groups[feature_name]['yaml'] = ft_path + elif ft_path.suffix == '.yml': + feature_name = ft_path.stem + if feature_name not in file_groups: + file_groups[feature_name] = {'yaml': None, 'yml': None} + file_groups[feature_name]['yml'] = ft_path + features = [] - for ft_filename in feature_files: - if not (ft_filename.endswith('.yml') or ft_filename.endswith('.yaml')): - continue - feature_path = pathlib.Path(feature_dir, ft_filename) - try: - feature_data = oebuild_util.read_yaml(feature_path) - feature_name = os.path.splitext(ft_filename)[0] - features.append((feature_name, feature_path, feature_data)) - except Exception as e: - logger.warning(""" - parse feature file '%s' failed, skipped. - Please check the file content. - """, feature_path) - continue - return features + for feature_name, extensions in file_groups.items(): + # Prefer .yaml over .yml + selected_file = extensions['yaml'] or extensions['yml'] + if selected_file: + try: + feature_data = oebuild_util.read_yaml(selected_file) + if not isinstance(feature_data, dict): + logger.warning(f"not valid yaml key:value format {feature_name}") + continue + features.append((feature_name, selected_file, feature_data)) + except Exception as e: + fname = selected_file.name + logger.warning(f""" + parse feature file '{fname}' failed, error {e}. + """) + continue + + return features def check_support_oebuild(self, yocto_dir): ''' @@ -450,7 +471,7 @@ endif with open(kconfig_path, 'w', encoding='utf-8') as kconfig_file: kconfig_file.write(write_data) - kconf = Kconfig(filename=kconfig_path) + kconf = Kconfig(filename=str(kconfig_path)) os.environ["MENUCONFIG_STYLE"] = "aquatic selection=fg:white,bg:blue" with oebuild_util.suppress_print(): menuconfig(kconf) @@ -495,9 +516,9 @@ endif Returns: """ - platform_path = os.path.join(yocto_oebuild_dir, 'platform') - if os.path.exists(platform_path): - platform_list = os.listdir(platform_path) + platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') + if platform_path.exists(): + platform_files = [f for f in platform_path.iterdir() if f.is_file() and f.suffix in ['.yml', '.yaml']] else: logger.error('platform dir is not exists') sys.exit(-1) @@ -511,8 +532,8 @@ endif platform_end = """ endchoice endif""" - for platform in platform_list: - platform_name = os.path.splitext(platform)[0].strip("\n") + for platform in platform_files: + platform_name = platform.stem.strip("\n") platform_info = ( f""" config PLATFORM_{platform_name.upper()}\n""" f""" bool "{platform_name}"\n\n""") @@ -535,21 +556,36 @@ endif comment "this is choose OS features" """ - feature_triples = self.get_feature_triple(yocto_oebuild_dir) + feature_triples = self.resolve_feature_files(yocto_oebuild_dir) for feature_name, _, feature_data in feature_triples: support_str = "" if 'support' in feature_data: - support_str = ("if PLATFORM_" + - feature_data['support'].upper().replace( - '|', '||PLATFORM_')) + raw_supports = feature_data['support'] + validated_support_str = self._validate_and_format_platforms( + raw_supports) + if validated_support_str: + support_str = validated_support_str + else: + logger.warning(f"{feature_name} support platform list {raw_supports} is not valid") + # support_str = ("if PLATFORM_" + + # feature_data['support'].upper().replace( + # '|', '||PLATFORM_')) feature_info = (f"""\nconfig FEATURE_{feature_name.upper()}\n""" f""" bool "{feature_name}" {support_str}\n\n""") feature_start += feature_info feature_start += "endif" - return feature_start + def _validate_and_format_platforms(self, raw_str: str): + import re + platforms = re.split(r'[|,，\s]+', raw_str) + platforms = [p.strip() for p in platforms if p.strip()] + if not platforms: + return "" + platform_cond = [f"PLATFORM_{p.upper()}" for p in platforms] + return "if " + "||".join(platform_cond) + def _kconfig_add_gcc_toolchain(self, yocto_oebuild_dir): """ add toolchain to kconfig @@ -560,20 +596,22 @@ endif """ toolchain_start = "" - cross_dir = os.path.join(yocto_oebuild_dir, "cross-tools") - if os.path.exists(cross_dir): - toolchain_list = os.listdir(os.path.join(cross_dir, 'configs')) - toolchain_start += """ + cross_dir = pathlib.Path(yocto_oebuild_dir, "cross-tools") + if cross_dir.exists(): + configs_dir = cross_dir / 'configs' + if configs_dir.exists(): + toolchain_list = os.listdir(configs_dir) + toolchain_start += """ if GCC-TOOLCHAIN """ - for config in toolchain_list: - if not re.search('xml', config): - toolchain_info = ( - f"""\nconfig GCC-TOOLCHAIN_{config.upper().lstrip("CONFIG_")}\n""" - f""" bool "{config.upper().lstrip("CONFIG_")}"\n""" - """ depends on GCC-TOOLCHAIN\n""") - toolchain_start += toolchain_info - toolchain_start += "endif" + for config in toolchain_list: + if not re.search('xml', config): + toolchain_info = ( + f"""\nconfig GCC-TOOLCHAIN_{config.upper().lstrip("CONFIG_")}\n""" + f""" bool "{config.upper().lstrip("CONFIG_")}"\n""" + """ depends on GCC-TOOLCHAIN\n""") + toolchain_start += toolchain_info + toolchain_start += "endif" return toolchain_start def _kconfig_add_llvm_toolchain(self, yocto_oebuild_dir): @@ -586,8 +624,8 @@ endif """ toolchain_start = "" - llvm_dir = os.path.join(yocto_oebuild_dir, "llvm-toolchain") - if os.path.exists(llvm_dir): + llvm_dir = pathlib.Path(yocto_oebuild_dir, "llvm-toolchain") + if llvm_dir.exists(): toolchain_start += """ config LLVM-TOOLCHAIN_AARCH64-LIB string "aarch64 lib dir" @@ -605,8 +643,8 @@ endif Returns: """ - nativesdk_dir = os.path.join(yocto_oebuild_dir, "nativesdk") - return os.path.exists(nativesdk_dir) + nativesdk_dir = pathlib.Path(yocto_oebuild_dir, "nativesdk") + return nativesdk_dir.exists() def _kconfig_add_common_config(self,): """ @@ -999,5 +1037,3 @@ def get_sdk_docker_image(yocto_dir): if docker_image is None: docker_image = oebuild_const.DEFAULT_SDK_DOCKER return docker_image - - -- Gitee From 410b347fd7858a57c768907ed0cf4d98a3ca6fc2 Mon Sep 17 00:00:00 2001 From: egg12138 Date: Mon, 13 Oct 2025 11:34:48 +0800 Subject: [PATCH 3/5] refactor: continous generate.py style migratation * as a fixup of the last commit Signed-off-by: egg12138 --- src/oebuild/app/plugins/generate/generate.py | 66 ++++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index a6abda9..8abba09 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -172,7 +172,7 @@ class Generate(OebuildCommand): parser_template = ParseTemplate(yocto_dir=yocto_dir) - yocto_oebuild_dir = os.path.join(yocto_dir, '.oebuild') + yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') try: self._add_platform_template(args=args, @@ -197,15 +197,16 @@ class Generate(OebuildCommand): logger.error(str(v_e)) sys.exit(-1) - if os.path.exists(os.path.join(build_dir, 'compile.yaml')): - os.remove(os.path.join(build_dir, 'compile.yaml')) + compile_yaml_path = pathlib.Path(build_dir, 'compile.yaml') + if compile_yaml_path.exists(): + compile_yaml_path.unlink() docker_image = get_docker_image( yocto_dir=self.configure.source_yocto_dir(), docker_tag='', configure=self.configure) - out_dir = pathlib.Path(os.path.join(build_dir, 'compile.yaml')) + out_dir = pathlib.Path(build_dir, 'compile.yaml') param = parser_template.get_default_generate_compile_conf_param() param['nativesdk_dir'] = self.params.get('nativesdk_dir', None) @@ -303,26 +304,25 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") sys.exit(-1) def _init_build_dir(self, args): - if not os.path.exists(self.configure.build_dir()): - os.makedirs(self.configure.build_dir()) + build_dir_path = pathlib.Path(self.configure.build_dir()) + if not build_dir_path.exists(): + build_dir_path.mkdir(parents=True) if args.directory is None or args.directory == '': - build_dir = os.path.join(self.configure.build_dir(), args.platform) + build_dir = build_dir_path / args.platform else: - build_dir = os.path.join(self.configure.build_dir(), - args.directory) + build_dir = build_dir_path / args.directory - if not os.path.abspath(build_dir).startswith( - self.configure.build_dir()): + if not pathlib.Path(build_dir).absolute().is_relative_to(build_dir_path.absolute()): logger.error("Build path must in oebuild workspace") return None # detects if a build directory already exists - if os.path.exists(build_dir): + if build_dir.exists(): logger.warning("the build directory %s already exists", build_dir) while not args.yes: in_res = input(f""" - do you want to overwrite it({os.path.basename(build_dir)})? the overwrite action + do you want to overwrite it({build_dir.name})? the overwrite action will replace the compile.yaml or toolchain.yaml to new and delete conf directory, enter Y for yes, N for no, C for create:""") if in_res not in ["Y", "y", "yes", "N", "n", "no", "C", "c", "create"]: @@ -334,16 +334,17 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") if in_res in ['C', 'c', 'create']: in_res = input(""" please enter now build name, we will create it under build directory:""") - build_dir = os.path.join(self.configure.build_dir(), in_res) - if os.path.exists(build_dir): + build_dir = build_dir_path / in_res + if build_dir.exists(): continue break - if os.path.exists(os.path.join(build_dir, "conf")): - rmtree(os.path.join(build_dir, "conf")) - elif os.path.exists(build_dir): + conf_dir = build_dir / "conf" + if conf_dir.exists(): + rmtree(conf_dir) + elif build_dir.exists(): rmtree(build_dir) - os.makedirs(build_dir, exist_ok=True) - return build_dir + build_dir.mkdir(parents=True, exist_ok=True) + return str(build_dir) def list_info(self, ): ''' @@ -429,7 +430,7 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") Determine if OeBuild is supported by checking if .oebuild exists in the yocto-meta-openeuler directory ''' - return os.path.exists(os.path.join(yocto_dir, '.oebuild')) + return pathlib.Path(yocto_dir, '.oebuild').exists() def create_kconfig(self, yocto_dir): """ @@ -437,7 +438,7 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") Returns: """ - yocto_oebuild_dir = os.path.join(yocto_dir, '.oebuild') + yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') write_data = "" target_list_data, gcc_toolchain_data, llvm_toolchain_data = \ @@ -518,7 +519,8 @@ endif """ platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') if platform_path.exists(): - platform_files = [f for f in platform_path.iterdir() if f.is_file() and f.suffix in ['.yml', '.yaml']] + platform_files = [f for f in platform_path.iterdir() + if f.is_file() and f.suffix in ['.yml', '.yaml']] else: logger.error('platform dir is not exists') sys.exit(-1) @@ -566,10 +568,8 @@ endif if validated_support_str: support_str = validated_support_str else: - logger.warning(f"{feature_name} support platform list {raw_supports} is not valid") - # support_str = ("if PLATFORM_" + - # feature_data['support'].upper().replace( - # '|', '||PLATFORM_')) + logger.warning(f"""{feature_name} support platform + {raw_supports} is invalid""") feature_info = (f"""\nconfig FEATURE_{feature_name.upper()}\n""" f""" bool "{feature_name}" {support_str}\n\n""") @@ -879,8 +879,8 @@ endif Returns: """ - source_cross_dir = os.path.join(self.configure.source_yocto_dir(), ".oebuild/cross-tools") - if not os.path.exists(source_cross_dir): + source_cross_dir = pathlib.Path(self.configure.source_yocto_dir(), ".oebuild/cross-tools") + if not source_cross_dir.exists(): logger.error('Build dependency not downloaded, not supported for build. Please ' 'download the latest yocto meta openeuler repository') sys.exit(-1) @@ -902,7 +902,7 @@ endif continue config_list.append("config_" + gcc_name) oebuild_util.write_yaml( - yaml_path=os.path.join(build_dir, 'toolchain.yaml'), + yaml_path=pathlib.Path(build_dir, 'toolchain.yaml'), data={ 'kind': oebuild_const.GCC_TOOLCHAIN, 'gcc_configs': config_list, @@ -942,8 +942,8 @@ endif Returns: """ - source_llvm_dir = os.path.join(self.configure.source_yocto_dir(), ".oebuild/llvm-toolchain") - if not os.path.exists(source_llvm_dir): + source_llvm_dir = pathlib.Path(self.configure.source_yocto_dir(), ".oebuild/llvm-toolchain") + if not source_llvm_dir.exists(): logger.error('Build dependency not downloaded, not supported for build. Please ' 'download the latest yocto meta openeuler repository') sys.exit(-1) @@ -959,7 +959,7 @@ endif } ) oebuild_util.write_yaml( - yaml_path=os.path.join(build_dir, 'toolchain.yaml'), + yaml_path=pathlib.Path(build_dir, 'toolchain.yaml'), data={ 'kind': oebuild_const.LLVM_TOOLCHAIN, 'llvm_lib': llvm_lib, -- Gitee From cbd07c328b5fe7922afac019c747071a27a8a210 Mon Sep 17 00:00:00 2001 From: Egg12138 Date: Wed, 15 Oct 2025 10:49:26 +0800 Subject: [PATCH 4/5] generate: refactor code structure To fit pylint better and for better code maintainability. Extract Kconfig generation logic from generate.py into new KconfigGenerator class.This separation of concerns makes the codebase more modular and easier to test. - Move all Kconfig-related methods to kconfig_generator.py - Extract parse_feature_files to parses.py for reuse - Clean up docstrings and comments - improve code style - Add proper class structure for Kconfig handling Signed-off-by: egg12138 --- src/oebuild/app/plugins/generate/generate.py | 484 ++---------------- .../app/plugins/generate/kconfig_generator.py | 351 +++++++++++++ src/oebuild/app/plugins/generate/parses.py | 59 ++- 3 files changed, 457 insertions(+), 437 deletions(-) create mode 100644 src/oebuild/app/plugins/generate/kconfig_generator.py diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index 8abba09..dae4fa7 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -17,43 +17,37 @@ import textwrap import os import sys import pathlib -import time from shutil import rmtree -from kconfiglib import Kconfig -from menuconfig import menuconfig from prettytable import PrettyTable from ruamel.yaml.scalarstring import LiteralScalarString from oebuild.command import OebuildCommand import oebuild.util as oebuild_util from oebuild.configure import Configure -from oebuild.parse_template import BaseParseTemplate, ParseTemplate, \ - get_docker_param_dict, parse_repos_layers_local_obj +from oebuild.parse_template import ( + BaseParseTemplate, ParseTemplate, + get_docker_param_dict, + parse_repos_layers_local_obj) from oebuild.m_log import logger from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const -from oebuild.app.plugins.generate.parses import parsers +from oebuild.app.plugins.generate.parses import parsers, parse_feature_files +from oebuild.app.plugins.generate.kconfig_generator import KconfigGenerator class Generate(OebuildCommand): - ''' - compile.yaml is generated according to different command parameters by generate - ''' + """Generate compile.yaml (and toolchain.yaml) from CLI options.""" - help_msg = 'help to mkdir build directory and generate compile.yaml' + help_msg = 'Create build dir and generate compile.yaml' description = textwrap.dedent('''\ - The generate command is the core command in the entire build process, which - is mainly used to customize the build configuration parameters and generate - a compile.yaml by customizing each parameter. In addition, for a large number - of configuration parameter input is not very convenient, generate provides a - way to specify compile.yaml, users can directly specify the file after - customizing the build configuration file + Customize build parameters and output compile.yaml; optionally + emit toolchain.yaml for GCC/LLVM builds. ''') def __init__(self): self.configure = Configure() - # params is use for record generate command param, had follow params + # Cache of parsed CLI parameters affecting generation # nativesdk_dir # toolchain_dir # llvm_toolchain_dir @@ -67,8 +61,8 @@ class Generate(OebuildCommand): super().__init__('generate', self.help_msg, self.description) def do_add_parser(self, parser_adder): + """Add arguments to the parser.""" parser = self._parser(parser_adder, usage=''' - %(prog)s ''') @@ -78,7 +72,7 @@ class Generate(OebuildCommand): # pylint:disable=[R0914,R0911,R0912,R0915,W1203,R0913] def do_run(self, args: argparse.Namespace, unknown=None): - # perpare parse help command + """The main entry point for the command.""" if self.pre_parse_help(args, unknown): sys.exit(0) if not self.configure.is_oebuild_dir(): @@ -88,17 +82,18 @@ class Generate(OebuildCommand): yocto_dir = self.configure.source_yocto_dir() if not self.check_support_oebuild(yocto_dir): logger.error( - 'Currently, yocto-meta-openeuler does not support oebuild, \ - please modify .oebuild/config and re-execute `oebuild update`' - ) + 'yocto-meta-openeuler does not support oebuild. ' + 'Update .oebuild/config and re-run `oebuild update`.') sys.exit(-1) if len(unknown) == 0: - config_path = self.create_kconfig(yocto_dir) + yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') + kconfig_generator = KconfigGenerator( + self.oebuild_kconfig_path, yocto_oebuild_dir) + config_path = kconfig_generator.create_kconfig() if not os.path.exists(config_path): sys.exit(0) g_command = self.generate_command(config_path) - # sys.exit(0) subprocess.check_output(f'rm -rf {config_path}', shell=True) args = args.parse_args(g_command) else: @@ -106,7 +101,7 @@ class Generate(OebuildCommand): auto_build = bool(args.auto_build) if args.nativesdk: - # this is for default directory nativesdk + # default dir for nativesdk if args.directory is None or args.directory == '': args.directory = "nativesdk" build_dir = self._init_build_dir(args=args) @@ -117,7 +112,7 @@ class Generate(OebuildCommand): sys.exit(0) if args.gcc: - # this is for default directory toolchain + # default dir for toolchain if args.directory is None or args.directory == '': args.directory = "toolchain" toolchain_name_list = args.gcc_name if args.gcc_name else [] @@ -129,7 +124,7 @@ class Generate(OebuildCommand): sys.exit(0) if args.llvm: - # this is for default directory is toolchain + # default dir for toolchain if args.directory is None or args.directory == '': args.directory = "toolchain" llvm_lib = args.llvm_lib @@ -233,7 +228,7 @@ class Generate(OebuildCommand): format_dir = f''' generate compile.yaml successful -please run follow command: +Run commands below: ============================================= cd {build_dir} @@ -247,7 +242,7 @@ oebuild bitbake format_dir = f''' generate compile.yaml successful -please run follow command: +Run commands below: ============================================= cd {build_dir} @@ -261,7 +256,7 @@ oebuild bitbake or oebuild bitbake buildtools-extended-tarball format_dir = f''' generate toolchain.yaml successful -please run follow command: +Run commands below: ============================================= cd {build_dir} @@ -282,8 +277,8 @@ oebuild toolchain except BaseParseTemplate as e_p: raise e_p else: - logger.error(""" -wrong platform, please run `oebuild generate -l` to view support platform""") + logger.error( + "Invalid platform. Run `oebuild generate -l` to list supported platforms.") sys.exit(-1) def _add_features_template(self, args, yocto_oebuild_dir, @@ -299,8 +294,8 @@ wrong platform, please run `oebuild generate -l` to view support platform""") except BaseParseTemplate as b_t: raise b_t else: - logger.error(""" -Wrong platform, please run `oebuild generate -l` to view support feature""") + logger.error( + "Invalid feature. Run `oebuild generate -l` to list features.") sys.exit(-1) def _init_build_dir(self, args): @@ -317,23 +312,21 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") logger.error("Build path must in oebuild workspace") return None - # detects if a build directory already exists + # If build dir exists, prompt/handle overwrite if build_dir.exists(): logger.warning("the build directory %s already exists", build_dir) while not args.yes: in_res = input(f""" - do you want to overwrite it({build_dir.name})? the overwrite action - will replace the compile.yaml or toolchain.yaml to new and delete conf directory, - enter Y for yes, N for no, C for create:""") + Overwrite {build_dir.name}? This will replace compile.yaml/toolchain.yaml and delete conf/ + Enter Y=yes, N=no, C=create : """) if in_res not in ["Y", "y", "yes", "N", "n", "no", "C", "c", "create"]: - print(""" - wrong input""") + print("Invalid input") continue if in_res in ['N', 'n', 'no']: return None if in_res in ['C', 'c', 'create']: - in_res = input(""" - please enter now build name, we will create it under build directory:""") + in_res = input( + "Enter new build name (will be created under build/):") build_dir = build_dir_path / in_res if build_dir.exists(): continue @@ -354,13 +347,12 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") self._list_feature() def _list_platform(self): - logger.info("=============================================") + logger.info("\n================= Available Platforms =================") yocto_dir = self.configure.source_yocto_dir() yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') list_platform = [f for f in platform_path.iterdir() if f.is_file()] - print("the platform list is:") - table = PrettyTable(['platform name']) + table = PrettyTable(['Platform Name']) table.align = "l" for platform in list_platform: if platform.suffix in ['.yml', '.yaml']: @@ -368,401 +360,28 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") print(table) def _list_feature(self): + logger.info("\n================= Available Features ==================") yocto_dir = self.configure.source_yocto_dir() yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") - feature_triples = self.resolve_feature_files(yocto_oebuild_dir) - print("the feature list is:") - table = PrettyTable(['feature name', 'support arch']) + feature_triples = parse_feature_files(yocto_oebuild_dir) + table = PrettyTable(['Feature Name', 'Supported Arch']) table.align = "l" for feature_name, _, feature_data in feature_triples: table.add_row([feature_name, feature_data.get('support') or "all"]) print(table) - - @staticmethod - def resolve_feature_files(oebuild_dir): - """ - both yaml and yml are supported, but yaml is preferred - """ - - feature_dir = pathlib.Path(oebuild_dir, 'features') - if not feature_dir.exists(): - logger.error("features directory does not exist under oebuild meta directory ") - sys.exit(-1) - - # Group files by base name to handle priority - file_groups = {} - - for ft_path in feature_dir.iterdir(): - if ft_path.is_file(): - if ft_path.suffix == '.yaml': - feature_name = ft_path.stem - if feature_name not in file_groups: - file_groups[feature_name] = {'yaml': None, 'yml': None} - file_groups[feature_name]['yaml'] = ft_path - elif ft_path.suffix == '.yml': - feature_name = ft_path.stem - if feature_name not in file_groups: - file_groups[feature_name] = {'yaml': None, 'yml': None} - file_groups[feature_name]['yml'] = ft_path - - features = [] - for feature_name, extensions in file_groups.items(): - # Prefer .yaml over .yml - selected_file = extensions['yaml'] or extensions['yml'] - if selected_file: - try: - feature_data = oebuild_util.read_yaml(selected_file) - if not isinstance(feature_data, dict): - logger.warning(f"not valid yaml key:value format {feature_name}") - continue - features.append((feature_name, selected_file, feature_data)) - except Exception as e: - fname = selected_file.name - logger.warning(f""" - parse feature file '{fname}' failed, error {e}. - """) - continue - - return features + logger.info( + """* 'Supported Arch' defaults to 'all' if not specified in the feature's .yaml file.""" + ) def check_support_oebuild(self, yocto_dir): ''' - Determine if OeBuild is supported by checking if .oebuild - exists in the yocto-meta-openeuler directory + Return True if /.oebuild exists. ''' return pathlib.Path(yocto_dir, '.oebuild').exists() - def create_kconfig(self, yocto_dir): - """ - create_kconfig - Returns: - - """ - yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') - write_data = "" - - target_list_data, gcc_toolchain_data, llvm_toolchain_data = \ - self._kconfig_add_target_list(yocto_oebuild_dir) - write_data += target_list_data + gcc_toolchain_data + llvm_toolchain_data - - # add auto-build config - write_data += """ -if NATIVESDK || GCC-TOOLCHAIN || LLVM-TOOLCHAIN -comment "if auto build nativesdk or gcc toolchain or llvm toolchain" - config AUTO-BUILD - bool "auto build" - default n -endif -""" - - platform_data = self._kconfig_add_choice_platform(yocto_oebuild_dir) - write_data += platform_data - - feature_data = self._kconfig_add_feature(yocto_oebuild_dir) - write_data += feature_data - - common_data = self._kconfig_add_common_config() - write_data += common_data - - if not os.path.exists( - pathlib.Path(self.oebuild_kconfig_path).absolute()): - os.makedirs(pathlib.Path(self.oebuild_kconfig_path).absolute()) - kconfig_path = pathlib.Path(self.oebuild_kconfig_path, - str(int(time.time()))) - - with open(kconfig_path, 'w', encoding='utf-8') as kconfig_file: - kconfig_file.write(write_data) - kconf = Kconfig(filename=str(kconfig_path)) - os.environ["MENUCONFIG_STYLE"] = "aquatic selection=fg:white,bg:blue" - with oebuild_util.suppress_print(): - menuconfig(kconf) - subprocess.check_output(f'rm -rf {kconfig_path}', shell=True) - config_path = pathlib.Path(os.getcwd(), '.config') - return config_path - - def _kconfig_add_target_list(self, yocto_oebuild_dir): - target_choice = textwrap.dedent(""" - comment "this is choose build target" - choice - prompt "choice build target" - config IMAGE\n - bool 'OS'\n -""") - gcc_toolchain_data = self._kconfig_add_gcc_toolchain(yocto_oebuild_dir) - if gcc_toolchain_data != "": - target_choice += ( - "config GCC-TOOLCHAIN \n" - " bool 'GCC TOOLCHAIN'\n\n") - llvm_toolchain_data = self._kconfig_add_llvm_toolchain(yocto_oebuild_dir) - if llvm_toolchain_data != "": - target_choice += ( - "config LLVM-TOOLCHAIN \n" - " bool 'LLVM TOOLCHAIN'\n\n" - ) - nativesdk_check = self._kconfig_check_nativesdk(yocto_oebuild_dir) - if nativesdk_check: - target_choice += ( - "config NATIVESDK \n" - " bool 'NATIVESDK'\n\n" - ) - target_choice += "endchoice" - return target_choice, gcc_toolchain_data, llvm_toolchain_data - - def _kconfig_add_choice_platform(self, yocto_oebuild_dir): - """ - add platform to kconfig - Args: - yocto_oebuild_dir: - - Returns: - - """ - platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') - if platform_path.exists(): - platform_files = [f for f in platform_path.iterdir() - if f.is_file() and f.suffix in ['.yml', '.yaml']] - else: - logger.error('platform dir is not exists') - sys.exit(-1) - platform_start = textwrap.dedent(""" - if IMAGE - comment "this is choose OS platform" - choice - prompt "choice platform" - default PLATFORM_QEMU-AARCH64\n - """) - platform_end = """ - endchoice - endif""" - for platform in platform_files: - platform_name = platform.stem.strip("\n") - platform_info = ( - f""" config PLATFORM_{platform_name.upper()}\n""" - f""" bool "{platform_name}"\n\n""") - platform_start += platform_info - platform_data = platform_start + platform_end - return platform_data - - def _kconfig_add_feature(self, yocto_oebuild_dir): - """ - add feature to kconfig - Args: - yocto_oebuild_dir: - - Returns: - - """ - - feature_start = """ - if IMAGE - comment "this is choose OS features" - """ - - feature_triples = self.resolve_feature_files(yocto_oebuild_dir) - for feature_name, _, feature_data in feature_triples: - support_str = "" - if 'support' in feature_data: - raw_supports = feature_data['support'] - validated_support_str = self._validate_and_format_platforms( - raw_supports) - if validated_support_str: - support_str = validated_support_str - else: - logger.warning(f"""{feature_name} support platform - {raw_supports} is invalid""") - - feature_info = (f"""\nconfig FEATURE_{feature_name.upper()}\n""" - f""" bool "{feature_name}" {support_str}\n\n""") - feature_start += feature_info - feature_start += "endif" - return feature_start - - def _validate_and_format_platforms(self, raw_str: str): - import re - platforms = re.split(r'[|,，\s]+', raw_str) - platforms = [p.strip() for p in platforms if p.strip()] - if not platforms: - return "" - platform_cond = [f"PLATFORM_{p.upper()}" for p in platforms] - return "if " + "||".join(platform_cond) - - def _kconfig_add_gcc_toolchain(self, yocto_oebuild_dir): - """ - add toolchain to kconfig - Args: - yocto_oebuild_dir: yocto_oebuild_dir - - Returns: - - """ - toolchain_start = "" - cross_dir = pathlib.Path(yocto_oebuild_dir, "cross-tools") - if cross_dir.exists(): - configs_dir = cross_dir / 'configs' - if configs_dir.exists(): - toolchain_list = os.listdir(configs_dir) - toolchain_start += """ - if GCC-TOOLCHAIN - """ - for config in toolchain_list: - if not re.search('xml', config): - toolchain_info = ( - f"""\nconfig GCC-TOOLCHAIN_{config.upper().lstrip("CONFIG_")}\n""" - f""" bool "{config.upper().lstrip("CONFIG_")}"\n""" - """ depends on GCC-TOOLCHAIN\n""") - toolchain_start += toolchain_info - toolchain_start += "endif" - return toolchain_start - - def _kconfig_add_llvm_toolchain(self, yocto_oebuild_dir): - """ - add toolchain to kconfig - Args: - yocto_oebuild_dir: yocto_oebuild_dir - - Returns: - - """ - toolchain_start = "" - llvm_dir = pathlib.Path(yocto_oebuild_dir, "llvm-toolchain") - if llvm_dir.exists(): - toolchain_start += """ - config LLVM-TOOLCHAIN_AARCH64-LIB - string "aarch64 lib dir" - default "None" - depends on LLVM-TOOLCHAIN - """ - return toolchain_start - - def _kconfig_check_nativesdk(self, yocto_oebuild_dir): - """ - add nativesdk to kconfig - Args: - yocto_oebuild_dir: yocto_oebuild_dir - - Returns: - - """ - nativesdk_dir = pathlib.Path(yocto_oebuild_dir, "nativesdk") - return nativesdk_dir.exists() - - def _kconfig_add_common_config(self,): - """ - Kconfig basic_config - Returns: - - """ - toolchain_help = ( - "this param is for external gcc toolchain dir, " - "if you want use your own toolchain") - llvm_toolchain_help = ( - "this param is for external llvm toolchain dir, " - "if you want use your own toolchain") - nativesdk_help = ( - "this param is for external nativesdk dir," - "the param will be useful when you want to build in host") - common_str = textwrap.dedent(""" - comment "this is common config" - """) - # choice build in platform - common_str += (""" - if IMAGE - choice - prompt "choice build in docker or host" - default BUILD_IN-DOCKER - config BUILD_IN-DOCKER - bool "docker" - config BUILD_IN-HOST - bool "host" - endchoice - endif - """) - # add no fetch - common_str += (""" - config COMMON_NO-FETCH - bool "no_fetch (this will set openeuler_fetch to disable)" - default n - depends on IMAGE - """) - # add no layer - common_str += (""" - config COMMON_NO-LAYER - bool "no_layer (this will disable layer repo update when startting bitbake environment)" - default n - depends on IMAGE - """) - # add sstate_mirrors - common_str += (""" - config COMMON_SSTATE-MIRRORS - string "sstate_mirrors (this param is for SSTATE_MIRRORS)" - default "None" - depends on IMAGE - """) - # add sstate_dir - common_str += (""" - config COMMON_SSTATE-DIR - string "sstate_dir (this param is for SSTATE_DIR)" - default "None" - depends on IMAGE - """) - # add tmp_dir - common_str += (""" - config COMMON_TMP-DIR - string "tmp_dir (this param is for tmp_dir)" - default "None" - depends on IMAGE && BUILD_IN-HOST - """) - # add gcc toolchain dir - common_str += (f""" - config COMMON_TOOLCHAIN-DIR - string "toolchain_dir ({toolchain_help})" - default "None" - depends on IMAGE - """) - # add llvm toolchain dir - common_str += (f""" - config COMMON_LLVM-TOOLCHAIN-DIR - string "llvm_toolchain_dir ({llvm_toolchain_help})" - default "None" - depends on IMAGE - """) - # add nativesdk dir - common_str += (f""" - config COMMON_NATIVESDK-DIR - string "nativesdk_dir ({nativesdk_help})" - default "None" - depends on IMAGE && BUILD_IN-HOST - """) - # add timestamp - common_str += (""" - config COMMON_DATETIME - string "datetime" - default "None" - depends on IMAGE - """) - # add cache_src_dir directory - common_str += (""" - config COMMON_CACHE_SRC_DIR - string "cache_src_dir (this param is cache src directory)" - default "None" - depends on IMAGE - """) - # add build directory - common_str += (""" - config COMMON_DIRECTORY - string "directory (this param is build directory)" - default "None" - """) - return common_str - def generate_command(self, config_path): """ - generate_command to oebuild generate - Args: - config_path: - - Returns: - + Parse .config and build argv list for 'oebuild generate'. """ with open(config_path, 'r', encoding='utf-8') as config_file: content = config_file.read() @@ -992,10 +611,7 @@ endif def get_docker_image(yocto_dir, docker_tag, configure: Configure): ''' - get docker image - first we search in yocto-meta-openeuler/.oebuild/env.yaml - second search in default ${workdir}/.oebuild/config.yaml - third get from user input + Resolve docker image from env, config defaults, or user selection, ordered by priority ''' docker_image = oebuild_util.get_docker_image_from_yocto(yocto_dir=yocto_dir) if docker_image is None: @@ -1006,15 +622,13 @@ def get_docker_image(yocto_dir, docker_tag, configure: Configure): else: # select docker image while True: - print(''' -If the system does not recognize which container image to use, select the -following container, enter it numerically, and enter q to exit:''') + print('Select a container image by number (q to quit):') image_list = check_docker_tag.get_tags() for key, value in enumerate(image_list): print( f"{key}, {oebuild_config.docker.repo_url}:{value}") - k = input("please entry number:") + k = input("Enter number: ") if k == "q": sys.exit(0) try: @@ -1022,7 +636,7 @@ following container, enter it numerically, and enter q to exit:''') docker_tag = image_list[index] break except IndexError: - print("please entry true number") + print("Enter a valid number") docker_tag = docker_tag.strip() docker_tag = docker_tag.strip('\n') docker_image = f"{oebuild_config.docker.repo_url}:{docker_tag}" diff --git a/src/oebuild/app/plugins/generate/kconfig_generator.py b/src/oebuild/app/plugins/generate/kconfig_generator.py new file mode 100644 index 0000000..94dcafd --- /dev/null +++ b/src/oebuild/app/plugins/generate/kconfig_generator.py @@ -0,0 +1,351 @@ +""" +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +""" +import os +import pathlib +import re +import subprocess +import sys +import textwrap +import time + +from kconfiglib import Kconfig +from menuconfig import menuconfig + +import oebuild.util as oebuild_util +from oebuild.m_log import logger +from oebuild.app.plugins.generate.parses import parse_feature_files + +class KconfigGenerator: + """Handles Kconfig file generation and menuconfig interaction.""" + + def __init__(self, oebuild_kconfig_path, yocto_oebuild_dir): + self.oebuild_kconfig_path = oebuild_kconfig_path + self.yocto_oebuild_dir = yocto_oebuild_dir + + def create_kconfig(self): + """Generate a temp Kconfig, launch menuconfig, and return the .config path.""" + write_data = "" + + target_list_data, gcc_toolchain_data, llvm_toolchain_data = \ + self._kconfig_add_target_list() + write_data += target_list_data + gcc_toolchain_data + llvm_toolchain_data + + # add auto-build config + write_data += ''' +if NATIVESDK || GCC-TOOLCHAIN || LLVM-TOOLCHAIN +comment "Enable auto build for nativesdk/GCC/LLVM toolchain" + config AUTO-BUILD + bool "auto build" + default n +endif +''' + + platform_data = self._kconfig_add_choice_platform() + write_data += platform_data + + feature_data = self._kconfig_add_feature() + write_data += feature_data + + common_data = self._kconfig_add_common_config() + write_data += common_data + + if not os.path.exists( + pathlib.Path(self.oebuild_kconfig_path).absolute()): + os.makedirs(pathlib.Path(self.oebuild_kconfig_path).absolute()) + kconfig_path = pathlib.Path(self.oebuild_kconfig_path, + str(int(time.time()))) + + with open(kconfig_path, 'w', encoding='utf-8') as kconfig_file: + kconfig_file.write(write_data) + kconf = Kconfig(filename=str(kconfig_path)) + os.environ["MENUCONFIG_STYLE"] = "aquatic selection=fg:white,bg:blue" + with oebuild_util.suppress_print(): + menuconfig(kconf) + subprocess.check_output(f'rm -rf {kconfig_path}', shell=True) + config_path = pathlib.Path(os.getcwd(), '.config') + return config_path + + def _kconfig_add_target_list(self): + target_choice = textwrap.dedent(''' + comment "Select build target" + choice + prompt "Select build target" + config IMAGE + bool 'OS' +''') + gcc_toolchain_data = self._kconfig_add_gcc_toolchain() + if gcc_toolchain_data != "": + target_choice += ( + "config GCC-TOOLCHAIN \n" + " bool 'GCC TOOLCHAIN'\n\n") + llvm_toolchain_data = self._kconfig_add_llvm_toolchain() + if llvm_toolchain_data != "": + target_choice += ( + "config LLVM-TOOLCHAIN \n" + " bool 'LLVM TOOLCHAIN'\n\n" + ) + nativesdk_check = self._kconfig_check_nativesdk() + if nativesdk_check: + target_choice += ( + "config NATIVESDK \n" + " bool 'NATIVESDK'\n\n" + ) + target_choice += "endchoice" + return target_choice, gcc_toolchain_data, llvm_toolchain_data + + def _kconfig_add_choice_platform(self): + """ + add platform to kconfig + Args: + yocto_oebuild_dir: + + Returns: + + """ + platform_path = pathlib.Path(self.yocto_oebuild_dir, 'platform') + if platform_path.exists(): + platform_files = [f for f in platform_path.iterdir() + if f.is_file() and f.suffix in ['.yml', '.yaml']] + else: + logger.error('Platform directory not found.') + sys.exit(-1) + platform_start = textwrap.dedent(''' + if IMAGE + comment "Select OS platform" + choice + prompt "Select platform" + default PLATFORM_QEMU-AARCH64 + ''') + platform_end = """ + endchoice + endif""" + for platform in platform_files: + platform_name = platform.stem.strip("\n") + platform_info = ( + f" config PLATFORM_{platform_name.upper()}\n" + f" bool \"{platform_name}\"\n\n") + platform_start += platform_info + platform_data = platform_start + platform_end + return platform_data + + def _kconfig_add_feature(self): + """ + add feature to kconfig + Args: + yocto_oebuild_dir: + + Returns: + + """ + + feature_start = ''' + if IMAGE + comment "Select OS features" + ''' + + feature_triples = parse_feature_files(self.yocto_oebuild_dir) + for ft_name, _, feature_data in feature_triples: + support_str = "" + if 'support' in feature_data: + raw_supports = feature_data['support'] + validated_support_str = self.validate_and_format_platforms( + raw_supports) + if validated_support_str: + support_str = validated_support_str + else: + logger.warning( + "supported platform str of feat %s is invalid: %s", + ft_name, raw_supports) + + feature_info = (f"\nconfig FEATURE_{ft_name.upper()}\n" + f" bool \"{ft_name}\" {support_str}\n\n") + feature_start += feature_info + feature_start += "endif" + return feature_start + + def validate_and_format_platforms(self, raw_str: str): + """ + strip delimeter and format to supported platforms conditions + """ + + platforms = re.split(r'[|,，\s]+', raw_str) + platforms = [p.strip() for p in platforms if p.strip()] + if not platforms: + return "" + platform_cond = [f"PLATFORM_{p.upper()}" for p in platforms] + return "if " + "||".join(platform_cond) + + def _kconfig_add_gcc_toolchain(self): + """ + add toolchain to kconfig + Args: + yocto_oebuild_dir: yocto_oebuild_dir + + Returns: + + """ + toolchain_start = "" + cross_dir = pathlib.Path(self.yocto_oebuild_dir, "cross-tools") + if cross_dir.exists(): + configs_dir = cross_dir / 'configs' + if configs_dir.exists(): + toolchain_list = os.listdir(configs_dir) + toolchain_start += ''' + if GCC-TOOLCHAIN + ''' + for config in toolchain_list: + if not re.search('xml', config): + toolchain_info = ( + f"""\nconfig GCC-TOOLCHAIN_{config.upper().lstrip("CONFIG_")}\n""" + f""" bool "{config.upper().lstrip("CONFIG_")}"\n""" + """ depends on GCC-TOOLCHAIN\n""") + toolchain_start += toolchain_info + toolchain_start += "endif" + return toolchain_start + + def _kconfig_add_llvm_toolchain(self): + """ + add toolchain to kconfig + Args: + yocto_oebuild_dir: yocto_oebuild_dir + + Returns: + + """ + toolchain_start = "" + llvm_dir = pathlib.Path(self.yocto_oebuild_dir, "llvm-toolchain") + if llvm_dir.exists(): + toolchain_start += ''' + config LLVM-TOOLCHAIN-AARCH64-LIB + string "aarch64 lib dir" + default "None" + depends on LLVM-TOOLCHAIN + ''' + return toolchain_start + + def _kconfig_check_nativesdk(self): + """ + add nativesdk to kconfig + Args: + yocto_oebuild_dir: yocto_oebuild_dir + + Returns: + + """ + nativesdk_dir = pathlib.Path(self.yocto_oebuild_dir, "nativesdk") + return nativesdk_dir.exists() + + def _kconfig_add_common_config(self,): + """ + Build shared/common Kconfig options. + Returns: + + """ + toolchain_help = ( + "External GCC toolchain directory [your own toolchain]") + llvm_toolchain_help = ( + "External LLVM toolchain directory [your own toolchain]") + nativesdk_help = ( + "External nativesdk directory [used when building on host]") + common_str = textwrap.dedent(''' + comment "Common options" + ''') + # choice build in platform + common_str += (''' + if IMAGE + choice + prompt "Select build environment" + default BUILD_IN-DOCKER + config BUILD_IN-DOCKER + bool "docker" + config BUILD_IN-HOST + bool "host" + endchoice + endif + ''') + # add no fetch + common_str += (''' + config COMMON_NO-FETCH + bool "no_fetch (disable source fetching)" + default n + depends on IMAGE + ''') + # add no layer + common_str += (''' + config COMMON_NO-LAYER + bool "no_layer (skip layer repo update on env setup)" + default n + depends on IMAGE + ''') + # add sstate_mirrors + common_str += (''' + config COMMON_SSTATE-MIRRORS + string "SSTATE_MIRRORS value" + default "None" + depends on IMAGE + ''') + # add sstate_dir + common_str += (''' + config COMMON_SSTATE-DIR + string "SSTATE_DIR path" + default "None" + depends on IMAGE + ''') + # add tmp_dir + common_str += (''' + config COMMON_TMP-DIR + string "TMPDIR path" + default "None" + depends on IMAGE && BUILD_IN-HOST + ''') + # add gcc toolchain dir + common_str += (f''' + config COMMON_TOOLCHAIN-DIR + string "toolchain_dir ({toolchain_help})" + default "None" + depends on IMAGE + ''') + # add llvm toolchain dir + common_str += (f''' + config COMMON_LLVM-TOOLCHAIN-DIR + string "llvm_toolchain_dir ({llvm_toolchain_help})" + default "None" + depends on IMAGE + ''') + # add nativesdk dir + common_str += (f''' + config COMMON_NATIVESDK-DIR + string "nativesdk_dir ({nativesdk_help})" + default "None" + depends on IMAGE && BUILD_IN-HOST + ''') + # add timestamp + common_str += (''' + config COMMON_DATETIME + string "datetime" + default "None" + depends on IMAGE + ''') + # add cache_src_dir directory + common_str += (''' + config COMMON_CACHE_SRC_DIR + string "cache_src_dir (src directory)" + default "None" + depends on IMAGE + ''') + # add build directory + common_str += (''' + config COMMON_DIRECTORY + string "directory (build directory name)" + default "None" + ''') + return common_str diff --git a/src/oebuild/app/plugins/generate/parses.py b/src/oebuild/app/plugins/generate/parses.py index 029825e..c1b9f66 100644 --- a/src/oebuild/app/plugins/generate/parses.py +++ b/src/oebuild/app/plugins/generate/parses.py @@ -10,8 +10,13 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -import oebuild.const as oebuild_const +import pathlib +import sys +from ruamel.yaml.error import YAMLError, MarkedYAMLError +import oebuild.const as oebuild_const +from oebuild.m_log import logger +import oebuild.util as oebuild_util def parsers(parser): ''' @@ -22,7 +27,7 @@ def parsers(parser): dest='list', action="store_true", help=''' - will list support archs and features + list supported archs and features ''') parser.add_argument('-p', @@ -186,3 +191,53 @@ def parsers(parser): ''') return parser + + +@staticmethod +def parse_feature_files(oebuild_dir): + """ + both yaml and yml are supported, but yaml is preferred + """ + + feature_dir = pathlib.Path(oebuild_dir, 'features') + if not feature_dir.exists(): + logger.error("Features directory not found under .oebuild.") + sys.exit(-1) + # Group files by base name to handle priority + file_groups = {} + for ft_path in feature_dir.iterdir(): + if ft_path.is_file(): + if ft_path.suffix == '.yaml': + feature_name = ft_path.stem + if feature_name not in file_groups: + file_groups[feature_name] = {'yaml': None, 'yml': None} + file_groups[feature_name]['yaml'] = ft_path + elif ft_path.suffix == '.yml': + feature_name = ft_path.stem + if feature_name not in file_groups: + file_groups[feature_name] = {'yaml': None, 'yml': None} + file_groups[feature_name]['yml'] = ft_path + features = [] + for feature_name, extensions in file_groups.items(): + # Prefer .yaml over .yml + selected_file = extensions['yaml'] or extensions['yml'] + if selected_file: + try: + feature_data = oebuild_util.read_yaml(selected_file) + if not isinstance(feature_data, dict): + logger.warning("Invalid YAML mapping in feature '%s'", feature_name) + continue + features.append((feature_name, selected_file, feature_data)) + except MarkedYAMLError as e: + fname = selected_file.name + logger.warning( + """Failed to parse feature file '%s': + Error: %s + At: Line %d, Column %d, + """, fname, e.problem, e.problem_mark.line + 1, e.problem_mark.column + 1) + continue + except YAMLError as e: + fname = selected_file.name + logger.warning("Failed to parse feature file '%s': %s", fname, e) + continue + return features -- Gitee From 2bb271006e05c445014017817b6fb7385b60ed91 Mon Sep 17 00:00:00 2001 From: Egg12138 Date: Wed, 15 Oct 2025 10:50:14 +0800 Subject: [PATCH 5/5] feat: update oebuild generate features/arch display * add scalling logic for list_feature to avoid chaos when terminal is too narrow * add a constructor for platform and feature table, make sure the style consistency, and easy display style customization * order feature list, now menuconfig and `oebuild generate -l` are sorted by feature name Signed-off-by: egg12138 --- src/oebuild/app/plugins/generate/generate.py | 49 ++++++++++++++++--- .../app/plugins/generate/kconfig_generator.py | 1 + src/oebuild/app/plugins/generate/parses.py | 2 + 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index dae4fa7..6ba15db 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -19,7 +19,7 @@ import sys import pathlib from shutil import rmtree -from prettytable import PrettyTable +from prettytable import PrettyTable, TableStyle, HRuleStyle, VRuleStyle from ruamel.yaml.scalarstring import LiteralScalarString from oebuild.command import OebuildCommand @@ -82,7 +82,7 @@ class Generate(OebuildCommand): yocto_dir = self.configure.source_yocto_dir() if not self.check_support_oebuild(yocto_dir): logger.error( - 'yocto-meta-openeuler does not support oebuild. ' + 'yocto-meta-openeuler does not container valid oebuild metadata ' 'Update .oebuild/config and re-run `oebuild update`.') sys.exit(-1) @@ -352,11 +352,12 @@ oebuild toolchain yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') list_platform = [f for f in platform_path.iterdir() if f.is_file()] - table = PrettyTable(['Platform Name']) - table.align = "l" + terminal_width = self._get_terminal_width() + table = self._build_table(['Platform Name'], terminal_width, title='Available Platforms') for platform in list_platform: if platform.suffix in ['.yml', '.yaml']: table.add_row([platform.stem]) + table.sortby = 'Platform Name' print(table) def _list_feature(self): @@ -364,8 +365,9 @@ oebuild toolchain yocto_dir = self.configure.source_yocto_dir() yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") feature_triples = parse_feature_files(yocto_oebuild_dir) - table = PrettyTable(['Feature Name', 'Supported Arch']) - table.align = "l" + terminal_width = self._get_terminal_width() + table = self._build_table(['Feature Name', 'Supported Arch'], + terminal_width, title='Available Features') for feature_name, _, feature_data in feature_triples: table.add_row([feature_name, feature_data.get('support') or "all"]) print(table) @@ -373,6 +375,41 @@ oebuild toolchain """* 'Supported Arch' defaults to 'all' if not specified in the feature's .yaml file.""" ) + def _build_table(self, headers, terminal_width, title=None): + narrow_charnum, narrow_colnum = 60, 10 + max_width = max(int(terminal_width * 0.9), 20) + table = PrettyTable(headers, max_width=max_width) + table.align = "l" + table.header = True + + col_width = max(10, max_width // max(len(headers), 1)) + for header in headers: + table.max_width[header] = col_width + + is_narrow = terminal_width < narrow_charnum or col_width < narrow_colnum + if is_narrow: + table.set_style(TableStyle.PLAIN_COLUMNS) + table.hrules = HRuleStyle.NONE + table.vrules = VRuleStyle.NONE + table.left_padding_width = 0 + table.right_padding_width = 0 + else: + table.set_style(TableStyle.SINGLE_BORDER) + table.hrules = HRuleStyle.FRAME + table.vrules = VRuleStyle.FRAME + table.left_padding_width = 1 + table.right_padding_width = 1 + if title: + table.title = title + return table + + @staticmethod + def _get_terminal_width(): + try: + return os.get_terminal_size().columns + except OSError: + return 80 + def check_support_oebuild(self, yocto_dir): ''' Return True if /.oebuild exists. diff --git a/src/oebuild/app/plugins/generate/kconfig_generator.py b/src/oebuild/app/plugins/generate/kconfig_generator.py index 94dcafd..b786ae6 100644 --- a/src/oebuild/app/plugins/generate/kconfig_generator.py +++ b/src/oebuild/app/plugins/generate/kconfig_generator.py @@ -24,6 +24,7 @@ import oebuild.util as oebuild_util from oebuild.m_log import logger from oebuild.app.plugins.generate.parses import parse_feature_files + class KconfigGenerator: """Handles Kconfig file generation and menuconfig interaction.""" diff --git a/src/oebuild/app/plugins/generate/parses.py b/src/oebuild/app/plugins/generate/parses.py index c1b9f66..59f80aa 100644 --- a/src/oebuild/app/plugins/generate/parses.py +++ b/src/oebuild/app/plugins/generate/parses.py @@ -18,6 +18,7 @@ import oebuild.const as oebuild_const from oebuild.m_log import logger import oebuild.util as oebuild_util + def parsers(parser): ''' for generate param parser @@ -240,4 +241,5 @@ def parse_feature_files(oebuild_dir): fname = selected_file.name logger.warning("Failed to parse feature file '%s': %s", fname, e) continue + features.sort(key=lambda x: x[0]) return features -- Gitee