diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index 3b0caa24db5429b6ef8504660d8ecf4323655132..6ba15dbad9a778d59812c203d0e8fd6452a54050 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -17,43 +17,37 @@ import textwrap import os import sys import pathlib -import time from shutil import rmtree -from kconfiglib import Kconfig -from menuconfig import menuconfig -from prettytable import PrettyTable +from prettytable import PrettyTable, TableStyle, HRuleStyle, VRuleStyle from ruamel.yaml.scalarstring import LiteralScalarString from oebuild.command import OebuildCommand import oebuild.util as oebuild_util from oebuild.configure import Configure -from oebuild.parse_template import BaseParseTemplate, ParseTemplate, \ - get_docker_param_dict, parse_repos_layers_local_obj +from oebuild.parse_template import ( + BaseParseTemplate, ParseTemplate, + get_docker_param_dict, + parse_repos_layers_local_obj) from oebuild.m_log import logger from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const -from oebuild.app.plugins.generate.parses import parsers +from oebuild.app.plugins.generate.parses import parsers, parse_feature_files +from oebuild.app.plugins.generate.kconfig_generator import KconfigGenerator class Generate(OebuildCommand): - ''' - compile.yaml is generated according to different command parameters by generate - ''' + """Generate compile.yaml (and toolchain.yaml) from CLI options.""" - help_msg = 'help to mkdir build directory and generate compile.yaml' + help_msg = 'Create build dir and generate compile.yaml' description = textwrap.dedent('''\ - The generate command is the core command in the entire build process, which - is mainly used to customize the build configuration parameters and generate - a compile.yaml by customizing each parameter. In addition, for a large number - of configuration parameter input is not very convenient, generate provides a - way to specify compile.yaml, users can directly specify the file after - customizing the build configuration file + Customize build parameters and output compile.yaml; optionally + emit toolchain.yaml for GCC/LLVM builds. ''') def __init__(self): self.configure = Configure() - # params is use for record generate command param, had follow params + # Cache of parsed CLI parameters affecting generation # nativesdk_dir # toolchain_dir # llvm_toolchain_dir @@ -67,8 +61,8 @@ class Generate(OebuildCommand): super().__init__('generate', self.help_msg, self.description) def do_add_parser(self, parser_adder): + """Add arguments to the parser.""" parser = self._parser(parser_adder, usage=''' - %(prog)s ''') @@ -78,7 +72,7 @@ class Generate(OebuildCommand): # pylint:disable=[R0914,R0911,R0912,R0915,W1203,R0913] def do_run(self, args: argparse.Namespace, unknown=None): - # perpare parse help command + """The main entry point for the command.""" if self.pre_parse_help(args, unknown): sys.exit(0) if not self.configure.is_oebuild_dir(): @@ -88,17 +82,18 @@ class Generate(OebuildCommand): yocto_dir = self.configure.source_yocto_dir() if not self.check_support_oebuild(yocto_dir): logger.error( - 'Currently, yocto-meta-openeuler does not support oebuild, \ - please modify .oebuild/config and re-execute `oebuild update`' - ) + 'yocto-meta-openeuler does not container valid oebuild metadata ' + 'Update .oebuild/config and re-run `oebuild update`.') sys.exit(-1) if len(unknown) == 0: - config_path = self.create_kconfig(yocto_dir) + yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') + kconfig_generator = KconfigGenerator( + self.oebuild_kconfig_path, yocto_oebuild_dir) + config_path = kconfig_generator.create_kconfig() if not os.path.exists(config_path): sys.exit(0) g_command = self.generate_command(config_path) - # sys.exit(0) subprocess.check_output(f'rm -rf {config_path}', shell=True) args = args.parse_args(g_command) else: @@ -106,7 +101,7 @@ class Generate(OebuildCommand): auto_build = bool(args.auto_build) if args.nativesdk: - # this is for default directory nativesdk + # default dir for nativesdk if args.directory is None or args.directory == '': args.directory = "nativesdk" build_dir = self._init_build_dir(args=args) @@ -117,7 +112,7 @@ class Generate(OebuildCommand): sys.exit(0) if args.gcc: - # this is for default directory toolchain + # default dir for toolchain if args.directory is None or args.directory == '': args.directory = "toolchain" toolchain_name_list = args.gcc_name if args.gcc_name else [] @@ -129,7 +124,7 @@ class Generate(OebuildCommand): sys.exit(0) if args.llvm: - # this is for default directory is toolchain + # default dir for toolchain if args.directory is None or args.directory == '': args.directory = "toolchain" llvm_lib = args.llvm_lib @@ -172,7 +167,7 @@ class Generate(OebuildCommand): parser_template = ParseTemplate(yocto_dir=yocto_dir) - yocto_oebuild_dir = os.path.join(yocto_dir, '.oebuild') + yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') try: self._add_platform_template(args=args, @@ -197,15 +192,16 @@ class Generate(OebuildCommand): logger.error(str(v_e)) sys.exit(-1) - if os.path.exists(os.path.join(build_dir, 'compile.yaml')): - os.remove(os.path.join(build_dir, 'compile.yaml')) + compile_yaml_path = pathlib.Path(build_dir, 'compile.yaml') + if compile_yaml_path.exists(): + compile_yaml_path.unlink() docker_image = get_docker_image( yocto_dir=self.configure.source_yocto_dir(), docker_tag='', configure=self.configure) - out_dir = pathlib.Path(os.path.join(build_dir, 'compile.yaml')) + out_dir = pathlib.Path(build_dir, 'compile.yaml') param = parser_template.get_default_generate_compile_conf_param() param['nativesdk_dir'] = self.params.get('nativesdk_dir', None) @@ -232,7 +228,7 @@ class Generate(OebuildCommand): format_dir = f''' generate compile.yaml successful -please run follow command: +Run commands below: ============================================= cd {build_dir} @@ -246,7 +242,7 @@ oebuild bitbake format_dir = f''' generate compile.yaml successful -please run follow command: +Run commands below: ============================================= cd {build_dir} @@ -260,7 +256,7 @@ oebuild bitbake or oebuild bitbake buildtools-extended-tarball format_dir = f''' generate toolchain.yaml successful -please run follow command: +Run commands below: ============================================= cd {build_dir} @@ -272,78 +268,76 @@ oebuild toolchain def _add_platform_template(self, args, yocto_oebuild_dir, parser_template: ParseTemplate): - if args.platform + '.yaml' in os.listdir( - os.path.join(yocto_oebuild_dir, 'platform')): + platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') + platform_files = [f.name for f in platform_path.iterdir() if f.is_file()] + if args.platform + '.yaml' in platform_files: try: - parser_template.add_template( - os.path.join(yocto_oebuild_dir, 'platform', - args.platform + '.yaml')) + platform_file = platform_path / (args.platform + '.yaml') + parser_template.add_template(platform_file) except BaseParseTemplate as e_p: raise e_p else: - logger.error(""" -wrong platform, please run `oebuild generate -l` to view support platform""") + logger.error( + "Invalid platform. Run `oebuild generate -l` to list supported platforms.") sys.exit(-1) def _add_features_template(self, args, yocto_oebuild_dir, parser_template: ParseTemplate): if args.features: + features_path = pathlib.Path(yocto_oebuild_dir, 'features') + feature_files = [f.name for f in features_path.iterdir() if f.is_file()] for feature in args.features: - if feature + '.yaml' in os.listdir( - os.path.join(yocto_oebuild_dir, 'features')): + if feature + '.yaml' in feature_files: try: - parser_template.add_template( - os.path.join(yocto_oebuild_dir, 'features', - feature + '.yaml')) + feature_file = features_path / (feature + '.yaml') + parser_template.add_template(feature_file) except BaseParseTemplate as b_t: raise b_t else: - logger.error(""" -Wrong platform, please run `oebuild generate -l` to view support feature""") + logger.error( + "Invalid feature. Run `oebuild generate -l` to list features.") sys.exit(-1) def _init_build_dir(self, args): - if not os.path.exists(self.configure.build_dir()): - os.makedirs(self.configure.build_dir()) + build_dir_path = pathlib.Path(self.configure.build_dir()) + if not build_dir_path.exists(): + build_dir_path.mkdir(parents=True) if args.directory is None or args.directory == '': - build_dir = os.path.join(self.configure.build_dir(), args.platform) + build_dir = build_dir_path / args.platform else: - build_dir = os.path.join(self.configure.build_dir(), - args.directory) + build_dir = build_dir_path / args.directory - if not os.path.abspath(build_dir).startswith( - self.configure.build_dir()): + if not pathlib.Path(build_dir).absolute().is_relative_to(build_dir_path.absolute()): logger.error("Build path must in oebuild workspace") return None - # detects if a build directory already exists - if os.path.exists(build_dir): + # If build dir exists, prompt/handle overwrite + if build_dir.exists(): logger.warning("the build directory %s already exists", build_dir) while not args.yes: in_res = input(f""" - do you want to overwrite it({os.path.basename(build_dir)})? the overwrite action - will replace the compile.yaml or toolchain.yaml to new and delete conf directory, - enter Y for yes, N for no, C for create:""") + Overwrite {build_dir.name}? This will replace compile.yaml/toolchain.yaml and delete conf/ + Enter Y=yes, N=no, C=create : """) if in_res not in ["Y", "y", "yes", "N", "n", "no", "C", "c", "create"]: - print(""" - wrong input""") + print("Invalid input") continue if in_res in ['N', 'n', 'no']: return None if in_res in ['C', 'c', 'create']: - in_res = input(""" - please enter now build name, we will create it under build directory:""") - build_dir = os.path.join(self.configure.build_dir(), in_res) - if os.path.exists(build_dir): + in_res = input( + "Enter new build name (will be created under build/):") + build_dir = build_dir_path / in_res + if build_dir.exists(): continue break - if os.path.exists(os.path.join(build_dir, "conf")): - rmtree(os.path.join(build_dir, "conf")) - elif os.path.exists(build_dir): + conf_dir = build_dir / "conf" + if conf_dir.exists(): + rmtree(conf_dir) + elif build_dir.exists(): rmtree(build_dir) - os.makedirs(build_dir, exist_ok=True) - return build_dir + build_dir.mkdir(parents=True, exist_ok=True) + return str(build_dir) def list_info(self, ): ''' @@ -353,368 +347,78 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") self._list_feature() def _list_platform(self): - logger.info("=============================================") + logger.info("\n================= Available Platforms =================") yocto_dir = self.configure.source_yocto_dir() - yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") - list_platform = os.listdir(os.path.join(yocto_oebuild_dir, 'platform')) - print("the platform list is:") - table = PrettyTable(['platform name']) - table.align = "l" + yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") + platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') + list_platform = [f for f in platform_path.iterdir() if f.is_file()] + terminal_width = self._get_terminal_width() + table = self._build_table(['Platform Name'], terminal_width, title='Available Platforms') for platform in list_platform: - if platform.endswith('.yml'): - table.add_row([platform.replace('.yml', '')]) - if platform.endswith('.yaml'): - table.add_row([platform.replace('.yaml', '')]) + if platform.suffix in ['.yml', '.yaml']: + table.add_row([platform.stem]) + table.sortby = 'Platform Name' print(table) - def _list_feature(self, ): + def _list_feature(self): + logger.info("\n================= Available Features ==================") yocto_dir = self.configure.source_yocto_dir() - yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") - list_feature = os.listdir(os.path.join(yocto_oebuild_dir, 'features')) - print("the feature list is:") - table = PrettyTable(['feature name', 'support arch']) - table.align = "l" - for feature in list_feature: - feature_name = feature.replace('.yml', '') - if feature.endswith('.yaml'): - feature_name = feature.replace('.yaml', '') - feat = oebuild_util.read_yaml( - pathlib.Path( - os.path.join(yocto_oebuild_dir, 'features', feature))) - if "support" in feat: - table.add_row([feature_name, feat.get('support')]) - else: - table.add_row([feature_name, "all"]) + yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") + feature_triples = parse_feature_files(yocto_oebuild_dir) + terminal_width = self._get_terminal_width() + table = self._build_table(['Feature Name', 'Supported Arch'], + terminal_width, title='Available Features') + for feature_name, _, feature_data in feature_triples: + table.add_row([feature_name, feature_data.get('support') or "all"]) print(table) + logger.info( + """* 'Supported Arch' defaults to 'all' if not specified in the feature's .yaml file.""" + ) + + def _build_table(self, headers, terminal_width, title=None): + narrow_charnum, narrow_colnum = 60, 10 + max_width = max(int(terminal_width * 0.9), 20) + table = PrettyTable(headers, max_width=max_width) + table.align = "l" + table.header = True + + col_width = max(10, max_width // max(len(headers), 1)) + for header in headers: + table.max_width[header] = col_width + + is_narrow = terminal_width < narrow_charnum or col_width < narrow_colnum + if is_narrow: + table.set_style(TableStyle.PLAIN_COLUMNS) + table.hrules = HRuleStyle.NONE + table.vrules = VRuleStyle.NONE + table.left_padding_width = 0 + table.right_padding_width = 0 + else: + table.set_style(TableStyle.SINGLE_BORDER) + table.hrules = HRuleStyle.FRAME + table.vrules = VRuleStyle.FRAME + table.left_padding_width = 1 + table.right_padding_width = 1 + if title: + table.title = title + return table + + @staticmethod + def _get_terminal_width(): + try: + return os.get_terminal_size().columns + except OSError: + return 80 def check_support_oebuild(self, yocto_dir): ''' - Determine if OeBuild is supported by checking if .oebuild - exists in the yocto-meta-openeuler directory + Return True if /.oebuild exists. ''' - return os.path.exists(os.path.join(yocto_dir, '.oebuild')) - - def create_kconfig(self, yocto_dir): - """ - create_kconfig - Returns: - - """ - yocto_oebuild_dir = os.path.join(yocto_dir, '.oebuild') - write_data = "" - - target_list_data, gcc_toolchain_data, llvm_toolchain_data = \ - self._kconfig_add_target_list(yocto_oebuild_dir) - write_data += target_list_data + gcc_toolchain_data + llvm_toolchain_data - - # add auto-build config - write_data += """ -if NATIVESDK || GCC-TOOLCHAIN || LLVM-TOOLCHAIN -comment "if auto build nativesdk or gcc toolchain or llvm toolchain" - config AUTO-BUILD - bool "auto build" - default n -endif -""" - - platform_data = self._kconfig_add_choice_platform(yocto_oebuild_dir) - write_data += platform_data - - feature_data = self._kconfig_add_feature(yocto_oebuild_dir) - write_data += feature_data - - common_data = self._kconfig_add_common_config() - write_data += common_data - - if not os.path.exists( - pathlib.Path(self.oebuild_kconfig_path).absolute()): - os.makedirs(pathlib.Path(self.oebuild_kconfig_path).absolute()) - kconfig_path = pathlib.Path(self.oebuild_kconfig_path, - str(int(time.time()))) - - with open(kconfig_path, 'w', encoding='utf-8') as kconfig_file: - kconfig_file.write(write_data) - kconf = Kconfig(filename=kconfig_path) - os.environ["MENUCONFIG_STYLE"] = "aquatic selection=fg:white,bg:blue" - with oebuild_util.suppress_print(): - menuconfig(kconf) - subprocess.check_output(f'rm -rf {kconfig_path}', shell=True) - config_path = pathlib.Path(os.getcwd(), '.config') - return config_path - - def _kconfig_add_target_list(self, yocto_oebuild_dir): - target_choice = textwrap.dedent(""" - comment "this is choose build target" - choice - prompt "choice build target" - config IMAGE\n - bool 'OS'\n -""") - gcc_toolchain_data = self._kconfig_add_gcc_toolchain(yocto_oebuild_dir) - if gcc_toolchain_data != "": - target_choice += ( - "config GCC-TOOLCHAIN \n" - " bool 'GCC TOOLCHAIN'\n\n") - llvm_toolchain_data = self._kconfig_add_llvm_toolchain(yocto_oebuild_dir) - if llvm_toolchain_data != "": - target_choice += ( - "config LLVM-TOOLCHAIN \n" - " bool 'LLVM TOOLCHAIN'\n\n" - ) - nativesdk_check = self._kconfig_check_nativesdk(yocto_oebuild_dir) - if nativesdk_check: - target_choice += ( - "config NATIVESDK \n" - " bool 'NATIVESDK'\n\n" - ) - target_choice += "endchoice" - return target_choice, gcc_toolchain_data, llvm_toolchain_data - - def _kconfig_add_choice_platform(self, yocto_oebuild_dir): - """ - add platform to kconfig - Args: - yocto_oebuild_dir: - - Returns: - - """ - platform_path = os.path.join(yocto_oebuild_dir, 'platform') - if os.path.exists(platform_path): - platform_list = os.listdir(platform_path) - else: - logger.error('platform dir is not exists') - sys.exit(-1) - platform_start = textwrap.dedent(""" - if IMAGE - comment "this is choose OS platform" - choice - prompt "choice platform" - default PLATFORM_QEMU-AARCH64\n - """) - platform_end = """ - endchoice - endif""" - for platform in platform_list: - platform_name = os.path.splitext(platform)[0].strip("\n") - platform_info = ( - f""" config PLATFORM_{platform_name.upper()}\n""" - f""" bool "{platform_name}"\n\n""") - platform_start += platform_info - platform_data = platform_start + platform_end - return platform_data - - def _kconfig_add_feature(self, yocto_oebuild_dir): - """ - add feature to kconfig - Args: - yocto_oebuild_dir: - - Returns: - - """ - feature_path = os.path.join(yocto_oebuild_dir, 'features') - if os.path.exists(feature_path): - feature_list = os.listdir(feature_path) - else: - logger.error('feature dir is not exists') - sys.exit(-1) - feature_start = """ - if IMAGE - comment "this is choose OS features" - """ - for feature in feature_list: - support_str = "" - feature_path = pathlib.Path(yocto_oebuild_dir, 'features', feature) - feature_data = oebuild_util.read_yaml(feature_path) - feature_name = os.path.splitext(feature)[0].strip("\n") - if 'support' in feature_data: - support_str = ("if PLATFORM_" + - feature_data['support'].upper().replace( - '|', '||PLATFORM_')) - - feature_info = (f"""\nconfig FEATURE_{feature_name.upper()}\n""" - f""" bool "{feature_name}" {support_str}\n\n""") - feature_start += feature_info - feature_start += "endif" - - return feature_start - - def _kconfig_add_gcc_toolchain(self, yocto_oebuild_dir): - """ - add toolchain to kconfig - Args: - yocto_oebuild_dir: yocto_oebuild_dir - - Returns: - - """ - toolchain_start = "" - cross_dir = os.path.join(yocto_oebuild_dir, "cross-tools") - if os.path.exists(cross_dir): - toolchain_list = os.listdir(os.path.join(cross_dir, 'configs')) - toolchain_start += """ - if GCC-TOOLCHAIN - """ - for config in toolchain_list: - if not re.search('xml', config): - toolchain_info = ( - f"""\nconfig GCC-TOOLCHAIN_{config.upper().lstrip("CONFIG_")}\n""" - f""" bool "{config.upper().lstrip("CONFIG_")}"\n""" - """ depends on GCC-TOOLCHAIN\n""") - toolchain_start += toolchain_info - toolchain_start += "endif" - return toolchain_start - - def _kconfig_add_llvm_toolchain(self, yocto_oebuild_dir): - """ - add toolchain to kconfig - Args: - yocto_oebuild_dir: yocto_oebuild_dir - - Returns: - - """ - toolchain_start = "" - llvm_dir = os.path.join(yocto_oebuild_dir, "llvm-toolchain") - if os.path.exists(llvm_dir): - toolchain_start += """ - config LLVM-TOOLCHAIN_AARCH64-LIB - string "aarch64 lib dir" - default "None" - depends on LLVM-TOOLCHAIN - """ - return toolchain_start - - def _kconfig_check_nativesdk(self, yocto_oebuild_dir): - """ - add nativesdk to kconfig - Args: - yocto_oebuild_dir: yocto_oebuild_dir - - Returns: - - """ - nativesdk_dir = os.path.join(yocto_oebuild_dir, "nativesdk") - return os.path.exists(nativesdk_dir) - - def _kconfig_add_common_config(self,): - """ - Kconfig basic_config - Returns: - - """ - toolchain_help = ( - "this param is for external gcc toolchain dir, " - "if you want use your own toolchain") - llvm_toolchain_help = ( - "this param is for external llvm toolchain dir, " - "if you want use your own toolchain") - nativesdk_help = ( - "this param is for external nativesdk dir," - "the param will be useful when you want to build in host") - common_str = textwrap.dedent(""" - comment "this is common config" - """) - # choice build in platform - common_str += (""" - if IMAGE - choice - prompt "choice build in docker or host" - default BUILD_IN-DOCKER - config BUILD_IN-DOCKER - bool "docker" - config BUILD_IN-HOST - bool "host" - endchoice - endif - """) - # add no fetch - common_str += (""" - config COMMON_NO-FETCH - bool "no_fetch (this will set openeuler_fetch to disable)" - default n - depends on IMAGE - """) - # add no layer - common_str += (""" - config COMMON_NO-LAYER - bool "no_layer (this will disable layer repo update when startting bitbake environment)" - default n - depends on IMAGE - """) - # add sstate_mirrors - common_str += (""" - config COMMON_SSTATE-MIRRORS - string "sstate_mirrors (this param is for SSTATE_MIRRORS)" - default "None" - depends on IMAGE - """) - # add sstate_dir - common_str += (""" - config COMMON_SSTATE-DIR - string "sstate_dir (this param is for SSTATE_DIR)" - default "None" - depends on IMAGE - """) - # add tmp_dir - common_str += (""" - config COMMON_TMP-DIR - string "tmp_dir (this param is for tmp_dir)" - default "None" - depends on IMAGE && BUILD_IN-HOST - """) - # add gcc toolchain dir - common_str += (f""" - config COMMON_TOOLCHAIN-DIR - string "toolchain_dir ({toolchain_help})" - default "None" - depends on IMAGE - """) - # add llvm toolchain dir - common_str += (f""" - config COMMON_LLVM-TOOLCHAIN-DIR - string "llvm_toolchain_dir ({llvm_toolchain_help})" - default "None" - depends on IMAGE - """) - # add nativesdk dir - common_str += (f""" - config COMMON_NATIVESDK-DIR - string "nativesdk_dir ({nativesdk_help})" - default "None" - depends on IMAGE && BUILD_IN-HOST - """) - # add timestamp - common_str += (""" - config COMMON_DATETIME - string "datetime" - default "None" - depends on IMAGE - """) - # add cache_src_dir directory - common_str += (""" - config COMMON_CACHE_SRC_DIR - string "cache_src_dir (this param is cache src directory)" - default "None" - depends on IMAGE - """) - # add build directory - common_str += (""" - config COMMON_DIRECTORY - string "directory (this param is build directory)" - default "None" - """) - return common_str + return pathlib.Path(yocto_dir, '.oebuild').exists() def generate_command(self, config_path): """ - generate_command to oebuild generate - Args: - config_path: - - Returns: - + Parse .config and build argv list for 'oebuild generate'. """ with open(config_path, 'r', encoding='utf-8') as config_file: content = config_file.read() @@ -831,8 +535,8 @@ endif Returns: """ - source_cross_dir = os.path.join(self.configure.source_yocto_dir(), ".oebuild/cross-tools") - if not os.path.exists(source_cross_dir): + source_cross_dir = pathlib.Path(self.configure.source_yocto_dir(), ".oebuild/cross-tools") + if not source_cross_dir.exists(): logger.error('Build dependency not downloaded, not supported for build. Please ' 'download the latest yocto meta openeuler repository') sys.exit(-1) @@ -854,7 +558,7 @@ endif continue config_list.append("config_" + gcc_name) oebuild_util.write_yaml( - yaml_path=os.path.join(build_dir, 'toolchain.yaml'), + yaml_path=pathlib.Path(build_dir, 'toolchain.yaml'), data={ 'kind': oebuild_const.GCC_TOOLCHAIN, 'gcc_configs': config_list, @@ -894,8 +598,8 @@ endif Returns: """ - source_llvm_dir = os.path.join(self.configure.source_yocto_dir(), ".oebuild/llvm-toolchain") - if not os.path.exists(source_llvm_dir): + source_llvm_dir = pathlib.Path(self.configure.source_yocto_dir(), ".oebuild/llvm-toolchain") + if not source_llvm_dir.exists(): logger.error('Build dependency not downloaded, not supported for build. Please ' 'download the latest yocto meta openeuler repository') sys.exit(-1) @@ -911,7 +615,7 @@ endif } ) oebuild_util.write_yaml( - yaml_path=os.path.join(build_dir, 'toolchain.yaml'), + yaml_path=pathlib.Path(build_dir, 'toolchain.yaml'), data={ 'kind': oebuild_const.LLVM_TOOLCHAIN, 'llvm_lib': llvm_lib, @@ -944,10 +648,7 @@ endif def get_docker_image(yocto_dir, docker_tag, configure: Configure): ''' - get docker image - first we search in yocto-meta-openeuler/.oebuild/env.yaml - second search in default ${workdir}/.oebuild/config.yaml - third get from user input + Resolve docker image from env, config defaults, or user selection, ordered by priority ''' docker_image = oebuild_util.get_docker_image_from_yocto(yocto_dir=yocto_dir) if docker_image is None: @@ -958,15 +659,13 @@ def get_docker_image(yocto_dir, docker_tag, configure: Configure): else: # select docker image while True: - print(''' -If the system does not recognize which container image to use, select the -following container, enter it numerically, and enter q to exit:''') + print('Select a container image by number (q to quit):') image_list = check_docker_tag.get_tags() for key, value in enumerate(image_list): print( f"{key}, {oebuild_config.docker.repo_url}:{value}") - k = input("please entry number:") + k = input("Enter number: ") if k == "q": sys.exit(0) try: @@ -974,7 +673,7 @@ following container, enter it numerically, and enter q to exit:''') docker_tag = image_list[index] break except IndexError: - print("please entry true number") + print("Enter a valid number") docker_tag = docker_tag.strip() docker_tag = docker_tag.strip('\n') docker_image = f"{oebuild_config.docker.repo_url}:{docker_tag}" diff --git a/src/oebuild/app/plugins/generate/kconfig_generator.py b/src/oebuild/app/plugins/generate/kconfig_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..b786ae64dc499ae137ea7d5119564972144ff266 --- /dev/null +++ b/src/oebuild/app/plugins/generate/kconfig_generator.py @@ -0,0 +1,352 @@ +""" +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +""" +import os +import pathlib +import re +import subprocess +import sys +import textwrap +import time + +from kconfiglib import Kconfig +from menuconfig import menuconfig + +import oebuild.util as oebuild_util +from oebuild.m_log import logger +from oebuild.app.plugins.generate.parses import parse_feature_files + + +class KconfigGenerator: + """Handles Kconfig file generation and menuconfig interaction.""" + + def __init__(self, oebuild_kconfig_path, yocto_oebuild_dir): + self.oebuild_kconfig_path = oebuild_kconfig_path + self.yocto_oebuild_dir = yocto_oebuild_dir + + def create_kconfig(self): + """Generate a temp Kconfig, launch menuconfig, and return the .config path.""" + write_data = "" + + target_list_data, gcc_toolchain_data, llvm_toolchain_data = \ + self._kconfig_add_target_list() + write_data += target_list_data + gcc_toolchain_data + llvm_toolchain_data + + # add auto-build config + write_data += ''' +if NATIVESDK || GCC-TOOLCHAIN || LLVM-TOOLCHAIN +comment "Enable auto build for nativesdk/GCC/LLVM toolchain" + config AUTO-BUILD + bool "auto build" + default n +endif +''' + + platform_data = self._kconfig_add_choice_platform() + write_data += platform_data + + feature_data = self._kconfig_add_feature() + write_data += feature_data + + common_data = self._kconfig_add_common_config() + write_data += common_data + + if not os.path.exists( + pathlib.Path(self.oebuild_kconfig_path).absolute()): + os.makedirs(pathlib.Path(self.oebuild_kconfig_path).absolute()) + kconfig_path = pathlib.Path(self.oebuild_kconfig_path, + str(int(time.time()))) + + with open(kconfig_path, 'w', encoding='utf-8') as kconfig_file: + kconfig_file.write(write_data) + kconf = Kconfig(filename=str(kconfig_path)) + os.environ["MENUCONFIG_STYLE"] = "aquatic selection=fg:white,bg:blue" + with oebuild_util.suppress_print(): + menuconfig(kconf) + subprocess.check_output(f'rm -rf {kconfig_path}', shell=True) + config_path = pathlib.Path(os.getcwd(), '.config') + return config_path + + def _kconfig_add_target_list(self): + target_choice = textwrap.dedent(''' + comment "Select build target" + choice + prompt "Select build target" + config IMAGE + bool 'OS' +''') + gcc_toolchain_data = self._kconfig_add_gcc_toolchain() + if gcc_toolchain_data != "": + target_choice += ( + "config GCC-TOOLCHAIN \n" + " bool 'GCC TOOLCHAIN'\n\n") + llvm_toolchain_data = self._kconfig_add_llvm_toolchain() + if llvm_toolchain_data != "": + target_choice += ( + "config LLVM-TOOLCHAIN \n" + " bool 'LLVM TOOLCHAIN'\n\n" + ) + nativesdk_check = self._kconfig_check_nativesdk() + if nativesdk_check: + target_choice += ( + "config NATIVESDK \n" + " bool 'NATIVESDK'\n\n" + ) + target_choice += "endchoice" + return target_choice, gcc_toolchain_data, llvm_toolchain_data + + def _kconfig_add_choice_platform(self): + """ + add platform to kconfig + Args: + yocto_oebuild_dir: + + Returns: + + """ + platform_path = pathlib.Path(self.yocto_oebuild_dir, 'platform') + if platform_path.exists(): + platform_files = [f for f in platform_path.iterdir() + if f.is_file() and f.suffix in ['.yml', '.yaml']] + else: + logger.error('Platform directory not found.') + sys.exit(-1) + platform_start = textwrap.dedent(''' + if IMAGE + comment "Select OS platform" + choice + prompt "Select platform" + default PLATFORM_QEMU-AARCH64 + ''') + platform_end = """ + endchoice + endif""" + for platform in platform_files: + platform_name = platform.stem.strip("\n") + platform_info = ( + f" config PLATFORM_{platform_name.upper()}\n" + f" bool \"{platform_name}\"\n\n") + platform_start += platform_info + platform_data = platform_start + platform_end + return platform_data + + def _kconfig_add_feature(self): + """ + add feature to kconfig + Args: + yocto_oebuild_dir: + + Returns: + + """ + + feature_start = ''' + if IMAGE + comment "Select OS features" + ''' + + feature_triples = parse_feature_files(self.yocto_oebuild_dir) + for ft_name, _, feature_data in feature_triples: + support_str = "" + if 'support' in feature_data: + raw_supports = feature_data['support'] + validated_support_str = self.validate_and_format_platforms( + raw_supports) + if validated_support_str: + support_str = validated_support_str + else: + logger.warning( + "supported platform str of feat %s is invalid: %s", + ft_name, raw_supports) + + feature_info = (f"\nconfig FEATURE_{ft_name.upper()}\n" + f" bool \"{ft_name}\" {support_str}\n\n") + feature_start += feature_info + feature_start += "endif" + return feature_start + + def validate_and_format_platforms(self, raw_str: str): + """ + strip delimeter and format to supported platforms conditions + """ + + platforms = re.split(r'[|,，\s]+', raw_str) + platforms = [p.strip() for p in platforms if p.strip()] + if not platforms: + return "" + platform_cond = [f"PLATFORM_{p.upper()}" for p in platforms] + return "if " + "||".join(platform_cond) + + def _kconfig_add_gcc_toolchain(self): + """ + add toolchain to kconfig + Args: + yocto_oebuild_dir: yocto_oebuild_dir + + Returns: + + """ + toolchain_start = "" + cross_dir = pathlib.Path(self.yocto_oebuild_dir, "cross-tools") + if cross_dir.exists(): + configs_dir = cross_dir / 'configs' + if configs_dir.exists(): + toolchain_list = os.listdir(configs_dir) + toolchain_start += ''' + if GCC-TOOLCHAIN + ''' + for config in toolchain_list: + if not re.search('xml', config): + toolchain_info = ( + f"""\nconfig GCC-TOOLCHAIN_{config.upper().lstrip("CONFIG_")}\n""" + f""" bool "{config.upper().lstrip("CONFIG_")}"\n""" + """ depends on GCC-TOOLCHAIN\n""") + toolchain_start += toolchain_info + toolchain_start += "endif" + return toolchain_start + + def _kconfig_add_llvm_toolchain(self): + """ + add toolchain to kconfig + Args: + yocto_oebuild_dir: yocto_oebuild_dir + + Returns: + + """ + toolchain_start = "" + llvm_dir = pathlib.Path(self.yocto_oebuild_dir, "llvm-toolchain") + if llvm_dir.exists(): + toolchain_start += ''' + config LLVM-TOOLCHAIN-AARCH64-LIB + string "aarch64 lib dir" + default "None" + depends on LLVM-TOOLCHAIN + ''' + return toolchain_start + + def _kconfig_check_nativesdk(self): + """ + add nativesdk to kconfig + Args: + yocto_oebuild_dir: yocto_oebuild_dir + + Returns: + + """ + nativesdk_dir = pathlib.Path(self.yocto_oebuild_dir, "nativesdk") + return nativesdk_dir.exists() + + def _kconfig_add_common_config(self,): + """ + Build shared/common Kconfig options. + Returns: + + """ + toolchain_help = ( + "External GCC toolchain directory [your own toolchain]") + llvm_toolchain_help = ( + "External LLVM toolchain directory [your own toolchain]") + nativesdk_help = ( + "External nativesdk directory [used when building on host]") + common_str = textwrap.dedent(''' + comment "Common options" + ''') + # choice build in platform + common_str += (''' + if IMAGE + choice + prompt "Select build environment" + default BUILD_IN-DOCKER + config BUILD_IN-DOCKER + bool "docker" + config BUILD_IN-HOST + bool "host" + endchoice + endif + ''') + # add no fetch + common_str += (''' + config COMMON_NO-FETCH + bool "no_fetch (disable source fetching)" + default n + depends on IMAGE + ''') + # add no layer + common_str += (''' + config COMMON_NO-LAYER + bool "no_layer (skip layer repo update on env setup)" + default n + depends on IMAGE + ''') + # add sstate_mirrors + common_str += (''' + config COMMON_SSTATE-MIRRORS + string "SSTATE_MIRRORS value" + default "None" + depends on IMAGE + ''') + # add sstate_dir + common_str += (''' + config COMMON_SSTATE-DIR + string "SSTATE_DIR path" + default "None" + depends on IMAGE + ''') + # add tmp_dir + common_str += (''' + config COMMON_TMP-DIR + string "TMPDIR path" + default "None" + depends on IMAGE && BUILD_IN-HOST + ''') + # add gcc toolchain dir + common_str += (f''' + config COMMON_TOOLCHAIN-DIR + string "toolchain_dir ({toolchain_help})" + default "None" + depends on IMAGE + ''') + # add llvm toolchain dir + common_str += (f''' + config COMMON_LLVM-TOOLCHAIN-DIR + string "llvm_toolchain_dir ({llvm_toolchain_help})" + default "None" + depends on IMAGE + ''') + # add nativesdk dir + common_str += (f''' + config COMMON_NATIVESDK-DIR + string "nativesdk_dir ({nativesdk_help})" + default "None" + depends on IMAGE && BUILD_IN-HOST + ''') + # add timestamp + common_str += (''' + config COMMON_DATETIME + string "datetime" + default "None" + depends on IMAGE + ''') + # add cache_src_dir directory + common_str += (''' + config COMMON_CACHE_SRC_DIR + string "cache_src_dir (src directory)" + default "None" + depends on IMAGE + ''') + # add build directory + common_str += (''' + config COMMON_DIRECTORY + string "directory (build directory name)" + default "None" + ''') + return common_str diff --git a/src/oebuild/app/plugins/generate/parses.py b/src/oebuild/app/plugins/generate/parses.py index 029825e472c42395d0ef724346a3965273407709..59f80aa1a56e83efc1f6c45f42523169663ed7d7 100644 --- a/src/oebuild/app/plugins/generate/parses.py +++ b/src/oebuild/app/plugins/generate/parses.py @@ -10,7 +10,13 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' +import pathlib +import sys + +from ruamel.yaml.error import YAMLError, MarkedYAMLError import oebuild.const as oebuild_const +from oebuild.m_log import logger +import oebuild.util as oebuild_util def parsers(parser): @@ -22,7 +28,7 @@ def parsers(parser): dest='list', action="store_true", help=''' - will list support archs and features + list supported archs and features ''') parser.add_argument('-p', @@ -186,3 +192,54 @@ def parsers(parser): ''') return parser + + +@staticmethod +def parse_feature_files(oebuild_dir): + """ + both yaml and yml are supported, but yaml is preferred + """ + + feature_dir = pathlib.Path(oebuild_dir, 'features') + if not feature_dir.exists(): + logger.error("Features directory not found under .oebuild.") + sys.exit(-1) + # Group files by base name to handle priority + file_groups = {} + for ft_path in feature_dir.iterdir(): + if ft_path.is_file(): + if ft_path.suffix == '.yaml': + feature_name = ft_path.stem + if feature_name not in file_groups: + file_groups[feature_name] = {'yaml': None, 'yml': None} + file_groups[feature_name]['yaml'] = ft_path + elif ft_path.suffix == '.yml': + feature_name = ft_path.stem + if feature_name not in file_groups: + file_groups[feature_name] = {'yaml': None, 'yml': None} + file_groups[feature_name]['yml'] = ft_path + features = [] + for feature_name, extensions in file_groups.items(): + # Prefer .yaml over .yml + selected_file = extensions['yaml'] or extensions['yml'] + if selected_file: + try: + feature_data = oebuild_util.read_yaml(selected_file) + if not isinstance(feature_data, dict): + logger.warning("Invalid YAML mapping in feature '%s'", feature_name) + continue + features.append((feature_name, selected_file, feature_data)) + except MarkedYAMLError as e: + fname = selected_file.name + logger.warning( + """Failed to parse feature file '%s': + Error: %s + At: Line %d, Column %d, + """, fname, e.problem, e.problem_mark.line + 1, e.problem_mark.column + 1) + continue + except YAMLError as e: + fname = selected_file.name + logger.warning("Failed to parse feature file '%s': %s", fname, e) + continue + features.sort(key=lambda x: x[0]) + return features