From d4cb6152018e5d604239ad9481808049eb03876b Mon Sep 17 00:00:00 2001 From: Egg12138 Date: Sun, 23 Nov 2025 22:33:18 +0800 Subject: [PATCH 1/3] env: add basic pyproject and ruff format - It is pretty bad with no a simple format settings and pyproject - add a simple pyproject.toml, setting two basic ruff format styles introduce flake8 F rule Signed-off-by: egg12318 --- .gitignore | 4 +- pyproject.toml | 28 ++ setup.py | 12 +- src/oebuild/__main__.py | 4 +- src/oebuild/app/conf/plugins.yaml | 93 ++-- src/oebuild/app/main.py | 265 ++++++----- src/oebuild/app/plugins/bitbake/base_build.py | 24 +- src/oebuild/app/plugins/bitbake/bitbake.py | 81 ++-- .../app/plugins/bitbake/in_container.py | 138 +++--- src/oebuild/app/plugins/bitbake/in_host.py | 96 ++-- src/oebuild/app/plugins/clear/clear.py | 50 ++- src/oebuild/app/plugins/demo/demo.py | 29 +- src/oebuild/app/plugins/deploy/com_target.py | 242 ++++++---- .../app/plugins/deploy/deploy_target.py | 44 +- .../app/plugins/deploy/target_dev/deploy.py | 378 +++++++++++----- .../app/plugins/docker-save/docker_save.py | 46 +- src/oebuild/app/plugins/generate/generate.py | 417 ++++++++++-------- .../app/plugins/generate/kconfig_generator.py | 198 +++++---- src/oebuild/app/plugins/generate/parses.py | 320 ++++++++------ src/oebuild/app/plugins/init/init.py | 93 ++-- src/oebuild/app/plugins/m_env/m_env.py | 192 +++++--- src/oebuild/app/plugins/m_plugin/m_plugin.py | 334 ++++++++------ src/oebuild/app/plugins/manifest/manifest.py | 107 +++-- .../app/plugins/mugentest/mugentest.py | 178 ++++---- .../app/plugins/neo-generate/__init__.py | 3 + src/oebuild/app/plugins/run_qemu/run_qemu.py | 120 ++--- src/oebuild/app/plugins/samples/samples.py | 43 +- .../app/plugins/toolchain/toolchain.py | 228 ++++++---- src/oebuild/app/plugins/update/update.py | 153 ++++--- src/oebuild/auto_completion.py | 45 +- src/oebuild/bashrc.py | 99 +++-- src/oebuild/bb/utils.py | 64 ++- src/oebuild/bblayers.py | 32 +- src/oebuild/check_docker_tag.py | 47 +- src/oebuild/command.py | 39 +- src/oebuild/configure.py | 133 +++--- src/oebuild/const.py | 62 +-- src/oebuild/docker_proxy.py | 173 ++++---- src/oebuild/local_conf.py | 173 +++++--- src/oebuild/m_log.py | 9 +- src/oebuild/oebuild_parser.py | 43 +- src/oebuild/ogit.py | 68 +-- src/oebuild/parse_env.py | 66 +-- src/oebuild/parse_param.py | 143 +++--- src/oebuild/parse_template.py | 248 +++++++---- src/oebuild/spec.py | 63 +-- src/oebuild/struct.py | 61 +-- src/oebuild/util.py | 251 ++++++----- src/oebuild/version.py | 4 +- 49 files changed, 3444 insertions(+), 2299 deletions(-) create mode 100644 pyproject.toml create mode 100644 src/oebuild/app/plugins/neo-generate/__init__.py diff --git a/.gitignore b/.gitignore index 8492c1c..63d65e2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ project __pycache__ oebuild.egg-info .vscode -./build -dist \ No newline at end of file +build/ +dist diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9352d09 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "oebuild" +version = "0.1.0" +description = "openEuler Embedded workspace generator" +readme = "README.md" +requires-python = ">=3.8" + +[project.scripts] +oebuild = "oebuild.app.main:main" + +[tool.ruff] +line-length = 79 + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # flake8 + "I", # isort + "UP", # pyupgrade +] +ignore = [ + "E402", # module level import not at top of file +] diff --git a/setup.py b/setup.py index 0b7b5c5..e491ee6 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os @@ -17,10 +17,10 @@ import setuptools SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) os.chdir(SCRIPT_DIR) -with open('README.md', 'r', encoding="utf-8") as f: +with open('README.md', 'r', encoding='utf-8') as f: long_description = f.read() -with open('src/oebuild/version.py', 'r', encoding="utf-8") as f: +with open('src/oebuild/version.py', 'r', encoding='utf-8') as f: __version__ = None # pylint: disable=W0122 exec(f.read()) @@ -36,7 +36,7 @@ setuptools.setup( description='', long_description=long_description, # http://docutils.sourceforge.net/FAQ.html#what-s-the-official-mime-type-for-restructuredtext-data - long_description_content_type="text/markdown", + long_description_content_type='text/markdown', url='', packages=setuptools.find_packages(where='src'), package_dir={'': 'src'}, @@ -56,7 +56,7 @@ setuptools.setup( 'dataclasses', 'reprint', 'prettytable', - 'kconfiglib' + 'kconfiglib', ], python_requires='>=3.8', entry_points={'console_scripts': ('oebuild = oebuild.app.main:main',)}, diff --git a/src/oebuild/__main__.py b/src/oebuild/__main__.py index 5b2cfc3..759ed7d 100644 --- a/src/oebuild/__main__.py +++ b/src/oebuild/__main__.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" from oebuild.app.main import main diff --git a/src/oebuild/app/conf/plugins.yaml b/src/oebuild/app/conf/plugins.yaml index 9455634..e1d7899 100644 --- a/src/oebuild/app/conf/plugins.yaml +++ b/src/oebuild/app/conf/plugins.yaml @@ -1,46 +1,49 @@ plugins: -- name: init - class: Init - path: plugins/init/init.py -- name: update - class: Update - path: plugins/update/update.py -- name: generate - class: Generate - path: plugins/generate/generate.py -- name: bitbake - class: Bitbake - path: plugins/bitbake/bitbake.py -- name: manifest - class: Manifest - path: plugins/manifest/manifest.py -- name: clear - class: Clear - path: plugins/clear/clear.py -- name: runqemu - class: RunQemu - path: plugins/run_qemu/run_qemu.py -- name: menv - class: Menv - path: plugins/m_env/m_env.py -- name: deploy-target - class: DeployTarget - path: plugins/deploy/deploy_target.py -- name: undeploy-target - class: UnDeployTarget - path: plugins/deploy/deploy_target.py -- name: mplugin - class: MPlugin - path: plugins/m_plugin/m_plugin.py -- name: toolchain - class: Toolchain - path: plugins/toolchain/toolchain.py -- name: docker-save - class: DockerSave - path: plugins/docker-save/docker_save.py -- name: samples - class: Samples - path: plugins/samples/samples.py -- name: mugentest - class: MugenTest - path: plugins/mugentest/mugentest.py + - name: init + class: Init + path: plugins/init/init.py + - name: update + class: Update + path: plugins/update/update.py + - name: generate + class: Generate + path: plugins/generate/generate.py + - name: neo-generate + class: NeoGenerate + path: plugins/neo-generate/neo_generate.py + - name: bitbake + class: Bitbake + path: plugins/bitbake/bitbake.py + - name: manifest + class: Manifest + path: plugins/manifest/manifest.py + - name: clear + class: Clear + path: plugins/clear/clear.py + - name: runqemu + class: RunQemu + path: plugins/run_qemu/run_qemu.py + - name: menv + class: Menv + path: plugins/m_env/m_env.py + - name: deploy-target + class: DeployTarget + path: plugins/deploy/deploy_target.py + - name: undeploy-target + class: UnDeployTarget + path: plugins/deploy/deploy_target.py + - name: mplugin + class: MPlugin + path: plugins/m_plugin/m_plugin.py + - name: toolchain + class: Toolchain + path: plugins/toolchain/toolchain.py + - name: docker-save + class: DockerSave + path: plugins/docker-save/docker_save.py + - name: samples + class: Samples + path: plugins/samples/samples.py + - name: mugentest + class: MugenTest + path: plugins/mugentest/mugentest.py diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py index 12e0a51..8207b68 100644 --- a/src/oebuild/app/main.py +++ b/src/oebuild/app/main.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,8 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" + import os import sys import pathlib @@ -27,14 +28,14 @@ from oebuild.command import OebuildCommand from oebuild.oebuild_parser import OebuildArgumentParser, OebuildHelpAction from oebuild.parse_template import get_docker_volumns -APP = "app" +APP = 'app' class OebuildApp: - ''' + """ The execution body of the oebuild tool, and all oebuild commands are resolved and executed by this body - ''' + """ def __init__(self): self.base_oebuild_dir = oebuild_util.get_base_oebuild() @@ -42,35 +43,46 @@ class OebuildApp: self.subparsers = {} self.cmd = None try: - plugins_dir = pathlib.Path(self.base_oebuild_dir, 'app/conf', 'plugins.yaml') - oebuild_plugins_path = os.path.expanduser('~') + '/.local/oebuild_plugins/' - append_plugins_dir = pathlib.Path(oebuild_plugins_path, 'append_plugins.yaml') - self.command_ext = self.get_command_ext(oebuild_util.read_yaml(plugins_dir)['plugins']) - if os.path.exists(append_plugins_dir) \ - and oebuild_util.read_yaml(append_plugins_dir): + plugins_dir = pathlib.Path( + self.base_oebuild_dir, 'app/conf', 'plugins.yaml' + ) + oebuild_plugins_path = ( + os.path.expanduser('~') + '/.local/oebuild_plugins/' + ) + append_plugins_dir = pathlib.Path( + oebuild_plugins_path, 'append_plugins.yaml' + ) + self.command_ext = self.get_command_ext( + oebuild_util.read_yaml(plugins_dir)['plugins'] + ) + if os.path.exists(append_plugins_dir) and oebuild_util.read_yaml( + append_plugins_dir + ): self.command_ext = self.get_command_ext( oebuild_util.read_yaml(append_plugins_dir)['plugins'], - self.command_ext) + self.command_ext, + ) self.command_spec = {} except Exception as e_p: raise e_p @staticmethod def get_command_ext(plugins: list, command_ext=None): - ''' + """ return command information object - ''' + """ if command_ext is None: command_ext = OrderedDict() for app in plugins: if 'status' not in app or app['status'] == 'enable': command_ext[app['name']] = _ExtCommand( - name=app['name'], - class_name=app['class'], - path=app['path']) + name=app['name'], class_name=app['class'], path=app['path'] + ) return command_ext - def _load_extension_specs(self, ): + def _load_extension_specs( + self, + ): self.command_spec = extension_commands(APP, self.command_ext) def _setup_parsers(self): @@ -80,58 +92,75 @@ class OebuildApp: # Add sub-parsers for the command_ext commands. for command_name in self.command_ext: - self.subparsers[command_name] = subparser_gen.add_parser(command_name, add_help=False) + self.subparsers[command_name] = subparser_gen.add_parser( + command_name, add_help=False + ) # Save the instance state. self.oebuild_parser = oebuild_parser - def make_parsers(self,): - ''' + def make_parsers( + self, + ): + """ Make a fresh instance of the top level argument parser and subparser generator, and return them in that order. The prog='oebuild' override avoids the absolute path of the main.py script showing up when West is run via the wrapper - ''' + """ parser = OebuildArgumentParser( prog='oebuild', - description='''The openEuler Embedded meta-tool. you can directly run + description="""The openEuler Embedded meta-tool. you can directly run oebuild in oebuild workspace to perform the build, for example: oebuild - ''', - epilog='''Run "oebuild -h" for help on each .''', - add_help=False, oebuild_app=self + """, + epilog="""Run "oebuild -h" for help on each .""", + add_help=False, + oebuild_app=self, ) - parser.add_argument('-h', '--help', action=OebuildHelpAction, nargs=0, - help='get help for oebuild or a command') + parser.add_argument( + '-h', + '--help', + action=OebuildHelpAction, + nargs=0, + help='get help for oebuild or a command', + ) - parser.add_argument('-v', '--version', action='version', - version=f'Oebuild version: v{__version__}', - help='print the program version and exit') + parser.add_argument( + '-v', + '--version', + action='version', + version=f'Oebuild version: v{__version__}', + help='print the program version and exit', + ) - subparser_gen = parser.add_subparsers(metavar='', - dest='command') + subparser_gen = parser.add_subparsers( + metavar='', dest='command' + ) return parser, subparser_gen def _check_command(self, args): - if args.help or \ - args.command is None or \ - args.command not in self.command_ext or \ - args.command == 'help': + if ( + args.help + or args.command is None + or args.command not in self.command_ext + or args.command == 'help' + ): self.help() return False return True def run_command(self, argv): - ''' + """ Parse command line arguments and run the OebuildCommand. If we're running an extension, instantiate it from its spec and re-parse arguments before running. - ''' + """ args, unknown = self.oebuild_parser.parse_known_args(args=argv) if not self._check_command(args=args): @@ -152,9 +181,9 @@ oebuild cmd.run(args, unknown) def run(self, argv): - ''' + """ the function will be exec first - ''' + """ self._load_extension_specs() # Set up initial argument parsers. This requires knowing # self.extensions, so it can't happen before now. @@ -163,31 +192,33 @@ oebuild # OK, we are all set. Run the command. self.run_command(argv) - def help(self,): - ''' + def help( + self, + ): + """ print help message - ''' + """ self.oebuild_parser.print_help() def check_user(): - ''' + """ check execute user must in normal user - ''' - if pwd.getpwuid(os.getuid())[0] == "root": - logger.error("can not use oebuild in root") + """ + if pwd.getpwuid(os.getuid())[0] == 'root': + logger.error('can not use oebuild in root') return False return True def extension_commands(pre_dir, commandlist: OrderedDict): - ''' + """ Get descriptions of available extension commands. The return value is an ordered map from project paths to lists of OebuildExtCommandSpec objects, for projects which define extension commands. The map's iteration order matches the manifest.projects order. - ''' + """ specs = OrderedDict() for key, value in commandlist.items(): specs[key] = get_spec(pre_dir, value) @@ -195,10 +226,10 @@ def extension_commands(pre_dir, commandlist: OrderedDict): return specs -class QuickBuild(): - ''' +class QuickBuild: + """ The build command will quickly generate the compile.yaml - ''' + """ def __init__(self, build_yaml_path): self.app = OebuildApp() @@ -207,28 +238,34 @@ class QuickBuild(): self.workdir = None self.build_dir = None - def _check_yaml(self,): + def _check_yaml( + self, + ): if not os.path.exists(self.build_yaml_path.absolute()): - logger.error("%s is not exists!", self.build_yaml_path) + logger.error('%s is not exists!', self.build_yaml_path) sys.exit(-1) data = oebuild_util.read_yaml(yaml_path=self.build_yaml_path) - compile_param = ParseCompileParam().parse_to_obj(compile_param_dict=data) + compile_param = ParseCompileParam().parse_to_obj( + compile_param_dict=data + ) self.compile_param = compile_param def run(self): - ''' + """ Execute oebuild commands in order. - ''' + """ if not Configure().is_oebuild_dir(): logger.error('Your current directory is not oebuild workspace') sys.exit(-1) self.workdir = Configure().oebuild_topdir() self._check_yaml() - if "compile.yaml" in os.listdir(): + if 'compile.yaml' in os.listdir(): self.build_dir = os.path.basename(os.getcwd()) else: - build_name = self.build_yaml_path.name.replace(".yaml", "").replace(".yml", "") + build_name = self.build_yaml_path.name.replace( + '.yaml', '' + ).replace('.yml', '') self.build_dir = os.path.basename(build_name) self.generate() @@ -236,9 +273,9 @@ class QuickBuild(): self.bitbake() def generate(self): - ''' + """ xxx - ''' + """ # judge if exist src/yocto-meta-openeuler, one-click function need yocto-meta-openeuler if not os.path.exists(Configure().source_yocto_dir()): logger.error(""" @@ -257,49 +294,75 @@ or self._init_build_dir() if self.compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: if self.compile_param.docker_param is None: - logger.error("param is error, build in docker need docker_param") + logger.error( + 'param is error, build in docker need docker_param' + ) sys.exit(-1) # check src and compile_dir if exists src_volumn_flag = False compile_volumn_flag = False for volumn in self.compile_param.docker_param.volumns: - volumn_split = volumn.split(":") - if oebuild_const.CONTAINER_SRC == volumn_split[1].strip(" "): + volumn_split = volumn.split(':') + if oebuild_const.CONTAINER_SRC == volumn_split[1].strip(' '): src_volumn_flag = True - if volumn_split[1].strip(" ").startswith(oebuild_const.CONTAINER_BUILD): + if ( + volumn_split[1] + .strip(' ') + .startswith(oebuild_const.CONTAINER_BUILD) + ): compile_volumn_flag = True if not src_volumn_flag: self.compile_param.docker_param.volumns.append( - f"{self.workdir}/src:{oebuild_const.CONTAINER_SRC}" + f'{self.workdir}/src:{oebuild_const.CONTAINER_SRC}' ) if not compile_volumn_flag: - volumn_dir = os.path.join(oebuild_const.CONTAINER_BUILD, - os.path.basename(self.build_dir)) + volumn_dir = os.path.join( + oebuild_const.CONTAINER_BUILD, + os.path.basename(self.build_dir), + ) self.compile_param.docker_param.volumns.append( - f"{os.path.abspath(self.build_dir)}:{volumn_dir}" + f'{os.path.abspath(self.build_dir)}:{volumn_dir}' + ) + self.compile_param.docker_param.volumns.extend( + get_docker_volumns( + { + 'cache_src_dir': self.compile_param.cache_src_dir, + 'toolchain_dir': self.compile_param.toolchain_dir, + 'llvm_toolchain_dir': self.compile_param.llvm_toolchain_dir, + 'sstate_mirrors': self.compile_param.sstate_mirrors, + 'sstate_dir': self.compile_param.sstate_dir, + } ) - self.compile_param.docker_param.volumns.extend(get_docker_volumns({ - "cache_src_dir": self.compile_param.cache_src_dir, - "toolchain_dir": self.compile_param.toolchain_dir, - "llvm_toolchain_dir": self.compile_param.llvm_toolchain_dir, - "sstate_mirrors": self.compile_param.sstate_mirrors, - "sstate_dir": self.compile_param.sstate_dir - })) - compile_param_dict = ParseCompileParam().parse_to_dict(compile_param=self.compile_param) - compile_yaml_path = os.path.join(self.build_dir, "compile.yaml") - oebuild_util.write_yaml(yaml_path=compile_yaml_path, data=compile_param_dict) + ) + compile_param_dict = ParseCompileParam().parse_to_dict( + compile_param=self.compile_param + ) + compile_yaml_path = os.path.join(self.build_dir, 'compile.yaml') + oebuild_util.write_yaml( + yaml_path=compile_yaml_path, data=compile_param_dict + ) def _init_build_dir(self): # check compile if exists, if exists, exit with -1 if os.path.exists(self.build_dir): build_dir = self.build_dir - logger.warning("the build directory %s already exists", build_dir) + logger.warning('the build directory %s already exists', build_dir) while True: in_res = input(f""" do you want to overwrite it({os.path.basename(build_dir)})? the overwrite action will replace the compile.yaml or toolchain.yaml to new and delete conf directory, enter Y for yes, N for no, C for create:""") - if in_res not in ["Y", "y", "yes", "N", "n", "no", "C", "c", "create"]: + if in_res not in [ + 'Y', + 'y', + 'yes', + 'N', + 'n', + 'no', + 'C', + 'c', + 'create', + ]: print(""" wrong input""") continue @@ -313,42 +376,44 @@ or continue break self.build_dir = build_dir - if os.path.exists(os.path.join(self.build_dir, "conf")): - rmtree(os.path.join(self.build_dir, "conf")) + if os.path.exists(os.path.join(self.build_dir, 'conf')): + rmtree(os.path.join(self.build_dir, 'conf')) elif os.path.exists(self.build_dir): rmtree(self.build_dir) os.makedirs(self.build_dir, exist_ok=True) def bitbake(self): - ''' + """ xxx - ''' + """ os.chdir(os.path.abspath(self.build_dir)) if self.compile_param.bitbake_cmds is None: - print("================================================\n\n" - "please enter follow directory for next steps!!!\n\n" - f"{os.path.abspath(self.build_dir)}\n\n" - "================================================\n") + print( + '================================================\n\n' + 'please enter follow directory for next steps!!!\n\n' + f'{os.path.abspath(self.build_dir)}\n\n' + '================================================\n' + ) return for bitbake_cmd in self.compile_param.bitbake_cmds: - bitbake_cmd: str = bitbake_cmd.strip(" ") - argv = [ - 'bitbake', - bitbake_cmd.lstrip("bitbake") - ] + bitbake_cmd: str = bitbake_cmd.strip(' ') + argv = ['bitbake', bitbake_cmd.lstrip('bitbake')] self.app.run(argv or sys.argv[1:]) - logger.info(""" + logger.info( + """ ====================================================================== Please enter the building directory according to the command prompt below: cd %s -""", os.path.dirname(os.path.abspath(self.build_dir))) +""", + os.path.dirname(os.path.abspath(self.build_dir)), + ) def main(argv=None): - ''' + """ oebuild main entrypoint - ''' + """ if not check_user(): return @@ -362,5 +427,5 @@ def main(argv=None): app.run(argv or sys.argv[1:]) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/src/oebuild/app/plugins/bitbake/base_build.py b/src/oebuild/app/plugins/bitbake/base_build.py index 39e3bcc..00df37b 100644 --- a/src/oebuild/app/plugins/bitbake/base_build.py +++ b/src/oebuild/app/plugins/bitbake/base_build.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,8 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" + import os import oebuild.const as oebuild_const @@ -17,26 +18,27 @@ from oebuild.bblayers import BBLayers class BaseBuild: - ''' + """ The class provides the basic methods that the build class inherits - ''' + """ def replace_local_conf(self, compile_param, local_path, src_dir=None): - ''' + """ replace some param in local.conf, the LocalConf will be instantiated and exec update - ''' + """ local_conf = LocalConf(local_path) local_conf.update(compile_param, src_dir) - def add_bblayers(self, bblayers_dir: str, pre_dir: str, base_dir: str, layers): - ''' + def add_bblayers( + self, bblayers_dir: str, pre_dir: str, base_dir: str, layers + ): + """ add_layers has two main functions, one is to initialize the compilation directory, and the other is to add the bblayer layer so that subsequent build directory file replacement operations can be successfully executed - ''' - bblayers = BBLayers(bblayers_dir=bblayers_dir, - base_dir=base_dir) + """ + bblayers = BBLayers(bblayers_dir=bblayers_dir, base_dir=base_dir) pre_dir = os.path.join(pre_dir, f'{oebuild_const.YOCTO_POKY}/..') bblayers.add_layer(pre_dir=pre_dir, layers=layers) diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 4799ffc..1ec5e58 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os import argparse @@ -30,19 +30,19 @@ import oebuild.const as oebuild_const class Bitbake(OebuildCommand): - ''' + """ Bitbake instructions can enter the build interactive environment and then directly run bitbake-related instructions,or run bitbake command directly, for example: `oebuild bitbake busybox` - ''' + """ help_msg = 'execute bitbake command' - description = textwrap.dedent(''' + description = textwrap.dedent(""" The bitbake command performs the build operation, and for the build environment, there are two types, one is to build in docker and the other is to build in the host. There are also two construction methods, one is to build directly, and the other is to call up the build environment to be operated freely by the user - ''') + """) def __init__(self): self.compile_conf_dir = os.path.join(os.getcwd(), 'compile.yaml') @@ -51,8 +51,9 @@ class Bitbake(OebuildCommand): super().__init__('bitbake', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [command] [--with-docker-image] --with-docker-image @@ -60,25 +61,27 @@ class Bitbake(OebuildCommand): docker_param->image will be modified from compile.yaml the command example like: oebuild bitbake --with-docker-image=demo:latest -''') +""", + ) parser_adder.add_argument( 'command', nargs='?', default=None, - help='''The name of the directory that will be initialized''') + help="""The name of the directory that will be initialized""", + ) return parser def do_run(self, args: argparse.ArgumentParser, unknown=None): - ''' + """ The BitBake execution logic is: the first step is to prepare the code that initializes the environment dependency, the second step to build the configuration file to the object, the third step to handle the container needed for compilation, and the fourth step to enter the build environment - ''' + """ if '-h' in unknown or '--help' in unknown: self.print_help_msg() sys.exit(0) @@ -89,7 +92,8 @@ class Bitbake(OebuildCommand): if not self.check_support_bitbake(): logger.error( - "Please do it in compile workspace which contain compile.yaml") + 'Please do it in compile workspace which contain compile.yaml' + ) sys.exit(-1) if not os.path.exists('.env'): @@ -97,23 +101,30 @@ class Bitbake(OebuildCommand): compile_param_dict = oebuild_util.read_yaml(self.compile_conf_dir) compile_param: CompileParam = ParseCompileParam.parse_to_obj( - compile_param_dict) + compile_param_dict + ) compile_param = self._deal_oe_params(oe_params, compile_param) # if has manifest.yaml, init layer repo with it - yocto_dir = os.path.join(self.configure.source_dir(), - "yocto-meta-openeuler") - manifest_path = os.path.join(yocto_dir, ".oebuild/manifest.yaml") - if compile_param.no_layer is None or compile_param.no_layer is not True: + yocto_dir = os.path.join( + self.configure.source_dir(), 'yocto-meta-openeuler' + ) + manifest_path = os.path.join(yocto_dir, '.oebuild/manifest.yaml') + if ( + compile_param.no_layer is None + or compile_param.no_layer is not True + ): if compile_param.cache_src_dir is not None: oebuild_util.sync_repo_from_cache( repo_list=compile_param.repos, src_dir=self.configure.source_dir(), - cache_src_dir=compile_param.cache_src_dir) + cache_src_dir=compile_param.cache_src_dir, + ) oebuild_util.download_repo_from_manifest( repo_list=compile_param.repos, src_dir=self.configure.source_dir(), - manifest_path=manifest_path) + manifest_path=manifest_path, + ) parse_env = ParseEnv(env_dir='.env') if compile_param.build_in == oebuild_const.BUILD_IN_HOST: @@ -130,15 +141,17 @@ class Bitbake(OebuildCommand): sys.exit(-1) in_container = InContainer(self.configure) - in_container.exec(parse_env=parse_env, - compile_param=compile_param, - command=command) - - def check_support_bitbake(self, ): - ''' + in_container.exec( + parse_env=parse_env, compile_param=compile_param, command=command + ) + + def check_support_bitbake( + self, + ): + """ The execution of the bitbake instruction mainly relies on compile.yaml, which is initialized by parsing the file - ''' + """ return os.path.exists(os.path.join(os.getcwd(), 'compile.yaml')) def _get_command(self, unknow: list): @@ -153,7 +166,7 @@ class Bitbake(OebuildCommand): oe_params = [] new_unknow = [] for item in unknow: - if item.startswith("--with-docker-image"): + if item.startswith('--with-docker-image'): oe_params.append(item) else: new_unknow.append(item) @@ -162,15 +175,17 @@ class Bitbake(OebuildCommand): def _deal_oe_params(self, oe_params, compile_param: CompileParam): is_modify = False for item in oe_params: - if item.startswith("--with-docker-image"): - item_split = item.split("=") + if item.startswith('--with-docker-image'): + item_split = item.split('=') if len(item_split) < 2: - logger.error("the format is --with-docker-image=xxx:yyy") + logger.error('the format is --with-docker-image=xxx:yyy') sys.exit(-1) if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: compile_param.docker_param.image = item_split[1] is_modify = True if is_modify: - oebuild_util.write_yaml(self.compile_conf_dir, - ParseCompileParam().parse_to_dict(compile_param)) + oebuild_util.write_yaml( + self.compile_conf_dir, + ParseCompileParam().parse_to_dict(compile_param), + ) return compile_param diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index 2bdb72e..d0f6ab4 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,8 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" + import os import sys @@ -26,9 +27,9 @@ import oebuild.const as oebuild_const class InContainer(BaseBuild): - ''' + """ bitbake command execute in container - ''' + """ def __init__(self, configure: Configure): self.configure = configure @@ -37,33 +38,37 @@ class InContainer(BaseBuild): self.bashrc = None def exec(self, parse_env: ParseEnv, compile_param: CompileParam, command): - ''' + """ execute bitbake command - ''' - logger.info("Bitbake starting ...") + """ + logger.info('Bitbake starting ...') # check docker image if exists docker_proxy = DockerProxy() docker_param = compile_param.docker_param if not docker_proxy.is_image_exists(docker_param.image): - logger.error('''The docker image does not exists, please run fellow command: - `oebuild update docker`''') + logger.error("""The docker image does not exists, please run fellow command: + `oebuild update docker`""") sys.exit(-1) self.container_id = oebuild_util.deal_env_container( - env=parse_env, docker_param=docker_param) + env=parse_env, docker_param=docker_param + ) self.bashrc = Bashrc() - self.bashrc.set_container(container=self.client.get_container(self.container_id)) + self.bashrc.set_container( + container=self.client.get_container(self.container_id) + ) self.exec_compile(compile_param=compile_param, command=command) - def exec_compile(self, compile_param: CompileParam, command: str = ""): - ''' + def exec_compile(self, compile_param: CompileParam, command: str = ''): + """ execute compile task - ''' + """ container: Container = self.client.get_container(self.container_id) # type: ignore - self.init_bash(container=container, - build_dir_name=os.path.basename(os.getcwd())) + self.init_bash( + container=container, build_dir_name=os.path.basename(os.getcwd()) + ) try: self.init_bitbake(container=container) @@ -72,32 +77,34 @@ class InContainer(BaseBuild): return # add bblayers, this action must before replace local_conf - bblayers_dir = os.path.join(os.getcwd(), "conf", "bblayers.conf") + bblayers_dir = os.path.join(os.getcwd(), 'conf', 'bblayers.conf') self.add_bblayers( bblayers_dir=bblayers_dir, pre_dir=oebuild_const.CONTAINER_SRC, base_dir=self.configure.source_dir(), - layers=compile_param.layers) + layers=compile_param.layers, + ) # replace local_conf local_path = os.path.join(os.getcwd(), 'conf', 'local.conf') self.replace_local_conf( - compile_param=compile_param, - local_path=local_path) + compile_param=compile_param, local_path=local_path + ) # add auto execute command for example: bitbake busybox - if command is not None and command != "": + if command is not None and command != '': content = self.bashrc.get_bashrc_content() new_content = self.bashrc.add_bashrc(content=content, line=command) self.bashrc.update_bashrc(content=new_content) res: ExecResult = self.client.container_exec_command( container=container, - command="bash .bashrc", + command='bash .bashrc', user=oebuild_const.CONTAINER_USER, params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "demux": True - }) + 'work_space': f'/home/{oebuild_const.CONTAINER_USER}', + 'demux': True, + }, + ) exit_code = 0 for line in res.output: if line[1] is not None: @@ -110,33 +117,35 @@ class InContainer(BaseBuild): else: content = self.bashrc.get_bashrc_content() for b_s in oebuild_const.BASH_BANNER.split('\n'): - b_s = f"echo {b_s}" + b_s = f'echo {b_s}' content = self.bashrc.add_bashrc(content=content, line=b_s) self.bashrc.update_bashrc(content=content) os.system( - f"docker exec -it -u {oebuild_const.CONTAINER_USER} {container.short_id} bash") + f'docker exec -it -u {oebuild_const.CONTAINER_USER} {container.short_id} bash' + ) self.bashrc.restore_bashrc() def init_bitbake(self, container: Container): - ''' + """ init_bitbake will start a container with pty and then check bblayers.conf and local.conf if exists in 10 seconds, otherwise raise init bitbake faild - ''' + """ self.client.check_change_ugid( - container=container, - container_user=oebuild_const.CONTAINER_USER) + container=container, container_user=oebuild_const.CONTAINER_USER + ) self._install_sudo(container=container) res = self.client.container_exec_command( container=container, - command=f"bash /home/{oebuild_const.CONTAINER_USER}/.bashrc", + command=f'bash /home/{oebuild_const.CONTAINER_USER}/.bashrc', user=oebuild_const.CONTAINER_USER, params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "stream": False - }) + 'work_space': f'/home/{oebuild_const.CONTAINER_USER}', + 'stream': False, + }, + ) if res.exit_code != 0: raise ValueError(res.output.decode()) # raise ValueError("bitbake init faild") @@ -147,15 +156,17 @@ class InContainer(BaseBuild): resp = self.client.container_exec_command( container=container, user='root', - command="which sudo", + command='which sudo', params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "stream": False - }) + 'work_space': f'/home/{oebuild_const.CONTAINER_USER}', + 'stream': False, + }, + ) if resp.exit_code != 0: logger.info( - "=========================install sudo===============================") - self._install_software(container=container, software="sudo") + '=========================install sudo===============================' + ) + self._install_software(container=container, software='sudo') def _replace_yum_mirror(self, container: Container): """ @@ -174,28 +185,30 @@ class InContainer(BaseBuild): sed -i 's/repo.openeuler.org/mirrors.huaweicloud.com\/openeuler/g' /etc/yum.repos.d/openEuler.repo """, params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "stream": False - }) + 'work_space': f'/home/{oebuild_const.CONTAINER_USER}', + 'stream': False, + }, + ) def _install_software(self, container: Container, software: str): resp = self.client.container_exec_command( container=container, user='root', - command=f"yum install {software} -y", + command=f'yum install {software} -y', params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "stream": True - }) + 'work_space': f'/home/{oebuild_const.CONTAINER_USER}', + 'stream': True, + }, + ) for line in resp.output: logger.info(line.decode().strip('\n')) def init_bash(self, container: Container, build_dir_name): - ''' + """ Bitbake will initialize the compilation environment by reading the user initialization script first, then making directional substitutions, and finally writing the initialization script - ''' + """ # read container default user .bashrc content content = self.bashrc.get_bashrc_content() @@ -204,23 +217,36 @@ sed -i 's/repo.openeuler.org/mirrors.huaweicloud.com\/openeuler/g' /etc/yum.repo init_proxy_command = self._set_container_proxy(host_proxy=host_proxy) # get nativesdk environment path automatic for next step - sdk_env_path = oebuild_util.get_nativesdk_environment(container=container) + sdk_env_path = oebuild_util.get_nativesdk_environment( + container=container + ) init_sdk_command = f'. {oebuild_const.NATIVESDK_DIR}/{sdk_env_path}' # get template dir for initialize yocto build environment - template_dir = f"{oebuild_const.CONTAINER_SRC}/yocto-meta-openeuler/.oebuild" + template_dir = ( + f'{oebuild_const.CONTAINER_SRC}/yocto-meta-openeuler/.oebuild' + ) set_template = f'export TEMPLATECONF="{template_dir}"' - init_oe_comand = f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ + init_oe_comand = ( + f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ {oebuild_const.CONTAINER_BUILD}/{build_dir_name}' - init_command = [init_proxy_command, init_sdk_command, set_template, init_oe_comand] + ) + init_command = [ + init_proxy_command, + init_sdk_command, + set_template, + init_oe_comand, + ] new_content = self.bashrc.init_bashrc_content(content, init_command) self.bashrc.update_bashrc(content=new_content) def _set_container_proxy(self, host_proxy): - init_proxy_command = "" + init_proxy_command = '' for key, value in host_proxy.items(): key_git = key.replace('_', '.') - command = f'export {key}={value}; git config --global {key_git} {value};' + command = ( + f'export {key}={value}; git config --global {key_git} {value};' + ) init_proxy_command = f'{init_proxy_command} {command}' return init_proxy_command diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py index b4a489a..c8834d6 100644 --- a/src/oebuild/app/plugins/bitbake/in_host.py +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,8 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" + import os import subprocess import pty @@ -25,9 +26,9 @@ import oebuild.const as oebuild_const class InHost(BaseBuild): - ''' + """ bitbake command execute in host - ''' + """ def __init__(self, configure: Configure): self.configure = configure @@ -35,71 +36,83 @@ class InHost(BaseBuild): self.bashrc = Bashrc() def exec(self, compile_param: CompileParam, command): - ''' + """ execute bitbake commands - ''' + """ # check nativesdk_dir, if not set, use default value if compile_param.nativesdk_dir is None: compile_param.nativesdk_dir = oebuild_const.NATIVESDK_DIR self._init_build_sh(build_dir=os.getcwd()) - self._mk_build_sh(nativesdk_dir=compile_param.nativesdk_dir, build_dir=os.getcwd()) + self._mk_build_sh( + nativesdk_dir=compile_param.nativesdk_dir, build_dir=os.getcwd() + ) self.init_bitbake() # add bblayers, this action must before replace local_conf - bblayers_dir = os.path.join(os.getcwd(), "conf", "bblayers.conf") + bblayers_dir = os.path.join(os.getcwd(), 'conf', 'bblayers.conf') self.add_bblayers( bblayers_dir=bblayers_dir, pre_dir=self.configure.source_dir(), base_dir=self.configure.source_dir(), - layers=compile_param.layers) + layers=compile_param.layers, + ) local_path = os.path.join(os.getcwd(), 'conf', 'local.conf') try: self.replace_local_conf( compile_param=compile_param, local_path=local_path, - src_dir=self.configure.source_dir()) + src_dir=self.configure.source_dir(), + ) except NativesdkNotExist as n_e: logger.error(str(n_e)) - logger.error("Please set valid nativesdk directory") + logger.error('Please set valid nativesdk directory') sys.exit(1) except NativesdkNotValid as n_e: logger.error(str(n_e)) - logger.error(''' + logger.error(""" The nativesdk path must be valid, it is recommended that you download the nativesdk script and then perform -initialization operations''') +initialization operations""") sys.exit(1) - if command is not None and command != "": + if command is not None and command != '': self._append_build_sh(str_list=[command], build_dir=os.getcwd()) self._bash_build() else: # run in Interactive mode banner_list = [] for b_s in oebuild_const.BASH_BANNER.split('\n'): - b_s = f"echo {b_s}{oebuild_const.BASH_END_FLAG}" + b_s = f'echo {b_s}{oebuild_const.BASH_END_FLAG}' banner_list.append(b_s) self._append_build_sh(str_list=banner_list, build_dir=os.getcwd()) - append_str = f"sed -i '/{oebuild_const.BASH_END_FLAG}/d' $HOME/.bashrc" + append_str = ( + f"sed -i '/{oebuild_const.BASH_END_FLAG}/d' $HOME/.bashrc" + ) self._append_build_sh(str_list=[append_str], build_dir=os.getcwd()) build_sh_dir = os.path.join(os.getcwd(), 'build.sh') - source_build_str = f"source {build_sh_dir}" + source_build_str = f'source {build_sh_dir}' # content = self.bashrc.get_bashrc_content() content = self.bashrc.get_bashrc_content() content = self.bashrc.restore_bashrc_content(old_content=content) - new_content = self.bashrc.add_bashrc(content, line=source_build_str) + new_content = self.bashrc.add_bashrc( + content, line=source_build_str + ) self.bashrc.update_bashrc(content=new_content) - pty.spawn("bash") - - def _bash_build(self,): - with subprocess.Popen('bash build.sh', - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - text=True) as s_p: + pty.spawn('bash') + + def _bash_build( + self, + ): + with subprocess.Popen( + 'bash build.sh', + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8', + text=True, + ) as s_p: if s_p.returncode is not None and s_p.returncode != 0: err_msg = '' if s_p.stderr is not None: @@ -123,15 +136,24 @@ initialization operations''') sdk_env_path = oebuild_util.get_nativesdk_environment(nativesdk_dir) init_sdk_command = f'. {nativesdk_dir}/{sdk_env_path}' # get template dir for initialize yocto build environment - template_dir = f"{self.configure.source_dir()}/yocto-meta-openeuler/.oebuild" + template_dir = ( + f'{self.configure.source_dir()}/yocto-meta-openeuler/.oebuild' + ) set_template = f'export TEMPLATECONF="{template_dir}"' - init_oe_command = f'. {self.configure.source_dir()}/yocto-poky/oe-init-build-env \ + init_oe_command = ( + f'. {self.configure.source_dir()}/yocto-poky/oe-init-build-env \ {build_dir}' + ) ps1_command = 'PS1="\\u\\h:\\W> "' self._append_build_sh( - str_list=[init_sdk_command, set_template, init_oe_command, ps1_command], - build_dir=build_dir + str_list=[ + init_sdk_command, + set_template, + init_oe_command, + ps1_command, + ], + build_dir=build_dir, ) def _init_build_sh(self, build_dir): @@ -143,16 +165,18 @@ initialization operations''') def _append_build_sh(self, str_list: list, build_dir): build_sh_dir = os.path.join(build_dir, 'build.sh') if not os.path.exists(build_sh_dir): - raise ValueError("build.sh not exists") + raise ValueError('build.sh not exists') with open(build_sh_dir, 'a', encoding='utf-8') as w_f: w_f.write('\n') w_f.write('\n'.join(str_list)) - def init_bitbake(self,): - ''' + def init_bitbake( + self, + ): + """ Bitbake will initialize the compilation environment by reading the user initialization script first, then making directional substitutions, and finally writing the initialization script - ''' - subprocess.getoutput("bash build.sh") + """ + subprocess.getoutput('bash build.sh') diff --git a/src/oebuild/app/plugins/clear/clear.py b/src/oebuild/app/plugins/clear/clear.py index fe19ac9..0f69e89 100644 --- a/src/oebuild/app/plugins/clear/clear.py +++ b/src/oebuild/app/plugins/clear/clear.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import textwrap @@ -26,16 +26,16 @@ from oebuild.m_log import logger class Clear(OebuildCommand): - ''' + """ for some clear task - ''' + """ help_msg = 'clear someone which oebuild generate' - description = textwrap.dedent('''\ + description = textwrap.dedent("""\ During the construction process using oebuild, a lot of temporary products will be generated, such as containers,so this command can remove unimportant products, such as containers - ''') + """) def __init__(self): self.configure = Configure() @@ -43,16 +43,20 @@ class Clear(OebuildCommand): super().__init__('clear', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [docker] -''') +""", + ) parser.add_argument( 'item', nargs='?', default=None, - help='''The name of the directory that will be initialized''') + help="""The name of the directory that will be initialized""", + ) return parser @@ -63,26 +67,28 @@ class Clear(OebuildCommand): args = args.parse_args(unknown) - if args.item == "docker": + if args.item == 'docker': try: self.client = DockerProxy() except DockerException: - logger.error("Please install docker first!!!") + logger.error('Please install docker first!!!') sys.exit(-1) self.clear_docker() - def clear_docker(self, ): - ''' + def clear_docker( + self, + ): + """ clear container - ''' + """ # get all build directory and get .env from every build directory - logger.info("Clearing container, please waiting ...") + logger.info('Clearing container, please waiting ...') env_list = [] build_list = os.listdir(self.configure.build_dir()) for build_dir in build_list: build_dir = os.path.join(self.configure.build_dir(), build_dir) - if os.path.exists(os.path.join(build_dir, ".env")): - env_list.append(os.path.join(build_dir, ".env")) + if os.path.exists(os.path.join(build_dir, '.env')): + env_list.append(os.path.join(build_dir, '.env')) # traversal every env file and get container_id, and then try to stop it and rm it for env in env_list: @@ -90,11 +96,13 @@ class Clear(OebuildCommand): try: container_id = env_conf['container']['short_id'] container = self.client.get_container( - container_id=container_id) + container_id=container_id + ) DockerProxy().stop_container(container=container) DockerProxy().delete_container(container=container) - logger.info("Delete container: %s successful", - container.short_id) + logger.info( + 'Delete container: %s successful', container.short_id + ) except DockerException: continue except KeyError: @@ -113,4 +121,4 @@ class Clear(OebuildCommand): # except: # continue - logger.info("clear container finished") + logger.info('clear container finished') diff --git a/src/oebuild/app/plugins/demo/demo.py b/src/oebuild/app/plugins/demo/demo.py index 5548ec7..86e464b 100644 --- a/src/oebuild/app/plugins/demo/demo.py +++ b/src/oebuild/app/plugins/demo/demo.py @@ -1,37 +1,46 @@ +""" +Demo plugin for oebuild command system. +Provides example implementation of OebuildCommand. +""" + import argparse -import textwrap import logging +import textwrap from oebuild.command import OebuildCommand -from oebuild.util import * from oebuild.configure import Configure logger = logging.getLogger() class Demo(OebuildCommand): + """ + Demo command class for testing and demonstration purposes. + """ def __init__(self): self.configure = Configure() super().__init__( - name='{}', - help='this is your help mesasge', - description=textwrap.dedent('''\ + name='demo', + help_msg='this is your help message', + description=textwrap.dedent("""\ this is your description message -''' - )) +"""), + ) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( parser_adder, - usage=''' + usage=""" %(prog)s [-m URL] [--mr REVISION] [--mf FILE] [directory] %(prog)s -l [--mf FILE] directory -''') +""", + ) return parser def do_run(self, args: argparse.Namespace, unknown=None): + """Execute the demo command.""" args = args.parse_args(unknown) - pass + # Demo command implementation would go here diff --git a/src/oebuild/app/plugins/deploy/com_target.py b/src/oebuild/app/plugins/deploy/com_target.py index 24f221d..d5de29e 100644 --- a/src/oebuild/app/plugins/deploy/com_target.py +++ b/src/oebuild/app/plugins/deploy/com_target.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os import sys @@ -24,15 +24,15 @@ from oebuild.parse_param import ParseCompileParam from oebuild.m_log import logger import oebuild.const as oebuild_const -TARGET_DIR_NAME = "target_dev" -TARGET_SCRIPT_NAME = "oebuild_dev" +TARGET_DIR_NAME = 'target_dev' +TARGET_SCRIPT_NAME = 'oebuild_dev' class ComTarget: - ''' + """ The class is used to deploy-target and undeploy-target, this is a main body, the deploy-target and undeploy-target is just a interface and finally go into ComTarget - ''' + """ def __init__(self) -> None: self.configure = Configure() @@ -45,7 +45,9 @@ class ComTarget: if self.client is not None: try: container = self.client.get_container(self.container_id) - self.client.delete_container(container=container, is_force=True) + self.client.delete_container( + container=container, is_force=True + ) except DockerException: print(f""" the container {self.container_id} failed to be destroyed, please run @@ -55,33 +57,37 @@ the container {self.container_id} failed to be destroyed, please run """) def exec(self, str_args: str, fun): - ''' + """ the exec as a common function that will be invoked by deploy-target and undeploy-target, it means this is a entry runner. - ''' + """ self.client = DockerProxy() if not self._check_compile_directory(): - logger.error("You must be worked in compile directory") + logger.error('You must be worked in compile directory') sys.exit(-1) if not self._check_if_docker_compile(): - logger.error("The deploy function only be supported in working with docker build") + logger.error( + 'The deploy function only be supported in working with docker build' + ) sys.exit(-1) if not self._check_yocto_poky(): - logger.error(''' + logger.error(""" Please make sure that yocto-poky source code be in src directory, or you can run: - oebuild update layer''') + oebuild update layer""") sys.exit(-1) if not self._check_conf_directory(): - logger.error('You must work in a exist build directory, ' - 'that mean you built before or initialize ' - 'environment at least') + logger.error( + 'You must work in a exist build directory, ' + 'that mean you built before or initialize ' + 'environment at least' + ) sys.exit(-1) - logger.info("Initializing environment, please wait ...") + logger.info('Initializing environment, please wait ...') self.deal_env_container(oebuild_const.DEFAULT_DOCKER) container: Container = self.client.get_container(self.container_id) self._make_and_copy_lib(container=container) @@ -90,91 +96,115 @@ the container {self.container_id} failed to be destroyed, please run content = self._get_bashrc_content(container=container) content = oebuild_util.add_bashrc( content=content, - line=f"export PATH=$PATH:/home/openeuler/{TARGET_DIR_NAME}") + line=f'export PATH=$PATH:/home/openeuler/{TARGET_DIR_NAME}', + ) content = oebuild_util.add_bashrc( content=content, - line=(f"mv -f /home/{oebuild_const.CONTAINER_USER}/{self.old_bashrc} " - f"/home/{oebuild_const.CONTAINER_USER}/.bashrc") - ) + line=( + f'mv -f /home/{oebuild_const.CONTAINER_USER}/{self.old_bashrc} ' + f'/home/{oebuild_const.CONTAINER_USER}/.bashrc' + ), + ) content = oebuild_util.add_bashrc( - content=content, - line=f"{TARGET_SCRIPT_NAME} {fun} {str_args}") - print(f"{TARGET_SCRIPT_NAME} {str_args}") + content=content, line=f'{TARGET_SCRIPT_NAME} {fun} {str_args}' + ) + print(f'{TARGET_SCRIPT_NAME} {str_args}') content = oebuild_util.add_bashrc( content=content, - line=f"rm -rf /home/openeuler/{TARGET_DIR_NAME} && exit") + line=f'rm -rf /home/openeuler/{TARGET_DIR_NAME} && exit', + ) self.update_bashrc(container=container, content=content) - os.system(f"docker exec -it -u {oebuild_const.CONTAINER_USER} {container.short_id} bash") + os.system( + f'docker exec -it -u {oebuild_const.CONTAINER_USER} {container.short_id} bash' + ) - def _check_conf_directory(self,): + def _check_conf_directory( + self, + ): # check if exists local.conf - if not os.path.exists(os.path.join(self.work_dir, "conf/local.conf")): + if not os.path.exists(os.path.join(self.work_dir, 'conf/local.conf')): return False # check if exists bblayers.conf - if not os.path.exists(os.path.join(self.work_dir, "conf/bblayers.conf")): + if not os.path.exists( + os.path.join(self.work_dir, 'conf/bblayers.conf') + ): return False return True def _check_if_docker_compile(self): - ''' + """ the deploy feature should only be runed in docker build type - ''' - compile_path = os.path.join(self.work_dir, "compile.yaml") + """ + compile_path = os.path.join(self.work_dir, 'compile.yaml') compile_dict = oebuild_util.read_yaml(yaml_path=compile_path) - parse_compile = ParseCompileParam.parse_to_obj(compile_param_dict=compile_dict) + parse_compile = ParseCompileParam.parse_to_obj( + compile_param_dict=compile_dict + ) if parse_compile.build_in != oebuild_const.BUILD_IN_DOCKER: return False return True - def _check_compile_directory(self,): - ''' + def _check_compile_directory( + self, + ): + """ The execution of the bitbake instruction mainly relies on compile.yaml, which is initialized by parsing the file - ''' + """ return os.path.exists(os.path.join(self.work_dir, 'compile.yaml')) - def _check_yocto_poky(self,): - ''' + def _check_yocto_poky( + self, + ): + """ package deploy need poky lib, so we have a detect about yocto-poky - ''' + """ return os.path.exists(self.configure.source_poky_dir()) def _make_and_copy_lib(self, container: Container): # everytime, we should make sure that script is updated, so we make a rm action before copy - container.exec_run(f"rm -rf /home/openeuler/{TARGET_DIR_NAME}") + container.exec_run(f'rm -rf /home/openeuler/{TARGET_DIR_NAME}') # copy package lib to docker curr_path = os.path.dirname(os.path.realpath(__file__)) lib_path = os.path.join(curr_path, TARGET_DIR_NAME) self.client.copy_to_container( container=container, source_path=lib_path, - to_path="/home/openeuler/") - container.exec_run(f"chmod 755 /home/openeuler/{TARGET_DIR_NAME}/{TARGET_SCRIPT_NAME}") + to_path='/home/openeuler/', + ) + container.exec_run( + f'chmod 755 /home/openeuler/{TARGET_DIR_NAME}/{TARGET_SCRIPT_NAME}' + ) def deal_env_container(self, docker_image: str): - ''' + """ This operation realizes the processing of the container, controls how the container is processed by parsing the env variable, if the container does not exist, or the original environment and the current environment that needs to be set are inconsistent, you need to create a new container, otherwise directly enable the sleeping container - ''' + """ cwd_name = os.path.basename(self.work_dir) volumns = [] - volumns.append("/dev/net/tun:/dev/net/tun") - volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) - volumns.append(os.path.join(self.configure.build_dir(), cwd_name) - + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, cwd_name)) + volumns.append('/dev/net/tun:/dev/net/tun') + volumns.append( + self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC + ) + volumns.append( + os.path.join(self.configure.build_dir(), cwd_name) + + ':' + + os.path.join(oebuild_const.CONTAINER_BUILD, cwd_name) + ) parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS container: Container = self.client.create_container( image=docker_image, parameters=parameters, volumes=volumns, - command="bash") + command='bash', + ) self.container_id = container.short_id container: Container = self.client.get_container(self.container_id) # type: ignore @@ -182,34 +212,41 @@ the container {self.container_id} failed to be destroyed, please run self.client.start_container(container) def bak_bash(self, container: Container): - ''' + """ before we alter .bashrc we should make a copy or named bak to another where, thus when we finished some thing we can restore it. - ''' + """ old_bash = oebuild_util.generate_random_str(6) self.client.container_exec_command( container=container, - command=(f"cp /home/{oebuild_const.CONTAINER_USER}/.bashrc " - f"/home/{oebuild_const.CONTAINER_USER}/{old_bash}"), - user="root", - params={'stream': False}) + command=( + f'cp /home/{oebuild_const.CONTAINER_USER}/.bashrc ' + f'/home/{oebuild_const.CONTAINER_USER}/{old_bash}' + ), + user='root', + params={'stream': False}, + ) self.old_bashrc = old_bash def init_bash(self, container: Container): - ''' + """ Bitbake will initialize the compilation environment by reading the user initialization script first, then making directional substitutions, and finally writing the initialization script - ''' + """ self._check_change_ugid(container=container) # read container default user .bashrc content content = self._get_bashrc_content(container=container) # get nativesdk environment path automatic for next step - sdk_env_path = oebuild_util.get_nativesdk_environment(container=container) + sdk_env_path = oebuild_util.get_nativesdk_environment( + container=container + ) init_sdk_command = f'. {oebuild_const.NATIVESDK_DIR}/{sdk_env_path}' build_dir_name = os.path.basename(self.work_dir) - init_oe_command = f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ + init_oe_command = ( + f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ {oebuild_const.CONTAINER_BUILD}/{build_dir_name}' + ) init_command = [init_sdk_command, init_oe_command] new_content = oebuild_util.init_bashrc_content(content, init_command) self.update_bashrc(container=container, content=new_content) @@ -217,27 +254,31 @@ the container {self.container_id} failed to be destroyed, please run def _get_bashrc_content(self, container: Container): content = self.client.container_exec_command( container=container, - command=f"cat /home/{oebuild_const.CONTAINER_USER}/.bashrc", - user="root", - params={'stream': False}).output + command=f'cat /home/{oebuild_const.CONTAINER_USER}/.bashrc', + user='root', + params={'stream': False}, + ).output return content.decode() def update_bashrc(self, container: Container, content: str): - ''' + """ update user initialization script by replace file, first create a file and writed content and copy it to container's .bashrc, finally remove it - ''' + """ tmp_file = self._set_tmpfile_content(content) self.client.copy_to_container( container=container, source_path=tmp_file, - to_path=f'/home/{oebuild_const.CONTAINER_USER}') + to_path=f'/home/{oebuild_const.CONTAINER_USER}', + ) container.exec_run( - cmd=(f"mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} " - f"/home/{oebuild_const.CONTAINER_USER}/.bashrc"), - user="root" + cmd=( + f'mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} ' + f'/home/{oebuild_const.CONTAINER_USER}/.bashrc' + ), + user='root', ) os.remove(tmp_file) @@ -246,58 +287,73 @@ the container {self.container_id} failed to be destroyed, please run tmp_file = oebuild_util.generate_random_str(6) if os.path.exists(tmp_file): continue - with open(tmp_file, 'w', encoding="utf-8") as w_f: + with open(tmp_file, 'w', encoding='utf-8') as w_f: w_f.write(content) break return tmp_file def restore_bashrc(self, container: Container): - ''' + """ Restoring .bashrc will strip out the command line content added during bitbake initialization - ''' + """ old_content = self._get_bashrc_content(container=container) - self.update_bashrc(container=container, - content=self._restore_bashrc_content(old_content=old_content)) + self.update_bashrc( + container=container, + content=self._restore_bashrc_content(old_content=old_content), + ) def _restore_bashrc_content(self, old_content): new_content = '' for line in old_content.split('\n'): line: str = line - if line.endswith(oebuild_const.BASH_END_FLAG) or line.replace(" ", '') == '': + if ( + line.endswith(oebuild_const.BASH_END_FLAG) + or line.replace(' ', '') == '' + ): continue new_content = new_content + line + '\n' return new_content def _check_change_ugid(self, container: Container): params = { - 'work_space': f"/home/{oebuild_const.CONTAINER_USER}", - 'stream': False} + 'work_space': f'/home/{oebuild_const.CONTAINER_USER}', + 'stream': False, + } res = self.client.container_exec_command( container=container, user='root', - command=f"id {oebuild_const.CONTAINER_USER}", - params=params) + command=f'id {oebuild_const.CONTAINER_USER}', + params=params, + ) if res.exit_code != 0: - raise ValueError("check docker user id faild") + raise ValueError('check docker user id faild') res_cont: str = res.output.decode() cuids = res_cont.split(' ') # get uid from container in default user - pattern = re.compile(r'(?<=uid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))') + pattern = re.compile( + r'(?<=uid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))' + ) match_uid = pattern.search(cuids[0]) if match_uid: cuid = match_uid.group() else: - raise ValueError(f"can not get container {oebuild_const.CONTAINER_USER} uid") + raise ValueError( + f'can not get container {oebuild_const.CONTAINER_USER} uid' + ) # get gid from container in default user - pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))') + pattern = re.compile( + r'(?<=gid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))' + ) match_gid = pattern.search(cuids[1]) if match_gid: cgid = match_gid.group() else: - raise ValueError(f"can not get container {oebuild_const.CONTAINER_USER} gid") + raise ValueError( + f'can not get container {oebuild_const.CONTAINER_USER} gid' + ) # judge host uid and gid are same with container uid and gid # if not same and change container uid and gid equal to host's uid and gid @@ -308,22 +364,24 @@ the container {self.container_id} failed to be destroyed, please run def _change_container_uid(self, container: Container, uid: int): params = { - 'work_space': f"/home/{oebuild_const.CONTAINER_USER}", - 'stream': False + 'work_space': f'/home/{oebuild_const.CONTAINER_USER}', + 'stream': False, } self.client.container_exec_command( container=container, user='root', - command=f"usermod -u {uid} {oebuild_const.CONTAINER_USER}", - params=params) + command=f'usermod -u {uid} {oebuild_const.CONTAINER_USER}', + params=params, + ) def _change_container_gid(self, container: Container, gid: int): params = { - 'work_space': f"/home/{oebuild_const.CONTAINER_USER}", - 'stream': False + 'work_space': f'/home/{oebuild_const.CONTAINER_USER}', + 'stream': False, } self.client.container_exec_command( container=container, user='root', - command=f"groupmod -g {gid} {oebuild_const.CONTAINER_USER}", - params=params) + command=f'groupmod -g {gid} {oebuild_const.CONTAINER_USER}', + params=params, + ) diff --git a/src/oebuild/app/plugins/deploy/deploy_target.py b/src/oebuild/app/plugins/deploy/deploy_target.py index cd144c3..b61630c 100644 --- a/src/oebuild/app/plugins/deploy/deploy_target.py +++ b/src/oebuild/app/plugins/deploy/deploy_target.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import textwrap @@ -22,29 +22,31 @@ logger = logging.getLogger() class DeployTarget(OebuildCommand): - ''' + """ we use package in a - ''' + """ help_msg = 'deploy software on line' - description = textwrap.dedent('''\ + description = textwrap.dedent("""\ Deploys a recipe's build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand. - ''') + """) def __init__(self) -> None: super().__init__('{}', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" oebuild deploy-target [-h] [-c] [-s] [-n] [-p] [--no-check-space] [-e SSH_EXEC] [-P PORT] [-I KEY] [-S | --no-strip] recipename target -''') +""", + ) return parser @@ -54,9 +56,11 @@ oebuild deploy-target [-h] [-c] [-s] [-n] [-p] [--no-check-space] [-e SSH_EXEC] sys.exit(0) str_args = ' '.join(unknown) com_target = ComTarget() - com_target.exec(str_args=str_args, fun="deploy-target") + com_target.exec(str_args=str_args, fun='deploy-target') - def print_help_msg(self, ): + def print_help_msg( + self, + ): print(""" usage: oebuild deploy-target [-h] [-c] [-s] [-n] [-p] [--no-check-space] [-e SSH_EXEC] [-P PORT] [-I KEY] [-S | --no-strip] recipename target @@ -91,25 +95,27 @@ options: class UnDeployTarget(OebuildCommand): - ''' + """ we use package in a - ''' + """ help_msg = 'undeploy software on line' - description = textwrap.dedent('''\ + description = textwrap.dedent("""\ Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target. - ''') + """) def __init__(self) -> None: super().__init__('{}', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" oebuild undeploy-target [-h] [-c] [-s] [-a] [-n] [-e SSH_EXEC] [-P PORT] [-I KEY] [recipename] target -''') +""", + ) return parser @@ -119,7 +125,7 @@ oebuild undeploy-target [-h] [-c] [-s] [-a] [-n] [-e SSH_EXEC] sys.exit(0) str_args = ' '.join(unknown) com_target = ComTarget() - com_target.exec(str_args=str_args, fun="undeploy-target") + com_target.exec(str_args=str_args, fun='undeploy-target') def print_help_msg(self): print(""" diff --git a/src/oebuild/app/plugins/deploy/target_dev/deploy.py b/src/oebuild/app/plugins/deploy/target_dev/deploy.py index 233e179..ccc407d 100644 --- a/src/oebuild/app/plugins/deploy/target_dev/deploy.py +++ b/src/oebuild/app/plugins/deploy/target_dev/deploy.py @@ -12,18 +12,23 @@ import shutil import subprocess import tempfile -import bb.utils import argparse_oe import oe.types - -from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError +from devtool import DevtoolError, exec_fakeroot, setup_tinfoil logger = logging.getLogger('devtool') -deploylist_path = '/.devtool' +DEPLOYLIST_PATH = '/.devtool' -def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): +def _prepare_remote_script( + deploy, + verbose=False, + dryrun=False, + undeployall=False, + nopreserve=False, + nocheckspace=False, +): """ Prepare a shell script for running on the target to deploy/undeploy files. We have to be careful what we put in this @@ -36,17 +41,17 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals lines.append('set -e') if undeployall: # Yes, I know this is crude - but it does work - lines.append('for entry in %s/*.list; do' % deploylist_path) + lines.append('for entry in %s/*.list; do' % DEPLOYLIST_PATH) lines.append('[ ! -f $entry ] && exit') lines.append('set `basename $entry | sed "s/.list//"`') if dryrun: if not deploy: lines.append('echo "Previously deployed files for $1:"') - lines.append('manifest="%s/$1.list"' % deploylist_path) - lines.append('preservedir="%s/$1.preserve"' % deploylist_path) + lines.append('manifest="%s/$1.list"' % DEPLOYLIST_PATH) + lines.append('preservedir="%s/$1.preserve"' % DEPLOYLIST_PATH) lines.append('if [ -f $manifest ] ; then') # Read manifest in reverse and delete files / remove empty dirs - lines.append(' sed \'1!G;h;$!d\' $manifest | while read file') + lines.append(" sed '1!G;h;$!d' $manifest | while read file") lines.append(' do') if dryrun: lines.append(' if [ ! -d $file ] ; then') @@ -76,17 +81,22 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals # partitions, but doing that is non-trivial # Find the part of the destination path that exists lines.append('checkpath="$2"') - lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]') + lines.append( + 'while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]' + ) lines.append('do') lines.append(' checkpath=`dirname "$checkpath"`') lines.append('done') lines.append( - r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') + r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")' + ) # First line of the file is the total space lines.append('total=`head -n1 $3`') lines.append('if [ $total -gt $freespace ] ; then') lines.append( - ' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') + ' echo "ERROR: insufficient space on target ' + '(available ${freespace}, needed ${total})"' + ) lines.append(' exit 1') lines.append('fi') if not nopreserve: @@ -140,8 +150,9 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals def deploy(args, config, basepath, workspace): """Entry point for the devtool 'deploy' subcommand""" import math - import oe.recipeutils + import oe.package + import oe.recipeutils # check_workspace_recipe(workspace, args.recipename, checksrc=False) @@ -159,25 +170,41 @@ def deploy(args, config, basepath, workspace): try: rd = tinfoil.parse_recipe(args.recipename) except Exception as e: - raise DevtoolError('Exception parsing recipe %s: %s' % - (args.recipename, e)) + raise DevtoolError( + 'Exception parsing recipe %s: %s' % (args.recipename, e) + ) recipe_outdir = rd.getVar('D') if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): - raise DevtoolError('No files to deploy - have you built the %s ' - 'recipe? If so, the install step has not installed ' - 'any files.' % args.recipename) + raise DevtoolError( + 'No files to deploy - have you built the %s ' + 'recipe? If so, the install step has not installed ' + 'any files.' % args.recipename + ) if args.strip and not args.dry_run: # Fakeroot copy to new destination srcdir = recipe_outdir - recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped') + recipe_outdir = os.path.join( + rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped' + ) if os.path.isdir(recipe_outdir): - exec_fakeroot(rd, "rm -rf %s" % recipe_outdir, shell=True) - exec_fakeroot(rd, "cp -af %s %s" % - (os.path.join(srcdir, '.'), recipe_outdir), shell=True) - os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) - oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), - rd.getVar('base_libdir'), rd) + exec_fakeroot(rd, 'rm -rf %s' % recipe_outdir, shell=True) + exec_fakeroot( + rd, + 'cp -af %s %s' % (os.path.join(srcdir, '.'), recipe_outdir), + shell=True, + ) + os.environ['PATH'] = ':'.join( + [os.environ['PATH'], rd.getVar('PATH') or ''] + ) + oe.package.strip_execs( + args.recipename, + recipe_outdir, + rd.getVar('STRIP'), + rd.getVar('libdir'), + rd.getVar('base_libdir'), + rd, + ) filelist = [] inodes = set({}) @@ -191,35 +218,42 @@ def deploy(args, config, basepath, workspace): if fstat.st_ino in inodes: fsize = 0 else: - fsize = int(math.ceil(float(fstat.st_size)/1024)) + fsize = int(math.ceil(float(fstat.st_size) / 1024)) inodes.add(fstat.st_ino) ftotalsize += fsize # The path as it would appear on the target - fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) + fpath = os.path.join( + destdir, os.path.relpath(root, recipe_outdir), fn + ) filelist.append((fpath, fsize)) if args.dry_run: - print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) + print( + 'Files to be deployed for %s on target %s:' + % (args.recipename, args.target) + ) for item, _ in filelist: print(' %s' % item) return 0 extraoptions = '' if args.no_host_check: - extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + extraoptions += ( + '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + ) if not args.show_status: extraoptions += ' -q' scp_sshexec = '' ssh_sshexec = 'ssh' if args.ssh_exec: - scp_sshexec = "-S %s" % args.ssh_exec + scp_sshexec = '-S %s' % args.ssh_exec ssh_sshexec = args.ssh_exec scp_port = '' ssh_port = '' if args.port: - scp_port = "-P %s" % args.port - ssh_port = "-p %s" % args.port + scp_port = '-P %s' % args.port + ssh_port = '-p %s' % args.port if args.key: extraoptions += ' -i %s' % args.key @@ -231,41 +265,78 @@ def deploy(args, config, basepath, workspace): tmpdir = tempfile.mkdtemp(prefix='devtool') try: tmpscript = '/tmp/devtool_deploy.sh' - tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') - shellscript = _prepare_remote_script(deploy=True, - verbose=args.show_status, - nopreserve=args.no_preserve, - nocheckspace=args.no_check_space) + tmpfilelist = os.path.join( + os.path.dirname(tmpscript), 'devtool_deploy.list' + ) + shellscript = _prepare_remote_script( + deploy=True, + verbose=args.show_status, + nopreserve=args.no_preserve, + nocheckspace=args.no_check_space, + ) # Write out the script to a file - with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: + with open( + os.path.join(tmpdir, os.path.basename(tmpscript)), 'w' + ) as f: f.write(shellscript) # Write out the file list - with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: + with open( + os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w' + ) as f: f.write('%d\n' % ftotalsize) for fpath, fsize in filelist: f.write('%s %d\n' % (fpath, fsize)) # Copy them to the target - ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, - extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + ret = subprocess.call( + 'scp %s %s %s %s/* %s:%s' + % ( + scp_sshexec, + scp_port, + extraoptions, + tmpdir, + args.target, + os.path.dirname(tmpscript), + ), + shell=True, + ) if ret != 0: - raise DevtoolError('Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target) + raise DevtoolError( + 'Failed to copy script to %s - rerun with -s to ' + 'get a complete error message' % args.target + ) finally: shutil.rmtree(tmpdir) # Now run the script - ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, - args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) + ret = exec_fakeroot( + rd, + "tar cf - . | %s %s %s %s 'sh %s %s %s %s'" + % ( + ssh_sshexec, + ssh_port, + extraoptions, + args.target, + tmpscript, + args.recipename, + destdir, + tmpfilelist, + ), + cwd=recipe_outdir, + shell=True, + ) if ret != 0: - raise DevtoolError('Deploy failed - rerun with -s to get a complete ' - 'error message') + raise DevtoolError( + 'Deploy failed - rerun with -s to get a complete error message' + ) logger.info('Successfully deployed %s' % recipe_outdir) files_list = [] for root, _, files in os.walk(recipe_outdir): for filename in files: - filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) + filename = os.path.relpath( + os.path.join(root, filename), recipe_outdir + ) files_list.append(os.path.join(destdir, filename)) finally: tinfoil.shutdown() @@ -277,27 +348,32 @@ def undeploy(args, config, basepath, workspace): """Entry point for the devtool 'undeploy' subcommand""" if args.all and args.recipename: raise argparse_oe.ArgumentUsageError( - 'Cannot specify -a/--all with a recipe name', 'undeploy-target') + 'Cannot specify -a/--all with a recipe name', 'undeploy-target' + ) elif not args.recipename and not args.all: raise argparse_oe.ArgumentUsageError( - 'If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') + "If you don't specify a recipe, you must specify -a/--all", + 'undeploy-target', + ) extraoptions = '' if args.no_host_check: - extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + extraoptions += ( + '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + ) if not args.show_status: extraoptions += ' -q' scp_sshexec = '' ssh_sshexec = 'ssh' if args.ssh_exec: - scp_sshexec = "-S %s" % args.ssh_exec + scp_sshexec = '-S %s' % args.ssh_exec ssh_sshexec = args.ssh_exec scp_port = '' ssh_port = '' if args.port: - scp_port = "-P %s" % args.port - ssh_port = "-p %s" % args.port + scp_port = '-P %s' % args.port + ssh_port = '-p %s' % args.port args.target = args.target.split(':')[0] @@ -305,25 +381,49 @@ def undeploy(args, config, basepath, workspace): try: tmpscript = '/tmp/devtool_undeploy.sh' shellscript = _prepare_remote_script( - deploy=False, dryrun=args.dry_run, undeployall=args.all) + deploy=False, dryrun=args.dry_run, undeployall=args.all + ) # Write out the script to a file with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: f.write(shellscript) # Copy it to the target - ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, - extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + ret = subprocess.call( + 'scp %s %s %s %s/* %s:%s' + % ( + scp_sshexec, + scp_port, + extraoptions, + tmpdir, + args.target, + os.path.dirname(tmpscript), + ), + shell=True, + ) if ret != 0: - raise DevtoolError('Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target) + raise DevtoolError( + 'Failed to copy script to %s - rerun with -s to ' + 'get a complete error message' % args.target + ) finally: shutil.rmtree(tmpdir) # Now run the script - ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, - extraoptions, args.target, tmpscript, args.recipename), shell=True) + ret = subprocess.call( + "%s %s %s %s 'sh %s %s'" + % ( + ssh_sshexec, + ssh_port, + extraoptions, + args.target, + tmpscript, + args.recipename, + ), + shell=True, + ) if ret != 0: - raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' - 'error message') + raise DevtoolError( + 'Undeploy failed - rerun with -s to get a complete error message' + ) if not args.all and not args.dry_run: logger.info('Successfully undeployed %s' % args.recipename) @@ -333,61 +433,135 @@ def undeploy(args, config, basepath, workspace): def register_commands(subparsers, context): """Register devtool subcommands from the deploy plugin""" - parser_deploy = subparsers.add_parser('deploy-target', - help='Deploy recipe output files to live target machine', - description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', - group='testbuild') + parser_deploy = subparsers.add_parser( + 'deploy-target', + help='Deploy recipe output files to live target machine', + description="Deploys a recipe's build output (i.e. the output of the " + "do_install task) to a live target machine over ssh. By default, " + "any existing files will be preserved instead of being overwritten " + "and will be restored if you run devtool undeploy-target. Note: " + "this only deploys the recipe itself and not any runtime " + "dependencies, so it is assumed that those have been installed " + "on the target beforehand.", + group='testbuild', + ) parser_deploy.add_argument('recipename', help='Recipe to deploy') parser_deploy.add_argument( - 'target', help='Live target machine running an ssh server: user@hostname[:destdir]') - parser_deploy.add_argument('-c', '--no-host-check', - help='Disable ssh host key checking', action='store_true') - parser_deploy.add_argument('-s', '--show-status', - help='Show progress/status output', action='store_true') + 'target', + help='Live target machine running an ssh server: user@hostname[:destdir]', + ) + parser_deploy.add_argument( + '-c', + '--no-host-check', + help='Disable ssh host key checking', + action='store_true', + ) + parser_deploy.add_argument( + '-s', + '--show-status', + help='Show progress/status output', + action='store_true', + ) parser_deploy.add_argument( - '-n', '--dry-run', help='List files to be deployed only', action='store_true') - parser_deploy.add_argument('-p', '--no-preserve', - help='Do not preserve existing files', action='store_true') + '-n', + '--dry-run', + help='List files to be deployed only', + action='store_true', + ) parser_deploy.add_argument( - '--no-check-space', help='Do not check for available space before deploying', action='store_true') - parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') + '-p', + '--no-preserve', + help='Do not preserve existing files', + action='store_true', + ) parser_deploy.add_argument( - '-P', '--port', help='Specify port to use for connection to the target') - parser_deploy.add_argument('-I', '--key', - help='Specify ssh private key for connection to the target') + '--no-check-space', + help='Do not check for available space before deploying', + action='store_true', + ) + parser_deploy.add_argument( + '-e', '--ssh-exec', help='Executable to use in place of ssh' + ) + parser_deploy.add_argument( + '-P', '--port', help='Specify port to use for connection to the target' + ) + parser_deploy.add_argument( + '-I', + '--key', + help='Specify ssh private key for connection to the target', + ) strip_opts = parser_deploy.add_mutually_exclusive_group(required=False) - strip_opts.add_argument('-S', '--strip', - help='Strip executables prior to deploying (default: %(default)s). ' - 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', - default=oe.types.boolean(context.config.get( - 'Deploy', 'strip', default='0')), - action='store_true') strip_opts.add_argument( - '--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') + '-S', + '--strip', + help='Strip executables prior to deploying (default: %(default)s). ' + 'The default value can be controlled by setting the strip option ' + 'in the [Deploy] section to True or False.', + default=oe.types.boolean( + context.config.get('Deploy', 'strip', default='0') + ), + action='store_true', + ) + strip_opts.add_argument( + '--no-strip', + help='Do not strip executables prior to deploy', + dest='strip', + action='store_false', + ) parser_deploy.set_defaults(func=deploy) - parser_undeploy = subparsers.add_parser('undeploy-target', - help='Undeploy recipe output files in live target machine', - description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', - group='testbuild') + parser_undeploy = subparsers.add_parser( + 'undeploy-target', + help='Undeploy recipe output files in live target machine', + description='Un-deploys recipe output files previously deployed to a ' + 'live target machine by devtool deploy-target.', + group='testbuild', + ) + parser_undeploy.add_argument( + 'recipename', + help='Recipe to undeploy (if not using -a/--all)', + nargs='?', + ) + parser_undeploy.add_argument( + 'target', + help='Live target machine running an ssh server: user@hostname', + ) + parser_undeploy.add_argument( + '-c', + '--no-host-check', + help='Disable ssh host key checking', + action='store_true', + ) parser_undeploy.add_argument( - 'recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') + '-s', + '--show-status', + help='Show progress/status output', + action='store_true', + ) parser_undeploy.add_argument( - 'target', help='Live target machine running an ssh server: user@hostname') - parser_undeploy.add_argument('-c', '--no-host-check', - help='Disable ssh host key checking', action='store_true') + '-a', + '--all', + help='Undeploy all recipes deployed on the target', + action='store_true', + ) parser_undeploy.add_argument( - '-s', '--show-status', help='Show progress/status output', action='store_true') + '-n', + '--dry-run', + help='List files to be undeployed only', + action='store_true', + ) parser_undeploy.add_argument( - '-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') + '-e', '--ssh-exec', help='Executable to use in place of ssh' + ) parser_undeploy.add_argument( - '-n', '--dry-run', help='List files to be undeployed only', action='store_true') - parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') + '-P', '--port', help='Specify port to use for connection to the target' + ) parser_undeploy.add_argument( - '-P', '--port', help='Specify port to use for connection to the target') - parser_undeploy.add_argument('-I', '--key', - help='Specify ssh private key for connection to the target') + '-I', + '--key', + help='Specify ssh private key for connection to the target', + ) parser_undeploy.set_defaults(func=undeploy) diff --git a/src/oebuild/app/plugins/docker-save/docker_save.py b/src/oebuild/app/plugins/docker-save/docker_save.py index 0337dec..5ab12c9 100644 --- a/src/oebuild/app/plugins/docker-save/docker_save.py +++ b/src/oebuild/app/plugins/docker-save/docker_save.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import sys @@ -25,18 +25,18 @@ from oebuild.docker_proxy import DockerProxy class DockerSave(OebuildCommand): - ''' + """ This class is designed to rapidly generate a customized container image, aiming to address scenarios where the compilation environment has been specially tailored but reuse of the container environment is required. - ''' + """ help_msg = 'help to save a docker image' - description = textwrap.dedent(''' + description = textwrap.dedent(""" This is designed to rapidly generate a customized container image, aiming to address scenarios where the compilation environment has been specially tailored but reuse of the container environment is required. - ''') + """) def __init__(self): self.configure = Configure() @@ -44,10 +44,12 @@ class DockerSave(OebuildCommand): super().__init__('docker-save', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [docker-image] -''') +""", + ) # Secondary command return parser @@ -59,31 +61,35 @@ class DockerSave(OebuildCommand): self.pre_parse_help(args, unknown) sys.exit(1) docker_image = unknown[0] - docker_image_split = docker_image.split(":") + docker_image_split = docker_image.split(':') if len(docker_image_split) != 2: - logger.error("the docker image format is repository:tag," - "should be set like openeuler:latest") + logger.error( + 'the docker image format is repository:tag,' + 'should be set like openeuler:latest' + ) sys.exit(-1) if not self.configure.is_oebuild_dir(): logger.error('Your current directory had not finished init') sys.exit(-1) - if ".env" not in os.listdir(): + if '.env' not in os.listdir(): # the command must run based on .env file - logger.error("dcommand need .env to get container id" - "so you must run it in compile directory") + logger.error( + 'dcommand need .env to get container id' + 'so you must run it in compile directory' + ) sys.exit(-1) - env_obj = ParseEnv(env_dir=".env") + env_obj = ParseEnv(env_dir='.env') if not self.client.is_container_exists(env_obj.container.short_id): - logger.error("the container id: %s is not exist in .env") + logger.error('the container id: %s is not exist in .env', env_obj.container.short_id) sys.exit(-1) container = self.client.get_container(env_obj.container.short_id) - logger.info("the docker image %s is generatting ...", docker_image) + logger.info('the docker image %s is generatting ...', docker_image) try: container.commit(docker_image_split[0], docker_image_split[1]) except APIError: - logger.error("save %s failed") + logger.error('save %s failed', docker_image) sys.exit(-1) - logger.info("the new docker image %s is generated", docker_image) + logger.info('the new docker image %s is generated', docker_image) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index 547aa1c..5cfefca 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import re @@ -26,9 +26,11 @@ from oebuild.command import OebuildCommand import oebuild.util as oebuild_util from oebuild.configure import Configure from oebuild.parse_template import ( - BaseParseTemplate, ParseTemplate, + BaseParseTemplate, + ParseTemplate, get_docker_param_dict, - parse_repos_layers_local_obj) + parse_repos_layers_local_obj, +) from oebuild.m_log import logger from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const @@ -40,10 +42,10 @@ class Generate(OebuildCommand): """Generate compile.yaml (and toolchain.yaml) from CLI options.""" help_msg = 'Create build dir and generate compile.yaml' - description = textwrap.dedent('''\ + description = textwrap.dedent("""\ Customize build parameters and output compile.yaml; optionally emit toolchain.yaml for GCC/LLVM builds. - ''') + """) def __init__(self): self.configure = Configure() @@ -56,15 +58,19 @@ class Generate(OebuildCommand): # tmp_dir # cache_src_dir self.params = {} - self.oebuild_kconfig_path = os.path.expanduser( - '~') + '/.local/oebuild_kconfig/' + self.oebuild_kconfig_path = ( + os.path.expanduser('~') + '/.local/oebuild_kconfig/' + ) super().__init__('generate', self.help_msg, self.description) def do_add_parser(self, parser_adder): """Add arguments to the parser.""" - parser = self._parser(parser_adder, usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s -''') +""", + ) parser = parsers(parser) @@ -82,15 +88,16 @@ class Generate(OebuildCommand): yocto_dir = self.configure.source_yocto_dir() if not self.check_support_oebuild(yocto_dir): logger.error( - 'yocto-meta-openeuler does not container valid ' - 'oebuild metadata ' - 'Update .oebuild/config and re-run `oebuild update`.') + 'yocto-meta-openeuler does not container valid oebuild metadata ' + 'Update .oebuild/config and re-run `oebuild update`.' + ) sys.exit(-1) if len(unknown) == 0: yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') kconfig_generator = KconfigGenerator( - self.oebuild_kconfig_path, yocto_oebuild_dir) + self.oebuild_kconfig_path, yocto_oebuild_dir + ) config_path = kconfig_generator.create_kconfig() if not os.path.exists(config_path): sys.exit(0) @@ -104,7 +111,7 @@ class Generate(OebuildCommand): if args.nativesdk: # default dir for nativesdk if args.directory is None or args.directory == '': - args.directory = "nativesdk" + args.directory = 'nativesdk' build_dir = self._init_build_dir(args=args) if build_dir is None: sys.exit(0) @@ -115,19 +122,21 @@ class Generate(OebuildCommand): if args.gcc: # default dir for toolchain if args.directory is None or args.directory == '': - args.directory = "toolchain" + args.directory = 'toolchain' toolchain_name_list = args.gcc_name if args.gcc_name else [] build_dir = self._init_build_dir(args=args) if build_dir is None: sys.exit(0) self.build_gcc(build_dir, toolchain_name_list, auto_build) - self._print_toolchain(build_dir=build_dir, ) + self._print_toolchain( + build_dir=build_dir, + ) sys.exit(0) if args.llvm: # default dir for toolchain if args.directory is None or args.directory == '': - args.directory = "toolchain" + args.directory = 'toolchain' llvm_lib = args.llvm_lib build_dir = self._init_build_dir(args=args) if build_dir is None: @@ -161,14 +170,21 @@ class Generate(OebuildCommand): self.list_info() sys.exit(0) + build_dir = self._init_build_dir(args=args) + + if build_dir is None: + sys.exit(1) + parser_template = ParseTemplate(yocto_dir=yocto_dir) yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') try: - self._add_platform_template(args=args, - yocto_oebuild_dir=yocto_oebuild_dir, - parser_template=parser_template) + self._add_platform_template( + args=args, + yocto_oebuild_dir=yocto_oebuild_dir, + parser_template=parser_template, + ) except BaseParseTemplate as b_t: logger.error(str(b_t)) sys.exit(-1) @@ -177,9 +193,11 @@ class Generate(OebuildCommand): sys.exit(-1) try: - self._add_features_template(args=args, - yocto_oebuild_dir=yocto_oebuild_dir, - parser_template=parser_template) + self._add_features_template( + args=args, + yocto_oebuild_dir=yocto_oebuild_dir, + parser_template=parser_template, + ) except BaseParseTemplate as b_t: logger.error(str(b_t)) self._list_feature() @@ -188,25 +206,24 @@ class Generate(OebuildCommand): logger.error(str(v_e)) sys.exit(-1) - build_dir = self._init_build_dir(args=args) - - if build_dir is None: - sys.exit(1) - compile_yaml_path = pathlib.Path(build_dir, 'compile.yaml') if compile_yaml_path.exists(): compile_yaml_path.unlink() docker_image = get_docker_image( - yocto_dir=self.configure.source_yocto_dir(), - docker_tag='', - configure=self.configure) + yocto_dir=self.configure.source_yocto_dir(), + docker_tag='', + configure=self.configure, + ) + + out_dir = pathlib.Path(build_dir, 'compile.yaml') param = parser_template.get_default_generate_compile_conf_param() param['nativesdk_dir'] = self.params.get('nativesdk_dir', None) param['toolchain_dir'] = self.params.get('toolchain_dir', None) param['llvm_toolchain_dir'] = self.params.get( - 'llvm_toolchain_dir', None) + 'llvm_toolchain_dir', None + ) param['build_in'] = args.build_in param['sstate_mirrors'] = self.params.get('sstate_mirrors', None) param['sstate_dir'] = self.params.get('sstate_dir', None) @@ -219,13 +236,13 @@ class Generate(OebuildCommand): param['compile_dir'] = build_dir param['cache_src_dir'] = self.params.get('cache_src_dir', None) oebuild_util.write_yaml( - compile_yaml_path, - parser_template.generate_compile_conf(param)) + out_dir, parser_template.generate_compile_conf(param) + ) self._print_generate(build_dir=build_dir) def _print_generate(self, build_dir): - format_dir = f''' + format_dir = f""" generate compile.yaml successful Run commands below: @@ -235,11 +252,11 @@ cd {build_dir} oebuild bitbake ============================================= -''' +""" logger.info(format_dir) def _print_nativesdk(self, build_dir): - format_dir = f''' + format_dir = f""" generate compile.yaml successful Run commands below: @@ -249,11 +266,11 @@ cd {build_dir} oebuild bitbake or oebuild bitbake buildtools-extended-tarball ============================================= -''' +""" logger.info(format_dir) def _print_toolchain(self, build_dir): - format_dir = f''' + format_dir = f""" generate toolchain.yaml successful Run commands below: @@ -263,14 +280,16 @@ cd {build_dir} oebuild toolchain ============================================= -''' +""" logger.info(format_dir) - def _add_platform_template(self, args, yocto_oebuild_dir, - parser_template: ParseTemplate): + def _add_platform_template( + self, args, yocto_oebuild_dir, parser_template: ParseTemplate + ): platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') platform_files = [ - f.name for f in platform_path.iterdir() if f.is_file()] + f.name for f in platform_path.iterdir() if f.is_file() + ] if args.platform + '.yaml' in platform_files: try: platform_file = platform_path / (args.platform + '.yaml') @@ -279,16 +298,18 @@ oebuild toolchain raise e_p else: logger.error( - "Invalid platform. Run `oebuild generate -l` " - "to list supported platforms.") + 'Invalid platform. Run `oebuild generate -l` to list supported platforms.' + ) sys.exit(-1) - def _add_features_template(self, args, yocto_oebuild_dir, - parser_template: ParseTemplate): + def _add_features_template( + self, args, yocto_oebuild_dir, parser_template: ParseTemplate + ): if args.features: features_path = pathlib.Path(yocto_oebuild_dir, 'features') feature_files = [ - f.name for f in features_path.iterdir() if f.is_file()] + f.name for f in features_path.iterdir() if f.is_file() + ] for feature in args.features: if feature + '.yaml' in feature_files: try: @@ -298,8 +319,8 @@ oebuild toolchain raise b_t else: logger.error( - "Invalid feature. Run `oebuild generate -l` " - "to list features.") + 'Invalid feature. Run `oebuild generate -l` to list features.' + ) sys.exit(-1) def _init_build_dir(self, args): @@ -312,33 +333,45 @@ oebuild toolchain else: build_dir = build_dir_path / args.directory - if not pathlib.Path(build_dir).absolute().is_relative_to( - build_dir_path.absolute()): - logger.error("Build path must in oebuild workspace") + if ( + not pathlib.Path(build_dir) + .absolute() + .is_relative_to(build_dir_path.absolute()) + ): + logger.error('Build path must in oebuild workspace') return None # If build dir exists, prompt/handle overwrite if build_dir.exists(): - logger.warning("the build directory %s already exists", build_dir) + logger.warning('the build directory %s already exists', build_dir) while not args.yes: in_res = input(f""" - Overwrite {build_dir.name}? This will replace " - "compile.yaml/toolchain.yaml and delete conf/ + Overwrite {build_dir.name}? This will replace compile.yaml/toolchain.yaml and delete conf/ Enter Y=yes, N=no, C=create : """) - if in_res not in ["Y", "y", "yes", "N", "n", "no", - "C", "c", "create"]: - print("Invalid input") + if in_res not in [ + 'Y', + 'y', + 'yes', + 'N', + 'n', + 'no', + 'C', + 'c', + 'create', + ]: + print('Invalid input') continue if in_res in ['N', 'n', 'no']: return None if in_res in ['C', 'c', 'create']: in_res = input( - "Enter new build name (will be created under build/):") + 'Enter new build name (will be created under build/):' + ) build_dir = build_dir_path / in_res if build_dir.exists(): continue break - conf_dir = build_dir / "conf" + conf_dir = build_dir / 'conf' if conf_dir.exists(): rmtree(conf_dir) elif build_dir.exists(): @@ -346,22 +379,27 @@ oebuild toolchain build_dir.mkdir(parents=True, exist_ok=True) return str(build_dir) - def list_info(self, ): - ''' + def list_info( + self, + ): + """ print platform list or feature list - ''' + """ self._list_platform() self._list_feature() def _list_platform(self): - terminal_width = self._get_terminal_width() - print(int(terminal_width * 0.9) * '=') + logger.info( + '\n================= Available Platforms =================' + ) yocto_dir = self.configure.source_yocto_dir() - yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") + yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') platform_path = pathlib.Path(yocto_oebuild_dir, 'platform') list_platform = [f for f in platform_path.iterdir() if f.is_file()] + terminal_width = self._get_terminal_width() table = self._build_table( - ['Platform Name'], terminal_width, title='Available Platforms') + ['Platform Name'], terminal_width, title='Available Platforms' + ) for platform in list_platform: if platform.suffix in ['.yml', '.yaml']: table.add_row([platform.stem]) @@ -369,53 +407,53 @@ oebuild toolchain print(table) def _list_feature(self): - terminal_width = self._get_terminal_width() - print(int(terminal_width * 0.9) * '=') + logger.info( + '\n================= Available Features ==================' + ) yocto_dir = self.configure.source_yocto_dir() - yocto_oebuild_dir = pathlib.Path(yocto_dir, ".oebuild") + yocto_oebuild_dir = pathlib.Path(yocto_dir, '.oebuild') feature_triples = parse_feature_files(yocto_oebuild_dir) - table = self._build_table(['Feature Name', 'Supported Arch'], - terminal_width, title='Available Features') + terminal_width = self._get_terminal_width() + table = self._build_table( + ['Feature Name', 'Supported Arch'], + terminal_width, + title='Available Features', + ) for feature_name, _, feature_data in feature_triples: - table.add_row([feature_name, feature_data.get('support') or "all"]) + table.add_row([feature_name, feature_data.get('support') or 'all']) print(table) logger.info( - """* 'Supported Arch' defaults to 'all' if not " - "specified in the feature's .yaml file.""" + """* 'Supported Arch' defaults to 'all' if not specified in the feature's .yaml file.""" ) def _build_table(self, headers, terminal_width, title=None): narrow_charnum, narrow_colnum = 60, 10 - max_width = max(int(terminal_width), 20) + max_width = max(int(terminal_width * 0.9), 20) table = PrettyTable(headers, max_width=max_width) - table.align = "l" + table.align = 'l' table.header = True col_width = max(10, max_width // max(len(headers), 1)) for header in headers: table.max_width[header] = col_width - is_narrow = (terminal_width < narrow_charnum or - col_width < narrow_colnum) + is_narrow = ( + terminal_width < narrow_charnum or col_width < narrow_colnum + ) if is_narrow: - table.set_style(TableStyle.MSWORD_FRIENDLY) - table.hrules = HRuleStyle.FRAME + table.set_style(TableStyle.PLAIN_COLUMNS) + table.hrules = HRuleStyle.NONE table.vrules = VRuleStyle.NONE table.left_padding_width = 0 table.right_padding_width = 0 - if title: - table.title = title - table.header_style = 'title' else: - table.set_style(TableStyle.MARKDOWN) + table.set_style(TableStyle.SINGLE_BORDER) table.hrules = HRuleStyle.FRAME - table.vrules = VRuleStyle.ALL + table.vrules = VRuleStyle.FRAME table.left_padding_width = 1 - table.right_padding_width = 2 + table.right_padding_width = 1 if title: table.title = title - table.header_style = 'title' - return table @staticmethod @@ -426,34 +464,35 @@ oebuild toolchain return 80 def check_support_oebuild(self, yocto_dir): - ''' + """ Return True if /.oebuild exists. - ''' + """ return pathlib.Path(yocto_dir, '.oebuild').exists() def generate_command(self, config_path): """ - Parse .config and build argv list for 'oebuild generate'. + Parse .config and build argv list for 'oebuild generate'. """ with open(config_path, 'r', encoding='utf-8') as config_file: content = config_file.read() # sys.exit(0) - content = re.sub('#.*|.*None.*', "", content) + content = re.sub('#.*|.*None.*', '', content) common_list = re.findall('(?<=CONFIG_COMMON_).*', content) - platform_search = re.search(r"(?<=CONFIG_PLATFORM_).*(?==y)", content) - feature_list = re.findall(r"(?<=CONFIG_FEATURE_).*(?==y)", content) - build_in = re.search(r"(?<=CONFIG_BUILD_IN-).*(?==y)", content) - nativesdk = re.search(r"(?<=CONFIG_NATIVESDK).*(?==y)", content) - gcc = re.search(r"(?<=CONFIG_GCC-TOOLCHAIN).*(?==y)", content) - gcc_list = re.findall(r"(?<=CONFIG_GCC-TOOLCHAIN_).*(?==y)", content) - llvm = re.search(r"(?<=CONFIG_LLVM-TOOLCHAIN).*(?==y)", content) + platform_search = re.search(r'(?<=CONFIG_PLATFORM_).*(?==y)', content) + feature_list = re.findall(r'(?<=CONFIG_FEATURE_).*(?==y)', content) + build_in = re.search(r'(?<=CONFIG_BUILD_IN-).*(?==y)', content) + nativesdk = re.search(r'(?<=CONFIG_NATIVESDK).*(?==y)', content) + gcc = re.search(r'(?<=CONFIG_GCC-TOOLCHAIN).*(?==y)', content) + gcc_list = re.findall(r'(?<=CONFIG_GCC-TOOLCHAIN_).*(?==y)', content) + llvm = re.search(r'(?<=CONFIG_LLVM-TOOLCHAIN).*(?==y)', content) llvm_lib = re.search( - r"(?<=CONFIG_LLVM-TOOLCHAIN_AARCH64-LIB).*", content) - auto_build = re.search(r"(?<=CONFIG_AUTO-BUILD).*(?==y)", content) + r'(?<=CONFIG_LLVM-TOOLCHAIN_AARCH64-LIB).*', content + ) + auto_build = re.search(r'(?<=CONFIG_AUTO-BUILD).*(?==y)', content) g_command = [] for basic in common_list: - basic_info = basic.lower().replace("\"", "").split('=') - basic_info[0] = basic_info[0].replace("-", "_") + basic_info = basic.lower().replace('"', '').split('=') + basic_info[0] = basic_info[0].replace('-', '_') if basic_info[0] == 'no_fetch': g_command += ['--' + basic_info[0]] continue @@ -465,8 +504,9 @@ oebuild toolchain if build_in: g_command += ['-b_in', build_in.group().lower()] - platform = platform_search.group( - ) if platform_search else 'qemu-aarch64' + platform = ( + platform_search.group() if platform_search else 'qemu-aarch64' + ) g_command += ['-p', platform.lower()] for feature in feature_list: @@ -503,46 +543,47 @@ oebuild toolchain """ compile_dir = os.path.join(self.configure.build_dir(), build_dir) - compile_yaml_path = f"{compile_dir}/compile.yaml" + compile_yaml_path = f'{compile_dir}/compile.yaml' common_yaml_path = os.path.join( - self.configure.source_yocto_dir(), '.oebuild/common.yaml') + self.configure.source_yocto_dir(), '.oebuild/common.yaml' + ) repos, layers, local_conf = parse_repos_layers_local_obj( - common_yaml_path) - info = { - 'repos': repos, - 'layers': layers, - 'local_conf': local_conf - } + common_yaml_path + ) + info = {'repos': repos, 'layers': layers, 'local_conf': local_conf} if build_in == 'host': info['build_in'] = 'host' else: docker_image = get_docker_image( yocto_dir=self.configure.source_yocto_dir(), - docker_tag="latest", - configure=self.configure + docker_tag='latest', + configure=self.configure, ) info['docker_param'] = get_docker_param_dict( docker_image=docker_image, dir_list={ - "src_dir": self.configure.source_dir(), - "compile_dir": compile_dir, - "toolchain_dir": None, - "llvm_toolchain_dir": None, - "sstate_mirrors": None - } + 'src_dir': self.configure.source_dir(), + 'compile_dir': compile_dir, + 'toolchain_dir': None, + 'llvm_toolchain_dir': None, + 'sstate_mirrors': None, + }, ) # add nativesdk conf nativesdk_yaml_path = os.path.join( - self.configure.source_yocto_dir(), '.oebuild/nativesdk/local.conf') + self.configure.source_yocto_dir(), '.oebuild/nativesdk/local.conf' + ) with open(nativesdk_yaml_path, 'r', encoding='utf-8') as f: - local_conf += f.read()+"\n" + local_conf += f.read() + '\n' info['local_conf'] = LiteralScalarString(local_conf) oebuild_util.write_yaml(compile_yaml_path, info) if auto_build: os.chdir(compile_dir) subprocess.run( 'oebuild bitbake buildtools-extended-tarball', - shell=True, check=False) + shell=True, + check=False, + ) def build_gcc(self, build_dir, gcc_name_list, auto_build): """ @@ -555,44 +596,51 @@ oebuild toolchain """ source_cross_dir = pathlib.Path( - self.configure.source_yocto_dir(), ".oebuild/cross-tools") + self.configure.source_yocto_dir(), '.oebuild/cross-tools' + ) if not source_cross_dir.exists(): - logger.error('Build dependency not downloaded, not supported ' - 'for build. Please download the latest yocto ' - 'meta openeuler repository') + logger.error( + 'Build dependency not downloaded, not supported for build. Please ' + 'download the latest yocto meta openeuler repository' + ) sys.exit(-1) # add toolchain.yaml to compile docker_param = get_docker_param_dict( docker_image=get_sdk_docker_image( - yocto_dir=self.configure.source_yocto_dir()), + yocto_dir=self.configure.source_yocto_dir() + ), dir_list={ - "src_dir": self.configure.source_dir(), - "compile_dir": build_dir, - "toolchain_dir": None, - "llvm_toolchain_dir": None, - "sstate_mirrors": None - } - ) + 'src_dir': self.configure.source_dir(), + 'compile_dir': build_dir, + 'toolchain_dir': None, + 'llvm_toolchain_dir': None, + 'sstate_mirrors': None, + }, + ) config_list = [] for gcc_name in gcc_name_list: - if gcc_name.startswith("config_"): + if gcc_name.startswith('config_'): config_list.append(gcc_name) continue - config_list.append("config_" + gcc_name) + config_list.append('config_' + gcc_name) oebuild_util.write_yaml( yaml_path=pathlib.Path(build_dir, 'toolchain.yaml'), data={ 'kind': oebuild_const.GCC_TOOLCHAIN, 'gcc_configs': config_list, - 'docker_param': docker_param - } + 'docker_param': docker_param, + }, ) if auto_build: - with subprocess.Popen('oebuild toolchain auto', shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=build_dir, - encoding="utf-8", text=True) as s_p: + with subprocess.Popen( + 'oebuild toolchain auto', + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=build_dir, + encoding='utf-8', + text=True, + ) as s_p: if s_p.returncode is not None and s_p.returncode != 0: err_msg = '' if s_p.stderr is not None: @@ -621,38 +669,45 @@ oebuild toolchain """ source_llvm_dir = pathlib.Path( - self.configure.source_yocto_dir(), ".oebuild/llvm-toolchain") + self.configure.source_yocto_dir(), '.oebuild/llvm-toolchain' + ) if not source_llvm_dir.exists(): - logger.error('Build dependency not downloaded, not supported ' - 'for build. Please download the latest yocto ' - 'meta openeuler repository') + logger.error( + 'Build dependency not downloaded, not supported for build. Please ' + 'download the latest yocto meta openeuler repository' + ) sys.exit(-1) # add toolchain.yaml to compile docker_param = get_docker_param_dict( docker_image=get_sdk_docker_image( - yocto_dir=self.configure.source_yocto_dir()), + yocto_dir=self.configure.source_yocto_dir() + ), dir_list={ - "src_dir": self.configure.source_dir(), - "compile_dir": build_dir, - "toolchain_dir": None, - "llvm_toolchain_dir": None, - "sstate_mirrors": None - } - ) + 'src_dir': self.configure.source_dir(), + 'compile_dir': build_dir, + 'toolchain_dir': None, + 'llvm_toolchain_dir': None, + 'sstate_mirrors': None, + }, + ) oebuild_util.write_yaml( yaml_path=pathlib.Path(build_dir, 'toolchain.yaml'), data={ 'kind': oebuild_const.LLVM_TOOLCHAIN, 'llvm_lib': llvm_lib, - 'docker_param': docker_param - } + 'docker_param': docker_param, + }, ) if auto_build: - with subprocess.Popen('oebuild toolchain auto', shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=build_dir, - encoding="utf-8", text=True) as s_p: + with subprocess.Popen( + 'oebuild toolchain auto', + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=build_dir, + encoding='utf-8', + text=True, + ) as s_p: if s_p.returncode is not None and s_p.returncode != 0: err_msg = '' if s_p.stderr is not None: @@ -672,12 +727,12 @@ oebuild toolchain def get_docker_image(yocto_dir, docker_tag, configure: Configure): - ''' - Resolve docker image from env, config defaults, or user " - "selection, ordered by priority - ''' + """ + Resolve docker image from env, config defaults, or user selection, ordered by priority + """ docker_image = oebuild_util.get_docker_image_from_yocto( - yocto_dir=yocto_dir) + yocto_dir=yocto_dir + ) if docker_image is None: check_docker_tag = CheckDockerTag(docker_tag, configure) oebuild_config = configure.parse_oebuild_config() @@ -690,29 +745,29 @@ def get_docker_image(yocto_dir, docker_tag, configure: Configure): image_list = check_docker_tag.get_tags() for key, value in enumerate(image_list): - print( - f"{key}, {oebuild_config.docker.repo_url}:{value}") - k = input("Enter number: ") - if k == "q": + print(f'{key}, {oebuild_config.docker.repo_url}:{value}') + k = input('Enter number: ') + if k == 'q': sys.exit(0) try: index = int(k) docker_tag = image_list[index] break except IndexError: - print("Enter a valid number") + print('Enter a valid number') docker_tag = docker_tag.strip() docker_tag = docker_tag.strip('\n') - docker_image = f"{oebuild_config.docker.repo_url}:{docker_tag}" + docker_image = f'{oebuild_config.docker.repo_url}:{docker_tag}' return docker_image def get_sdk_docker_image(yocto_dir): - ''' + """ get toolchain docker image - ''' + """ docker_image = oebuild_util.get_sdk_docker_image_from_yocto( - yocto_dir=yocto_dir) + yocto_dir=yocto_dir + ) if docker_image is None: docker_image = oebuild_const.DEFAULT_SDK_DOCKER return docker_image diff --git a/src/oebuild/app/plugins/generate/kconfig_generator.py b/src/oebuild/app/plugins/generate/kconfig_generator.py index b786ae6..3a5c348 100644 --- a/src/oebuild/app/plugins/generate/kconfig_generator.py +++ b/src/oebuild/app/plugins/generate/kconfig_generator.py @@ -9,6 +9,7 @@ EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. """ + import os import pathlib import re @@ -21,12 +22,19 @@ from kconfiglib import Kconfig from menuconfig import menuconfig import oebuild.util as oebuild_util -from oebuild.m_log import logger from oebuild.app.plugins.generate.parses import parse_feature_files +from oebuild.m_log import logger + +# *** +# THIS KCONFIG GENERATOR IS TOO WEAK. +# DO NOT USE THIS FILE IN NEO-GENERATE*** +# *** class KconfigGenerator: - """Handles Kconfig file generation and menuconfig interaction.""" + """ + Handles Kconfig file generation and menuconfig interaction. + """ def __init__(self, oebuild_kconfig_path, yocto_oebuild_dir): self.oebuild_kconfig_path = oebuild_kconfig_path @@ -34,21 +42,24 @@ class KconfigGenerator: def create_kconfig(self): """Generate a temp Kconfig, launch menuconfig, and return the .config path.""" - write_data = "" + write_data = '' - target_list_data, gcc_toolchain_data, llvm_toolchain_data = \ + target_list_data, gcc_toolchain_data, llvm_toolchain_data = ( self._kconfig_add_target_list() - write_data += target_list_data + gcc_toolchain_data + llvm_toolchain_data + ) + write_data += ( + target_list_data + gcc_toolchain_data + llvm_toolchain_data + ) # add auto-build config - write_data += ''' + write_data += """ if NATIVESDK || GCC-TOOLCHAIN || LLVM-TOOLCHAIN comment "Enable auto build for nativesdk/GCC/LLVM toolchain" config AUTO-BUILD bool "auto build" default n endif -''' +""" platform_data = self._kconfig_add_choice_platform() write_data += platform_data @@ -60,15 +71,17 @@ endif write_data += common_data if not os.path.exists( - pathlib.Path(self.oebuild_kconfig_path).absolute()): + pathlib.Path(self.oebuild_kconfig_path).absolute() + ): os.makedirs(pathlib.Path(self.oebuild_kconfig_path).absolute()) - kconfig_path = pathlib.Path(self.oebuild_kconfig_path, - str(int(time.time()))) + kconfig_path = pathlib.Path( + self.oebuild_kconfig_path, str(int(time.time())) + ) with open(kconfig_path, 'w', encoding='utf-8') as kconfig_file: kconfig_file.write(write_data) kconf = Kconfig(filename=str(kconfig_path)) - os.environ["MENUCONFIG_STYLE"] = "aquatic selection=fg:white,bg:blue" + os.environ['MENUCONFIG_STYLE'] = 'aquatic selection=fg:white,bg:blue' with oebuild_util.suppress_print(): menuconfig(kconf) subprocess.check_output(f'rm -rf {kconfig_path}', shell=True) @@ -76,31 +89,27 @@ endif return config_path def _kconfig_add_target_list(self): - target_choice = textwrap.dedent(''' + target_choice = textwrap.dedent(""" comment "Select build target" choice prompt "Select build target" config IMAGE bool 'OS' -''') +""") gcc_toolchain_data = self._kconfig_add_gcc_toolchain() - if gcc_toolchain_data != "": + if gcc_toolchain_data != '': target_choice += ( - "config GCC-TOOLCHAIN \n" - " bool 'GCC TOOLCHAIN'\n\n") + "config GCC-TOOLCHAIN \n bool 'GCC TOOLCHAIN'\n\n" + ) llvm_toolchain_data = self._kconfig_add_llvm_toolchain() - if llvm_toolchain_data != "": + if llvm_toolchain_data != '': target_choice += ( - "config LLVM-TOOLCHAIN \n" - " bool 'LLVM TOOLCHAIN'\n\n" + "config LLVM-TOOLCHAIN \n bool 'LLVM TOOLCHAIN'\n\n" ) nativesdk_check = self._kconfig_check_nativesdk() if nativesdk_check: - target_choice += ( - "config NATIVESDK \n" - " bool 'NATIVESDK'\n\n" - ) - target_choice += "endchoice" + target_choice += "config NATIVESDK \n bool 'NATIVESDK'\n\n" + target_choice += 'endchoice' return target_choice, gcc_toolchain_data, llvm_toolchain_data def _kconfig_add_choice_platform(self): @@ -114,26 +123,30 @@ endif """ platform_path = pathlib.Path(self.yocto_oebuild_dir, 'platform') if platform_path.exists(): - platform_files = [f for f in platform_path.iterdir() - if f.is_file() and f.suffix in ['.yml', '.yaml']] + platform_files = [ + f + for f in platform_path.iterdir() + if f.is_file() and f.suffix in ['.yml', '.yaml'] + ] else: logger.error('Platform directory not found.') sys.exit(-1) - platform_start = textwrap.dedent(''' + platform_start = textwrap.dedent(""" if IMAGE comment "Select OS platform" choice prompt "Select platform" default PLATFORM_QEMU-AARCH64 - ''') + """) platform_end = """ endchoice endif""" for platform in platform_files: - platform_name = platform.stem.strip("\n") + platform_name = platform.stem.strip('\n') platform_info = ( - f" config PLATFORM_{platform_name.upper()}\n" - f" bool \"{platform_name}\"\n\n") + f' config PLATFORM_{platform_name.upper()}\n' + f' bool "{platform_name}"\n\n' + ) platform_start += platform_info platform_data = platform_start + platform_end return platform_data @@ -148,29 +161,34 @@ endif """ - feature_start = ''' + feature_start = """ if IMAGE comment "Select OS features" - ''' + """ feature_triples = parse_feature_files(self.yocto_oebuild_dir) for ft_name, _, feature_data in feature_triples: - support_str = "" + support_str = '' if 'support' in feature_data: raw_supports = feature_data['support'] validated_support_str = self.validate_and_format_platforms( - raw_supports) + raw_supports + ) if validated_support_str: support_str = validated_support_str else: logger.warning( - "supported platform str of feat %s is invalid: %s", - ft_name, raw_supports) - - feature_info = (f"\nconfig FEATURE_{ft_name.upper()}\n" - f" bool \"{ft_name}\" {support_str}\n\n") + 'supported platform str of feat %s is invalid: %s', + ft_name, + raw_supports, + ) + + feature_info = ( + f'\nconfig FEATURE_{ft_name.upper()}\n' + f' bool "{ft_name}" {support_str}\n\n' + ) feature_start += feature_info - feature_start += "endif" + feature_start += 'endif' return feature_start def validate_and_format_platforms(self, raw_str: str): @@ -181,9 +199,9 @@ endif platforms = re.split(r'[|,，\s]+', raw_str) platforms = [p.strip() for p in platforms if p.strip()] if not platforms: - return "" - platform_cond = [f"PLATFORM_{p.upper()}" for p in platforms] - return "if " + "||".join(platform_cond) + return '' + platform_cond = [f'PLATFORM_{p.upper()}' for p in platforms] + return 'if ' + '||'.join(platform_cond) def _kconfig_add_gcc_toolchain(self): """ @@ -194,23 +212,24 @@ endif Returns: """ - toolchain_start = "" - cross_dir = pathlib.Path(self.yocto_oebuild_dir, "cross-tools") + toolchain_start = '' + cross_dir = pathlib.Path(self.yocto_oebuild_dir, 'cross-tools') if cross_dir.exists(): configs_dir = cross_dir / 'configs' if configs_dir.exists(): toolchain_list = os.listdir(configs_dir) - toolchain_start += ''' + toolchain_start += """ if GCC-TOOLCHAIN - ''' + """ for config in toolchain_list: if not re.search('xml', config): toolchain_info = ( - f"""\nconfig GCC-TOOLCHAIN_{config.upper().lstrip("CONFIG_")}\n""" - f""" bool "{config.upper().lstrip("CONFIG_")}"\n""" - """ depends on GCC-TOOLCHAIN\n""") + f"""\nconfig GCC-TOOLCHAIN_{config.upper().lstrip('CONFIG_')}\n""" + f""" bool "{config.upper().lstrip('CONFIG_')}"\n""" + """ depends on GCC-TOOLCHAIN\n""" + ) toolchain_start += toolchain_info - toolchain_start += "endif" + toolchain_start += 'endif' return toolchain_start def _kconfig_add_llvm_toolchain(self): @@ -222,15 +241,15 @@ endif Returns: """ - toolchain_start = "" - llvm_dir = pathlib.Path(self.yocto_oebuild_dir, "llvm-toolchain") + toolchain_start = '' + llvm_dir = pathlib.Path(self.yocto_oebuild_dir, 'llvm-toolchain') if llvm_dir.exists(): - toolchain_start += ''' + toolchain_start += """ config LLVM-TOOLCHAIN-AARCH64-LIB string "aarch64 lib dir" default "None" depends on LLVM-TOOLCHAIN - ''' + """ return toolchain_start def _kconfig_check_nativesdk(self): @@ -242,26 +261,31 @@ endif Returns: """ - nativesdk_dir = pathlib.Path(self.yocto_oebuild_dir, "nativesdk") + nativesdk_dir = pathlib.Path(self.yocto_oebuild_dir, 'nativesdk') return nativesdk_dir.exists() - def _kconfig_add_common_config(self,): + def _kconfig_add_common_config( + self, + ): """ Build shared/common Kconfig options. Returns: """ toolchain_help = ( - "External GCC toolchain directory [your own toolchain]") + 'External GCC toolchain directory [your own toolchain]' + ) llvm_toolchain_help = ( - "External LLVM toolchain directory [your own toolchain]") + 'External LLVM toolchain directory [your own toolchain]' + ) nativesdk_help = ( - "External nativesdk directory [used when building on host]") - common_str = textwrap.dedent(''' + 'External nativesdk directory [used when building on host]' + ) + common_str = textwrap.dedent(""" comment "Common options" - ''') + """) # choice build in platform - common_str += (''' + common_str += """ if IMAGE choice prompt "Select build environment" @@ -272,81 +296,81 @@ endif bool "host" endchoice endif - ''') + """ # add no fetch - common_str += (''' + common_str += """ config COMMON_NO-FETCH bool "no_fetch (disable source fetching)" default n depends on IMAGE - ''') + """ # add no layer - common_str += (''' + common_str += """ config COMMON_NO-LAYER bool "no_layer (skip layer repo update on env setup)" default n depends on IMAGE - ''') + """ # add sstate_mirrors - common_str += (''' + common_str += """ config COMMON_SSTATE-MIRRORS string "SSTATE_MIRRORS value" default "None" depends on IMAGE - ''') + """ # add sstate_dir - common_str += (''' + common_str += """ config COMMON_SSTATE-DIR string "SSTATE_DIR path" default "None" depends on IMAGE - ''') + """ # add tmp_dir - common_str += (''' + common_str += """ config COMMON_TMP-DIR string "TMPDIR path" default "None" depends on IMAGE && BUILD_IN-HOST - ''') + """ # add gcc toolchain dir - common_str += (f''' + common_str += f""" config COMMON_TOOLCHAIN-DIR string "toolchain_dir ({toolchain_help})" default "None" depends on IMAGE - ''') + """ # add llvm toolchain dir - common_str += (f''' + common_str += f""" config COMMON_LLVM-TOOLCHAIN-DIR string "llvm_toolchain_dir ({llvm_toolchain_help})" default "None" depends on IMAGE - ''') + """ # add nativesdk dir - common_str += (f''' + common_str += f""" config COMMON_NATIVESDK-DIR string "nativesdk_dir ({nativesdk_help})" default "None" depends on IMAGE && BUILD_IN-HOST - ''') + """ # add timestamp - common_str += (''' + common_str += """ config COMMON_DATETIME string "datetime" default "None" depends on IMAGE - ''') + """ # add cache_src_dir directory - common_str += (''' + common_str += """ config COMMON_CACHE_SRC_DIR string "cache_src_dir (src directory)" default "None" depends on IMAGE - ''') + """ # add build directory - common_str += (''' + common_str += """ config COMMON_DIRECTORY string "directory (build directory name)" default "None" - ''') + """ return common_str diff --git a/src/oebuild/app/plugins/generate/parses.py b/src/oebuild/app/plugins/generate/parses.py index 59f80aa..47444cf 100644 --- a/src/oebuild/app/plugins/generate/parses.py +++ b/src/oebuild/app/plugins/generate/parses.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,188 +8,231 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import pathlib import sys -from ruamel.yaml.error import YAMLError, MarkedYAMLError +from ruamel.yaml.error import MarkedYAMLError, YAMLError + import oebuild.const as oebuild_const -from oebuild.m_log import logger import oebuild.util as oebuild_util +from oebuild.m_log import logger -def parsers(parser): - ''' +def parsers(parser, include_features=True): + """ for generate param parser - ''' - parser.add_argument('-l', - '--list', - dest='list', - action="store_true", - help=''' + """ + parser.add_argument( + '-l', + '--list', + dest='list', + action='store_true', + help=""" list supported archs and features - ''') + """, + ) - parser.add_argument('-p', - '--platform', - dest='platform', - default="qemu-aarch64", - help=''' + parser.add_argument( + '-p', + '--platform', + dest='platform', + default='qemu-aarch64', + help=""" this param is for arch, you can find it in yocto-meta-openeuler/.oebuild/platform - ''') + """, + ) - parser.add_argument('-s', - '--state_mirrors', - dest='sstate_mirrors', - help=''' + parser.add_argument( + '-s', + '--state_mirrors', + dest='sstate_mirrors', + help=""" this param is for SSTATE_MIRRORS - ''') + """, + ) - parser.add_argument('-s_dir', - '--sstate_dir', - dest='sstate_dir', - help=''' + parser.add_argument( + '-s_dir', + '--sstate_dir', + dest='sstate_dir', + help=""" this param is for SSTATE_DIR - ''') + """, + ) - parser.add_argument('-m', - '--tmp_dir', - dest='tmp_dir', - help=''' + parser.add_argument( + '-m', + '--tmp_dir', + dest='tmp_dir', + help=""" this param is for tmp directory, the build result will be stored in - ''') + """, + ) - parser.add_argument('-f', - '--features', - dest='features', - action='append', - help=''' + if include_features: + parser.add_argument( + '-f', + '--features', + dest='features', + action='append', + help=""" this param is feature, it's a reuse command - ''') + """, + ) - parser.add_argument('-d', - '--directory', - dest='directory', - help=''' + parser.add_argument( + '-d', + '--directory', + dest='directory', + help=""" this param is build directory, the default is same to platform - ''') + """, + ) - parser.add_argument('-y', - '--yes', - dest='yes', - action="store_true", - help=''' + parser.add_argument( + '-y', + '--yes', + dest='yes', + action='store_true', + help=""" this param is default action, if use, the default action is yes in next steps - ''') + """, + ) - parser.add_argument('-t', - '--toolchain_dir', - dest='toolchain_dir', - default='', - help=''' + parser.add_argument( + '-t', + '--toolchain_dir', + dest='toolchain_dir', + default='', + help=""" this param is for external gcc toolchain dir, if you want use your own toolchain - ''') + """, + ) - parser.add_argument('-lt', - '--llvm_toolchain_dir', - dest='llvm_toolchain_dir', - default='', - help=''' + parser.add_argument( + '-lt', + '--llvm_toolchain_dir', + dest='llvm_toolchain_dir', + default='', + help=""" this param is for external llvm toolchain dir, if you want use your own toolchain - ''') + """, + ) - parser.add_argument('-n', - '--nativesdk_dir', - dest='nativesdk_dir', - default='', - help=''' + parser.add_argument( + '-n', + '--nativesdk_dir', + dest='nativesdk_dir', + default='', + help=""" this param is for external nativesdk dir, the param will be useful when you want to build in host - ''') + """, + ) - parser.add_argument('-dt', - '--datetime', - dest="datetime", - help=''' + parser.add_argument( + '-dt', + '--datetime', + dest='datetime', + help=""" this param is add DATETIME to local.conf, the value format is 20231212010101 - ''') + """, + ) - parser.add_argument('-ny', - '--no_layer', - dest="no_layer", - action="store_true", - help=''' + parser.add_argument( + '-ny', + '--no_layer', + dest='no_layer', + action='store_true', + help=""" this param will not fetch layer repo when startting bitbake environment - ''') + """, + ) - parser.add_argument('-nf', - '--no_fetch', - dest="no_fetch", - action="store_true", - help=''' + parser.add_argument( + '-nf', + '--no_fetch', + dest='no_fetch', + action='store_true', + help=""" this param is set openeuler_fetch in local.conf, the default value is enable, if set -nf, the OPENEULER_FETCH will set to 'disable' - ''') - - parser.add_argument('-b_in', - '--build_in', - dest='build_in', - choices=[ - oebuild_const.BUILD_IN_DOCKER, - oebuild_const.BUILD_IN_HOST - ], - default=oebuild_const.BUILD_IN_DOCKER, - help=''' + """, + ) + + parser.add_argument( + '-b_in', + '--build_in', + dest='build_in', + choices=[oebuild_const.BUILD_IN_DOCKER, oebuild_const.BUILD_IN_HOST], + default=oebuild_const.BUILD_IN_DOCKER, + help=""" This parameter marks the mode at build time, and is built in the container by docker - ''') + """, + ) - parser.add_argument('--nativesdk', - dest='nativesdk', - action="store_true", - help=''' + parser.add_argument( + '--nativesdk', + dest='nativesdk', + action='store_true', + help=""" This parameter is used to indicate whether to build an SDK - ''') + """, + ) - parser.add_argument('--cache_src_dir', - dest='cache_src_dir', - help=''' + parser.add_argument( + '--cache_src_dir', + dest='cache_src_dir', + help=""" if you want to get src repo from a directory otherwise remote, you can set this parameter - ''') + """, + ) - parser.add_argument('--gcc', - dest='gcc', - action="store_true", - help=''' + parser.add_argument( + '--gcc', + dest='gcc', + action='store_true', + help=""" This parameter is used to indicate whether to build an toolchain - ''') + """, + ) - parser.add_argument('--gcc_name', - dest='gcc_name', - action='append', - help=''' + parser.add_argument( + '--gcc_name', + dest='gcc_name', + action='append', + help=""" This parameter is used to gcc toolchain config name - ''') - parser.add_argument('--llvm', - dest='llvm', - action="store_true", - help=''' + """, + ) + parser.add_argument( + '--llvm', + dest='llvm', + action='store_true', + help=""" This parameter is used to indicate whether to build an toolchain - ''') + """, + ) - parser.add_argument('--llvm_lib', - dest='llvm_lib', - help=''' + parser.add_argument( + '--llvm_lib', + dest='llvm_lib', + help=""" This parameter is used to indicate whether to build an toolchain - ''') + """, + ) - parser.add_argument('--auto_build', - dest='auto_build', - action="store_true", - help=''' + parser.add_argument( + '--auto_build', + dest='auto_build', + action='store_true', + help=""" This parameter is used for nativesdk and toolchain build - ''') + """, + ) return parser @@ -202,7 +245,7 @@ def parse_feature_files(oebuild_dir): feature_dir = pathlib.Path(oebuild_dir, 'features') if not feature_dir.exists(): - logger.error("Features directory not found under .oebuild.") + logger.error('Features directory not found under .oebuild.') sys.exit(-1) # Group files by base name to handle priority file_groups = {} @@ -226,7 +269,9 @@ def parse_feature_files(oebuild_dir): try: feature_data = oebuild_util.read_yaml(selected_file) if not isinstance(feature_data, dict): - logger.warning("Invalid YAML mapping in feature '%s'", feature_name) + logger.warning( + "Invalid YAML mapping in feature '%s'", feature_name + ) continue features.append((feature_name, selected_file, feature_data)) except MarkedYAMLError as e: @@ -235,11 +280,18 @@ def parse_feature_files(oebuild_dir): """Failed to parse feature file '%s': Error: %s At: Line %d, Column %d, - """, fname, e.problem, e.problem_mark.line + 1, e.problem_mark.column + 1) + """, + fname, + e.problem, + e.problem_mark.line + 1, + e.problem_mark.column + 1, + ) continue except YAMLError as e: fname = selected_file.name - logger.warning("Failed to parse feature file '%s': %s", fname, e) + logger.warning( + "Failed to parse feature file '%s': %s", fname, e + ) continue features.sort(key=lambda x: x[0]) return features diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py index 0d408bb..1008bc3 100644 --- a/src/oebuild/app/plugins/init/init.py +++ b/src/oebuild/app/plugins/init/init.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import shutil @@ -25,14 +25,14 @@ import oebuild.const as oebuild_const class Init(OebuildCommand): - ''' + """ Directory initialization directive, mainly used to initialize the OEbuild project directory, running this directive needs to be followed by the directory name to be initialized - ''' + """ help_msg = 'Initialize an OEBUILD working directory' - description = textwrap.dedent('''\ + description = textwrap.dedent("""\ Initialize the OEbuild working directory, and after executing this command, a new directory will be created as the OEBUILD working directory based on the current path. After initialization, the working directory will create an .oebuild @@ -44,7 +44,7 @@ class Init(OebuildCommand): certain changes according to their own needs。 This file is to meet the user's global consideration of the build configuration of OEbuild, and can be easily called by third parties - ''') + """) def __init__(self): self.configure = Configure() @@ -56,32 +56,36 @@ class Init(OebuildCommand): def do_add_parser(self, parser_adder): self._parser( parser_adder, - usage='''%(prog)s [directory] [-u yocto_remote_url] [-b branch]''') + usage="""%(prog)s [directory] [-u yocto_remote_url] [-b branch]""", + ) parser_adder.add_argument( '-u', '--yocto_remote_url', dest='yocto_remote_url', - help='''Specifies the remote of yocto-meta-openeuler''') + help="""Specifies the remote of yocto-meta-openeuler""", + ) parser_adder.add_argument( '-b', '--branch', dest='branch', - help='''Specifies the branch of yocto-meta-openeuler''') + help="""Specifies the branch of yocto-meta-openeuler""", + ) parser_adder.add_argument( 'directory', nargs='?', default=None, - help='''The name of the directory that will be initialized''') + help="""The name of the directory that will be initialized""", + ) return parser_adder def do_run(self, args: argparse.ArgumentParser, unknown=None): - ''' + """ detach target dicrectory if finished init, if inited, just put out err msg and exit - ''' + """ # perpare parse help command if self.pre_parse_help(args, unknown): @@ -98,43 +102,47 @@ class Init(OebuildCommand): if args.directory is None: logger.error("'oebuild init' need param directory") - logger.info("\noebuild init help:") + logger.info('\noebuild init help:') self.print_help_msg() sys.exit(1) # check if oebuild workspace exist, if exist, given notice if Configure().is_oebuild_dir(pathlib.Path(args.directory)): - logger.info("the %s has already been initialized", - str(pathlib.Path(args.directory).absolute())) + logger.info( + 'the %s has already been initialized', + str(pathlib.Path(args.directory).absolute()), + ) self._print_notice(args.directory) return if not self.init_workspace(args.directory): - logger.error("mkdir %s failed", args.directory) + logger.error('mkdir %s failed', args.directory) sys.exit(-1) curr_dir = os.getcwd() os.chdir(args.directory) oebuild_config: Config = self.configure.parse_oebuild_config() - yocto_config: ConfigBasicRepo = \ - oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] + yocto_config: ConfigBasicRepo = oebuild_config.basic_repo[ + oebuild_const.YOCTO_META_OPENEULER + ] if args.yocto_remote_url is not None: yocto_config.remote_url = args.yocto_remote_url if args.branch is not None: yocto_config.branch = args.branch - oebuild_config.basic_repo[ - oebuild_const.YOCTO_META_OPENEULER] = yocto_config + oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] = ( + yocto_config + ) self.configure.update_oebuild_config(oebuild_config) - logger.info("init %s successful", args.directory) + logger.info('init %s successful', args.directory) os.chdir(curr_dir) self._print_notice(args.directory) def _print_notice(self, directory): - format_msg = f''' + format_msg = f""" There is a build configuration example file under {directory}/.oebuild/compile.yaml.sample, if you want to block complex generate instructions, you can directly copy a configuration file, and then modify it according to your own needs, and then execute @@ -143,13 +151,13 @@ please execute the follow commands next cd {pathlib.Path(directory).absolute()} oebuild update - ''' + """ print(format_msg) def init_workspace(self, directory): - ''' + """ init workspace will copy config file and make new src directory - ''' + """ try: os.mkdir(pathlib.Path(directory).absolute()) except FileExistsError: @@ -163,49 +171,50 @@ please execute the follow commands next @staticmethod def create_oebuild_directory(updir: str): - ''' + """ create oebuild config directory - ''' + """ try: - oebuild_dir = os.path.join(updir, ".oebuild") + oebuild_dir = os.path.join(updir, '.oebuild') os.mkdir(oebuild_dir) return oebuild_dir except FileExistsError: - logger.error("mkdir .oebuild failed") - return "" + logger.error('mkdir .oebuild failed') + return '' @staticmethod def create_src_directory(updir: str): - ''' + """ this is desctiption - ''' + """ try: - src_dir = os.path.join(updir, "src") + src_dir = os.path.join(updir, 'src') os.makedirs(src_dir) return src_dir except FileExistsError: - logger.error("mkdir src failed") + logger.error('mkdir src failed') return None @staticmethod def copy_config_file(updir: str): - ''' + """ copy oebuild config to some directory - ''' + """ try: config = oebuild_util.get_config_yaml_dir() shutil.copyfile(config, os.path.join(updir, oebuild_const.CONFIG)) except FileNotFoundError: - logger.error("mkdir config faild") + logger.error('mkdir config faild') @staticmethod def copy_compile_file(updir: str): - ''' + """ copy oebuild compile.yaml.sample to some directory - ''' + """ try: compil = oebuild_util.get_compile_yaml_dir() - shutil.copyfile(compil, - os.path.join(updir, oebuild_const.COMPILE_YAML)) + shutil.copyfile( + compil, os.path.join(updir, oebuild_const.COMPILE_YAML) + ) except FileNotFoundError: - logger.error("mkdir compile.yaml.sample failed") + logger.error('mkdir compile.yaml.sample failed') diff --git a/src/oebuild/app/plugins/m_env/m_env.py b/src/oebuild/app/plugins/m_env/m_env.py index 996286b..538bf22 100644 --- a/src/oebuild/app/plugins/m_env/m_env.py +++ b/src/oebuild/app/plugins/m_env/m_env.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import fcntl @@ -30,60 +30,70 @@ from oebuild.m_log import logger class Menv(OebuildCommand): - ''' + """ the class is used to manager sdk environment, the sdk environment refers to openEuler Embedded image sdk, you can use the sdk to develop some apps that can run in openEuler Embedded system you built. for example, the sdk with qt image can be used to develop apps runned in qt system, the sdk with ros image can be used to develop apps runned in ros system - ''' + """ help_msg = 'Update the basic environment required for the build' - description = textwrap.dedent(''' + description = textwrap.dedent(""" This is an environment management function that allows you to configure the environment through SDK files or unzipped setup files, and you can view, delete, and activate the corresponding environment. These operations will not have any impact on your current machine - ''') + """) def __init__(self): self.configure = Configure() - self.oebuild_env_path = os.path.expanduser( - '~') + '/.local/oebuild_env/' + self.oebuild_env_path = ( + os.path.expanduser('~') + '/.local/oebuild_env/' + ) self.oebuild_env_yaml_path = pathlib.Path( - os.path.join(self.oebuild_env_path, 'oebuild_env.yaml')) + os.path.join(self.oebuild_env_path, 'oebuild_env.yaml') + ) self.oebuild_env_command = ['list', 'create', 'activate', 'remove'] super().__init__('menv', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [create list remove activate][command] create: [-d -f] Create an environment -n env_name list: View existing environment remove: -n Delete specified environment activate: -n Activate specified environment -''') - - parser.add_argument('-d', - '--directory', - dest='directory', - help=''' +""", + ) + + parser.add_argument( + '-d', + '--directory', + dest='directory', + help=""" this param is build directory - ''') - - parser.add_argument('-f', - '--file', - dest='file', - help=''' + """, + ) + + parser.add_argument( + '-f', + '--file', + dest='file', + help=""" this param is build file - ''') - - parser.add_argument('-n', - '--env_name', - dest='env_name', - help=''' + """, + ) + + parser.add_argument( + '-n', + '--env_name', + dest='env_name', + help=""" this param is env_name - ''') + """, + ) # Secondary command return parser @@ -91,9 +101,11 @@ class Menv(OebuildCommand): def do_run(self, args: argparse.Namespace, unknown=None): # perpare parse help command if unknown[0] not in self.oebuild_env_command or ( - len(set(unknown[1:]).intersection({'-d', '-f', '-n'})) == 0 - and unknown[0] != 'list'): + len(set(unknown[1:]).intersection({'-d', '-f', '-n'})) == 0 + and unknown[0] != 'list' + ): unknown = ['-h'] + command = None else: command = unknown[0] unknown = unknown[1:] @@ -104,9 +116,9 @@ class Menv(OebuildCommand): if command == 'create': if not args.env_name: - print(''' + print(""" Please enter the correct command: oebuild menv create [-d -f] Create an environment -n env_name - ''') + """) sys.exit(1) self.create_environment(args=args) elif command == 'activate': @@ -137,16 +149,22 @@ Please enter the correct command: oebuild menv create [-d -f] Create an environm 'Please enter the correct command: oebuild menv remove -n env_name' ) sys.exit(1) + else: + # This handles the case where command is None (help was shown) + sys.exit(0) def create_environment(self, args): - ''' + """ create environment file in ~/.local/oebuild_env/ and do something in next step - ''' + """ # Check if the file path exists if args.directory and os.path.isdir(args.directory): setup_file_path = os.path.abspath(args.directory) - sdk_name = args.env_name if args.env_name else args.directory.split( - '/')[-1] + sdk_name = ( + args.env_name + if args.env_name + else args.directory.split('/')[-1] + ) self.create_or_update_env_yaml(sdk_name, args.directory) print( f' Created Environment successfully \n {sdk_name.ljust(30)}{setup_file_path}' @@ -155,8 +173,15 @@ Please enter the correct command: oebuild menv create [-d -f] Create an environm # Creating an environment if args.file and os.path.exists(args.file): - sdk_name = args.env_name if args.env_name else (args.file.split( - '/')[-1].replace('.sh', '') if args.file else None) + sdk_name = ( + args.env_name + if args.env_name + else ( + args.file.split('/')[-1].replace('.sh', '') + if args.file + else None + ) + ) setup_file_path = self.oebuild_env_path + sdk_name self.create_or_update_env_yaml(sdk_name, setup_file_path) self.execute_sdk_file(args.file, setup_file_path) @@ -187,43 +212,63 @@ Please enter the correct command: oebuild menv create [-d -f] Create an environm try: file_path = os.path.join( setup_file_path, - re.search('environment-setup.*?(?=\')', file_list).group()) + re.search("environment-setup.*?(?=')", file_list).group(), + ) absolute_file_path = os.path.abspath(file_path) # Splice Execution Command shell_command = '. ' + absolute_file_path print(shell_command) print('setup_file matching successful') - subprocess.check_output('cp ~/.bashrc ~/.bashrc_back', - shell=True) + subprocess.check_output( + 'cp ~/.bashrc ~/.bashrc_back', shell=True + ) # Obtain the current terminal height and length - terminal_info = fcntl.ioctl(sys.stdout.fileno(), - termios.TIOCGWINSZ, "1234") + terminal_info = fcntl.ioctl( + sys.stdout.fileno(), termios.TIOCGWINSZ, '1234' + ) rows_and_cloumns = struct.unpack('HH', terminal_info) rows_command = f'stty rows {rows_and_cloumns[0]} columns {rows_and_cloumns[1]}' subprocess.check_output( - rf"sed -i '$a\{rows_command}' ~/.bashrc", shell=True) + rf"sed -i '$a\{rows_command}' ~/.bashrc", shell=True + ) # Add the command to execute the setup file in the .bashrc file in the # working directory subprocess.check_output( - rf"sed -i '$a\{shell_command}' ~/.bashrc", shell=True) + rf"sed -i '$a\{shell_command}' ~/.bashrc", shell=True + ) # Replace Console Prompt subprocess.check_output( rf"sed -i 's/\$ /({env_name})>>>>> /g' ~/.bashrc", - shell=True) + shell=True, + ) subprocess.check_output( r"sed -i '$a\mv ~/.bashrc_back ~/.bashrc -f' ~/.bashrc", - shell=True) + shell=True, + ) # Add prompt words - separator = "====================================================" - prompt_one = "Your environment is ready" - prompt_two = "Please proceed with the subsequent operations here" + separator = ( + '====================================================' + ) + prompt_one = 'Your environment is ready' + prompt_two = ( + 'Please proceed with the subsequent operations here' + ) wrap = '\\n###!###\\n' - prompt_words = separator + wrap + prompt_one + wrap + prompt_two + wrap + separator + prompt_words = ( + separator + + wrap + + prompt_one + + wrap + + prompt_two + + wrap + + separator + ) subprocess.check_output( rf'''sed -i '$a\echo "{prompt_words}"' ~/.bashrc''', - shell=True) - pty.spawn("/bin/bash") + shell=True, + ) + pty.spawn('/bin/bash') except SubprocessError as s_e: print('Please provide the valid folder path') logger.error(str(s_e)) @@ -254,9 +299,11 @@ Please enter the correct command: oebuild menv create [-d -f] Create an environm else: print('Extracting sdk...............') subprocess.check_output( - f'sh {sdk_file} -d {setup_file_path} -y', shell=True) - subprocess.check_output(f'chmod -R 755 {setup_file_path}', - shell=True) + f'sh {sdk_file} -d {setup_file_path} -y', shell=True + ) + subprocess.check_output( + f'chmod -R 755 {setup_file_path}', shell=True + ) except SubprocessError as s_e: print('Please provide the valid folder path') logger.error(str(s_e)) @@ -274,23 +321,25 @@ Please enter the correct command: oebuild menv create [-d -f] Create an environm """ if not os.path.exists(self.oebuild_env_yaml_path.absolute()): if not os.path.exists( - os.path.dirname(self.oebuild_env_yaml_path.absolute())): + os.path.dirname(self.oebuild_env_yaml_path.absolute()) + ): os.makedirs( - os.path.dirname(self.oebuild_env_yaml_path.absolute())) + os.path.dirname(self.oebuild_env_yaml_path.absolute()) + ) os.mknod(self.oebuild_env_yaml_path) env_dict = oebuild_util.read_yaml(self.oebuild_env_yaml_path) if env_dict and 'env_config' in env_dict: - env_list = self.input_or_update_dict(env_name, setup_file_path, - env_dict['env_config']) + env_list = self.input_or_update_dict( + env_name, setup_file_path, env_dict['env_config'] + ) env_dict['env_config'] = env_list oebuild_util.write_yaml(self.oebuild_env_yaml_path, env_dict) return env_dict = { - 'env_config': [{ - 'env_name': env_name, - 'env_value': setup_file_path - }] + 'env_config': [ + {'env_name': env_name, 'env_value': setup_file_path} + ] } oebuild_util.write_yaml(self.oebuild_env_yaml_path, env_dict) @@ -326,10 +375,10 @@ Do you want to overwrite the path of the original environment configuration(Y/N) return env_list def activate_environment(self, env_name): - ''' + """ activate the sdk environment, is means that environment shell will be sourced, and open a new pty, so that developer can compile app with sdk environment - ''' + """ env_dict = oebuild_util.read_yaml(self.oebuild_env_yaml_path) if env_dict and 'env_config' in env_dict: setup_file_path = self._get_environment(env_name, env_dict) @@ -371,13 +420,13 @@ Do you want to overwrite the path of the original environment configuration(Y/N) if env_dict and 'env_config' in env_dict: env_list = [] for env_data in env_dict['env_config']: - if env_data['env_name'] != env_name: env_list.append(env_data) elif '/.local/oebuild_env/' in env_data['env_value']: try: subprocess.check_output( - f'rm -rf {env_data["env_value"]}', shell=True) + f'rm -rf {env_data["env_value"]}', shell=True + ) except SubprocessError as s_e: print('Fail deleted') logger.error(str(s_e)) @@ -385,7 +434,8 @@ Do you want to overwrite the path of the original environment configuration(Y/N) if len(env_list) == len(env_dict['env_config']): logger.error( - 'The environment does not exist, please check the input') + 'The environment does not exist, please check the input' + ) sys.exit(-1) env_dict['env_config'] = env_list oebuild_util.write_yaml(self.oebuild_env_yaml_path, env_dict) diff --git a/src/oebuild/app/plugins/m_plugin/m_plugin.py b/src/oebuild/app/plugins/m_plugin/m_plugin.py index 1b1e71f..26e97a9 100644 --- a/src/oebuild/app/plugins/m_plugin/m_plugin.py +++ b/src/oebuild/app/plugins/m_plugin/m_plugin.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import copy @@ -34,26 +34,34 @@ class MPlugin(OebuildCommand): """ help_msg = 'manage the oebuild plugin' - description = textwrap.dedent(''' + description = textwrap.dedent(""" This is a plugin management function that supports users to customize plugin and add them to the oebuild for use. plugin only affect locally installed oebuilds, and supports viewing personal existing plugin and uninstalling plugin. - ''') + """) def __init__(self): self.configure = Configure() self.oebuild_plugin_commands = [ - 'install', 'list', 'enable', 'disable', 'remove' + 'install', + 'list', + 'enable', + 'disable', + 'remove', ] - self.oebuild_plugin_path = os.path.expanduser( - '~') + '/.local/oebuild_plugins/' - self.oebuild_plugin_yaml_path = pathlib.Path(self.oebuild_plugin_path, - 'append_plugins.yaml') - self.oebuild_plugin_repository = pathlib.Path(self.oebuild_plugin_path, - 'appends') + self.oebuild_plugin_path = ( + os.path.expanduser('~') + '/.local/oebuild_plugins/' + ) + self.oebuild_plugin_yaml_path = pathlib.Path( + self.oebuild_plugin_path, 'append_plugins.yaml' + ) + self.oebuild_plugin_repository = pathlib.Path( + self.oebuild_plugin_path, 'appends' + ) plugin_dir = pathlib.Path(oebuild_util.get_plugins_yaml_path()) self.command_ext = OebuildApp().get_command_ext( - oebuild_util.read_yaml(plugin_dir)['plugins']) + oebuild_util.read_yaml(plugin_dir)['plugins'] + ) super().__init__('mplugin', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: @@ -68,8 +76,9 @@ class MPlugin(OebuildCommand): Returns: """ - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [install list remove enable/disable][command] install: -f file_path -n plugin_name @@ -77,34 +86,43 @@ class MPlugin(OebuildCommand): list: enable/disable: enable/disable -n plugin_name remove: -n plugin_name -''') - parser.add_argument('-f', - '--file', - dest='file', - help=''' +""", + ) + parser.add_argument( + '-f', + '--file', + dest='file', + help=""" this param is python file - ''') + """, + ) - parser.add_argument('-n', - '--plugin_name', - dest='plugin_name', - help=''' + parser.add_argument( + '-n', + '--plugin_name', + dest='plugin_name', + help=""" this param is plugin name - ''') + """, + ) - parser.add_argument('-d', - '--dir_path', - dest='dir_path', - help=''' + parser.add_argument( + '-d', + '--dir_path', + dest='dir_path', + help=""" this param is dir path - ''') + """, + ) - parser.add_argument('-m', - '--major', - dest='major', - help=''' + parser.add_argument( + '-m', + '--major', + dest='major', + help=""" this param is major class - ''') + """, + ) return parser @@ -125,8 +143,9 @@ class MPlugin(OebuildCommand): if not unknown: unknown = ['-h'] elif unknown[0] not in self.oebuild_plugin_commands or ( - len(set(unknown[1:]).intersection({'-f', '-n', '-d'})) == 0 - and unknown[0] != 'list'): + len(set(unknown[1:]).intersection({'-f', '-n', '-d'})) == 0 + and unknown[0] != 'list' + ): unknown = ['-h'] else: command = unknown[0] @@ -138,9 +157,11 @@ class MPlugin(OebuildCommand): if not os.path.exists(self.oebuild_plugin_yaml_path.absolute()): if not os.path.exists( - os.path.dirname(self.oebuild_plugin_yaml_path.absolute())): + os.path.dirname(self.oebuild_plugin_yaml_path.absolute()) + ): os.makedirs( - os.path.dirname(self.oebuild_plugin_yaml_path.absolute())) + os.path.dirname(self.oebuild_plugin_yaml_path.absolute()) + ) os.mknod(self.oebuild_plugin_yaml_path) plugin_dict = oebuild_util.read_yaml(self.oebuild_plugin_yaml_path) plugin_dict_old = copy.deepcopy(plugin_dict) @@ -152,59 +173,70 @@ class MPlugin(OebuildCommand): { 'file': args.file, 'plugin_name': args.plugin_name, - 'command': command - }, plugin_dict, plugin_dict_old) + 'command': command, + }, + plugin_dict, + plugin_dict_old, + ) sys.exit(0) elif args.dir_path and os.path.exists(args.dir_path): file = str( - pathlib.Path(args.dir_path, - args.major.split('/')[-1])) + pathlib.Path(args.dir_path, args.major.split('/')[-1]) + ) self._install_for_dir_check(args=args, file=file) self.install_plugin( { 'file': file, 'plugin_name': args.plugin_name, 'command': command, - 'dir_path': args.dir_path - }, plugin_dict, plugin_dict_old) + 'dir_path': args.dir_path, + }, + plugin_dict, + plugin_dict_old, + ) sys.exit(0) - logger.error("the %s not exist, please check the plugin file path", - args.file) + logger.error( + 'the %s not exist, please check the plugin file path', + args.file, + ) sys.exit(1) elif command == 'list': self.list_plugin(plugin_dict=plugin_dict) elif command in ['enable', 'disable']: - self.enable_disable_plugin(command=command, - plugin_dict=plugin_dict, - args=args) + self.enable_disable_plugin( + command=command, plugin_dict=plugin_dict, args=args + ) elif command == 'remove': self.remove_plugin(args.plugin_name) def list_plugin(self, plugin_dict): - ''' + """ list plugin infomation with format like 'plugin_name status plugin_path' - ''' + """ if plugin_dict and 'plugins' in plugin_dict: print("""# oebuild plugin:\n#""") print(f'{"name".ljust(20)}{"status".ljust(20)}{"path"}') for plugin_data in plugin_dict['plugins']: - print(f'{str(plugin_data["name"]).ljust(20)}' - f'{str(plugin_data["status"]).ljust(20)}' - f'{str(plugin_data["path"])}') + print( + f'{str(plugin_data["name"]).ljust(20)}' + f'{str(plugin_data["status"]).ljust(20)}' + f'{str(plugin_data["path"])}' + ) else: logger.error('No plugin has been created yet') sys.exit(-1) def enable_disable_plugin(self, command, plugin_dict, args): - ''' + """ enable plugin or disable plugin - ''' + """ if plugin_dict and 'plugins' in plugin_dict: for plugin_data in plugin_dict['plugins']: if plugin_data['name'] == args.plugin_name: plugin_data['status'] = command - oebuild_util.write_yaml(self.oebuild_plugin_yaml_path, - plugin_dict) + oebuild_util.write_yaml( + self.oebuild_plugin_yaml_path, plugin_dict + ) print('change success') logger.error('the plugin %s does not exist', args.plugin_name) sys.exit(-1) @@ -214,17 +246,19 @@ class MPlugin(OebuildCommand): def _install_for_dir_check(self, args, file): if not args.major: - logger.error(" Please specify the major file ") + logger.error(' Please specify the major file ') sys.exit(-1) if not os.path.exists(file): - logger.error("the %s not exist, please check the plugin file path", - file) + logger.error( + 'the %s not exist, please check the plugin file path', file + ) sys.exit(-1) def _install_param_check(self, args, plugin_dict): if not args.plugin_name: logger.error( - 'Please enter the correct command: Missing -n parameter ') + 'Please enter the correct command: Missing -n parameter ' + ) sys.exit(-1) if args.plugin_name == 'mplugin': @@ -233,23 +267,28 @@ class MPlugin(OebuildCommand): if plugin_dict is not None: append_command_ext = OebuildApp().get_command_ext( - plugin_dict['plugins']) + plugin_dict['plugins'] + ) else: append_command_ext = {} - if args.plugin_name in self.command_ext.keys() \ - or args.plugin_name in append_command_ext.keys(): + if ( + args.plugin_name in self.command_ext.keys() + or args.plugin_name in append_command_ext.keys() + ): while True: user_input = input( 'Do you want to overwrite the existing ' - f'plugin ({args.plugin_name}) in oebuild(Y/N)') + f'plugin ({args.plugin_name}) in oebuild(Y/N)' + ) if user_input.lower() == 'y': break if user_input.lower() == 'n': sys.exit(0) - def create_or_update_plugin_yaml(self, plugin_name, class_name, - python_file_name, plugin_dict): + def create_or_update_plugin_yaml( + self, plugin_name, class_name, python_file_name, plugin_dict + ): """ Args: @@ -269,19 +308,23 @@ class MPlugin(OebuildCommand): 'plugin_name': plugin_name, 'class_name': class_name, 'python_file_name': python_file_name, - 'plugin_status': "enable" - }, plugin_dict['plugins']) + 'plugin_status': 'enable', + }, + plugin_dict['plugins'], + ) plugin_dict['plugins'] = plugin_list oebuild_util.write_yaml(self.oebuild_plugin_yaml_path, plugin_dict) return old_plugin_path plugin_dict = { - 'plugins': [{ - 'name': plugin_name, - 'class': class_name, - 'path': python_file_name, - 'status': 'enable' - }] + 'plugins': [ + { + 'name': plugin_name, + 'class': class_name, + 'path': python_file_name, + 'status': 'enable', + } + ] } oebuild_util.write_yaml(self.oebuild_plugin_yaml_path, plugin_dict) return old_plugin_path @@ -308,17 +351,20 @@ class MPlugin(OebuildCommand): if plugin_data['name'] == plugin_obj['plugin_name']: plugin_data['class'] = plugin_obj['class_name'] old_plugin_path = os.path.abspath( - os.path.dirname(plugin_data['path'])) + os.path.dirname(plugin_data['path']) + ) plugin_data['path'] = plugin_obj['python_file_name'] plugin_data['status'] = plugin_obj['plugin_status'] insert_flag = False if insert_flag: - plugin_list.append({ - 'name': plugin_obj['plugin_name'], - 'class': plugin_obj['class_name'], - 'path': plugin_obj['python_file_name'], - 'status': plugin_obj['plugin_status'] - }) + plugin_list.append( + { + 'name': plugin_obj['plugin_name'], + 'class': plugin_obj['class_name'], + 'path': plugin_obj['python_file_name'], + 'status': plugin_obj['plugin_status'], + } + ) return plugin_list, old_plugin_path def query_method(self, file_path): @@ -331,21 +377,31 @@ class MPlugin(OebuildCommand): """ with open(file_path, 'r', encoding='UTF-8') as file: - def_name = "" - class_name = "" + def_name = '' + class_name = '' for file_line in file: if file_line.startswith('def') or file_line.startswith( - ' def'): + ' def' + ): if re.search(r'(?<=def)\s+\w+', file_line): - def_name += re.search(r'(?<=def)\s+\w+', - file_line).group() - def_name += "," + def_name += re.search( + r'(?<=def)\s+\w+', file_line + ).group() + def_name += ',' if file_line.startswith('class') or file_line.startswith( - ' class'): - if re.search(r'((?<=class)\s+\w+\(OebuildCommand\))', file_line) and \ - not class_name: - class_name = re.search(r'(?<=class)\s+\w+', - file_line).group().strip() + ' class' + ): + if ( + re.search( + r'((?<=class)\s+\w+\(OebuildCommand\))', file_line + ) + and not class_name + ): + class_name = ( + re.search(r'(?<=class)\s+\w+', file_line) + .group() + .strip() + ) return def_name, class_name def remove_plugin(self, plugin_name): @@ -362,20 +418,25 @@ class MPlugin(OebuildCommand): if plugin_data['name'] == plugin_name: plugin_dict['plugins'].remove(plugin_data) delete_path = os.path.abspath( - pathlib.Path(os.path.dirname(plugin_data['path']), - '..')) - subprocess.check_output(f'rm -rf {delete_path}', - shell=True) - oebuild_util.write_yaml(self.oebuild_plugin_yaml_path, - plugin_dict) + pathlib.Path( + os.path.dirname(plugin_data['path']), '..' + ) + ) + subprocess.check_output( + f'rm -rf {delete_path}', shell=True + ) + oebuild_util.write_yaml( + self.oebuild_plugin_yaml_path, plugin_dict + ) print('delete success') return logger.error('the plugin %s does not exist', plugin_name) else: logger.error('No plugin has been created yet') - def install_plugin(self, install_plugin_object, plugin_dict, - plugin_dict_old): + def install_plugin( + self, install_plugin_object, plugin_dict, plugin_dict_old + ): """ install plugin the install_plugin_object container follow item file: xxx @@ -403,53 +464,68 @@ class MPlugin(OebuildCommand): file_name_dir = file_split_info[-2] else: file_name = pathlib.Path('plugin_info', file_split_info[-1]) - file_name_dir = "" - file_path = pathlib.Path(self.oebuild_plugin_repository, - install_plugin_object['plugin_name'], - file_name) + file_name_dir = '' + file_path = pathlib.Path( + self.oebuild_plugin_repository, + install_plugin_object['plugin_name'], + file_name, + ) old_plugin_path = self.create_or_update_plugin_yaml( - install_plugin_object['plugin_name'], class_name, str(file_path), - plugin_dict) + install_plugin_object['plugin_name'], + class_name, + str(file_path), + plugin_dict, + ) if old_plugin_path != '': subprocess.check_output( f'mv {old_plugin_path} ~/.local/{old_plugin_path.split("/")[-1]}', - shell=True) + shell=True, + ) file_dir_path = pathlib.Path( self.oebuild_plugin_repository, - install_plugin_object['plugin_name']).absolute() + install_plugin_object['plugin_name'], + ).absolute() if not os.path.exists(pathlib.Path(file_dir_path, 'plugin_info')): os.makedirs(pathlib.Path(file_dir_path, 'plugin_info')) - if file_name_dir and not os.path.exists(pathlib.Path(file_dir_path, file_name_dir)): + if file_name_dir and not os.path.exists( + pathlib.Path(file_dir_path, file_name_dir) + ): os.makedirs(pathlib.Path(file_dir_path, file_name_dir)) if 'dir_path' not in install_plugin_object: subprocess.check_output( - f"cp {install_plugin_object['file']} {file_path}", shell=True) + f'cp {install_plugin_object["file"]} {file_path}', shell=True + ) else: subprocess.check_output( - f"cp -r {install_plugin_object['dir_path']} {file_dir_path}", - shell=True) + f'cp -r {install_plugin_object["dir_path"]} {file_dir_path}', + shell=True, + ) command_info = subprocess.run( - ['oebuild', f"{install_plugin_object['plugin_name']}", '-h'], + ['oebuild', f'{install_plugin_object["plugin_name"]}', '-h'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - check=False) + check=False, + ) if command_info.returncode != 0: - logger.error("\nError Message!!!!!!!!!!!!! \n\n %s ", - command_info.stderr) + logger.error( + '\nError Message!!!!!!!!!!!!! \n\n %s ', command_info.stderr + ) logger.error( 'Installation failed. There is an issue with your code. ' - 'Please check and fix it before reinstalling.') + 'Please check and fix it before reinstalling.' + ) - oebuild_util.write_yaml(self.oebuild_plugin_yaml_path, - plugin_dict_old) + oebuild_util.write_yaml( + self.oebuild_plugin_yaml_path, plugin_dict_old + ) subprocess.check_output(f'rm -rf {file_dir_path}', shell=True) if old_plugin_path != '': @@ -457,19 +533,21 @@ class MPlugin(OebuildCommand): subprocess.check_output( f'cp -r ~/.local/{old_plugin_path.split("/")[-1]} {file_dir_path}', - shell=True) + shell=True, + ) subprocess.check_output( f'rm -rf ~/.local/{old_plugin_path.split("/")[-1]}', - shell=True) + shell=True, + ) sys.exit(-1) if old_plugin_path != '': subprocess.check_output( - f'rm -rf ~/.local/{old_plugin_path.split("/")[-1]}', - shell=True) + f'rm -rf ~/.local/{old_plugin_path.split("/")[-1]}', shell=True + ) print( - f"{install_plugin_object['command'].title()} plugin successfully \n" + f'{install_plugin_object["command"].title()} plugin successfully \n' f'{"name".ljust(20)}{"status".ljust(20)}{"path"} \n' - f"{install_plugin_object['plugin_name'].ljust(20)}{'enable'.ljust(20)}{file_path}" + f'{install_plugin_object["plugin_name"].ljust(20)}{"enable".ljust(20)}{file_path}' ) diff --git a/src/oebuild/app/plugins/manifest/manifest.py b/src/oebuild/app/plugins/manifest/manifest.py index c209e3e..4967379 100644 --- a/src/oebuild/app/plugins/manifest/manifest.py +++ b/src/oebuild/app/plugins/manifest/manifest.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import textwrap @@ -27,21 +27,21 @@ from oebuild.ogit import OGit class Manifest(OebuildCommand): - ''' + """ manifest provides the manifest function of generating dependent source repositories in the build working directory, and can restore relevant source repositories based on the manifest file - ''' + """ help_msg = 'generate manifest from oebuild workspace' - description = textwrap.dedent('''\ + description = textwrap.dedent("""\ manifest provides the manifest function of generating dependent source repositories in the build working directory, and can restore relevant source repositories based on the manifest file, also you can download single repo, for zlib example: oebuild manifest download zlib - ''') + """) def __init__(self): self.configure = Configure() @@ -49,19 +49,23 @@ class Manifest(OebuildCommand): super().__init__('manifest', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [create / download] [repo] [-f MANIFEST_DIR] -''') +""", + ) - parser.add_argument('-f', - '--manifest_dir', - dest='manifest_dir', - help=''' + parser.add_argument( + '-f', + '--manifest_dir', + dest='manifest_dir', + help=""" specify a manifest path to perform the create or restore operation - ''') + """, + ) return parser @@ -70,14 +74,14 @@ class Manifest(OebuildCommand): logger.error('Your current directory had not finished init') sys.exit(-1) - subrepo = "" - command = "" + subrepo = '' + command = '' if not (unknown and unknown[0] in self.manifest_command): unknown = ['-h'] else: command = unknown[0] unknown = unknown[1:] - if len(unknown) > 0 and not unknown[0].startswith("-"): + if len(unknown) > 0 and not unknown[0].startswith('-'): subrepo = unknown[0] unknown = unknown[1:] @@ -86,8 +90,13 @@ class Manifest(OebuildCommand): sys.exit(0) args = args.parse_args(unknown) - manifest_dir = args.manifest_dir if args.manifest_dir else \ - (self.configure.source_yocto_dir() + '/.oebuild/manifest.yaml') + manifest_dir = ( + args.manifest_dir + if args.manifest_dir + else ( + self.configure.source_yocto_dir() + '/.oebuild/manifest.yaml' + ) + ) if command == 'create': self._create_manifest(manifest_dir) elif command == 'download': @@ -103,7 +112,7 @@ class Manifest(OebuildCommand): local_dir = os.path.join(self.configure.source_dir(), repo_dir) try: repo = Repo(local_dir) - remote_url = repo.remote("upstream").url + remote_url = repo.remote('upstream').url version = repo.head.commit.hexsha except git.GitError: continue @@ -112,27 +121,32 @@ class Manifest(OebuildCommand): manifest_list[repo_dir] = { 'remote_url': remote_url, - 'version': version + 'version': version, } - print("\r", end="") + print('\r', end='') progress = int((index + 1) / len(src_list) * 100) - print(f"Expose progress: {progress}%: ", - "▋" * (progress // 2), - end="") + print( + f'Expose progress: {progress}%: ', + '▋' * (progress // 2), + end='', + ) sys.stdout.flush() print() manifest_list = dict(sorted(manifest_list.items(), key=lambda s: s[0])) - oebuild_util.write_yaml(yaml_path=pathlib.Path(manifest_dir), - data={'manifest_list': manifest_list}) + oebuild_util.write_yaml( + yaml_path=pathlib.Path(manifest_dir), + data={'manifest_list': manifest_list}, + ) self._add_manifest_banner(manifest_dir=os.path.abspath(manifest_dir)) print( - f"expose successful, the directory is {os.path.abspath(manifest_dir)}" + f'expose successful, the directory is {os.path.abspath(manifest_dir)}' ) def _add_manifest_banner(self, manifest_dir): - oebuild_conf_dir = os.path.join(oebuild_util.get_base_oebuild(), - 'app/conf') + oebuild_conf_dir = os.path.join( + oebuild_util.get_base_oebuild(), 'app/conf' + ) manifest_banner_dir = os.path.join(oebuild_conf_dir, 'manifest_banner') with open(manifest_banner_dir, 'r', encoding='utf-8') as r_f: @@ -148,39 +162,44 @@ class Manifest(OebuildCommand): manifest_data = oebuild_util.read_yaml(pathlib.Path(manifest_dir)) manifest_list = manifest_data.get('manifest_list', {}) src_dir = self.configure.source_dir() - if subrepo != "": + if subrepo != '': if subrepo in manifest_list: self._download_repo(src_dir, subrepo, manifest_list[subrepo]) return - logger.error("%s not in manifest.yaml", subrepo) + logger.error('%s not in manifest.yaml', subrepo) sys.exit(-1) final_res = [] for key, value in manifest_list.items(): if not self._download_repo(src_dir, key, value): final_res.append(key) if len(final_res) > 0: - print("") - print("the list package download failed:") + print('') + print('the list package download failed:') for item in final_res: remote_url = manifest_list[item]['remote_url'] version = manifest_list[item]['version'] - print(f"{item}: {remote_url}, {version}") - print("you can manually download them!!!") + print(f'{item}: {remote_url}, {version}') + print('you can manually download them!!!') else: print(""" all package download successful!!!""") def _download_repo(self, src_dir, key, value): - logger.info("====================download %s=====================", - key) - repo_git = OGit(os.path.join(src_dir, key), - remote_url=value['remote_url'], - branch=None) + logger.info( + '====================download %s=====================', key + ) + repo_git = OGit( + os.path.join(src_dir, key), + remote_url=value['remote_url'], + branch=None, + ) if repo_git.check_out_version(version=value['version']): logger.info( - "====================download %s successful=====================", - key) + '====================download %s successful=====================', + key, + ) return True logger.warning( - "====================download %s failed=====================", key) + '====================download %s failed=====================', key + ) return False diff --git a/src/oebuild/app/plugins/mugentest/mugentest.py b/src/oebuild/app/plugins/mugentest/mugentest.py index 3743320..e5ecfad 100644 --- a/src/oebuild/app/plugins/mugentest/mugentest.py +++ b/src/oebuild/app/plugins/mugentest/mugentest.py @@ -20,12 +20,13 @@ class MugenTest(OebuildCommand): MugenTest class allows running Mugen test cases for openEuler Embedded OS. It supports remote environments. """ + name = 'mugentest' help_msg = 'This command allows you to run Mugen test cases for openEuler Embedded OS.' - description = textwrap.dedent('''\ + description = textwrap.dedent("""\ Run Mugen test cases for openEuler Embedded systems. Configure remote testing and specify the test case from a predefined list. - ''') + """) def __init__(self): """ @@ -34,7 +35,7 @@ class MugenTest(OebuildCommand): super().__init__( name=self.name, help_msg=self.help_msg, - description=self.description + description=self.description, ) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: @@ -43,7 +44,7 @@ class MugenTest(OebuildCommand): """ parser = self._parser( parser_adder, - usage=textwrap.dedent('''\ + usage=textwrap.dedent("""\ %(prog)s --mugen-path --ip --user --password [other options] Then select the test suite from the following options: @@ -51,19 +52,29 @@ class MugenTest(OebuildCommand): 2 -- OS Basic Test 3 -- Embedded Security Config Test 4 -- Embedded Application Development Test - ''') + """), ) - parser.add_argument('--mugen-path', required=True, - help='Specify the path to the Mugen installation') - parser.add_argument('--ip', required=True, - help='IP address for remote testing') - parser.add_argument('--user', required=True, - help='Username for remote login') - parser.add_argument('--password', required=True, - help='Password for remote login') - parser.add_argument('--port', required=False, default=22, - help='SSH port (default is 22)') + parser.add_argument( + '--mugen-path', + required=True, + help='Specify the path to the Mugen installation', + ) + parser.add_argument( + '--ip', required=True, help='IP address for remote testing' + ) + parser.add_argument( + '--user', required=True, help='Username for remote login' + ) + parser.add_argument( + '--password', required=True, help='Password for remote login' + ) + parser.add_argument( + '--port', + required=False, + default=22, + help='SSH port (default is 22)', + ) # 注释掉的参数 # parser.add_argument('--env', choices=['qemu', 'bsp'], required=True, # help='Specify the test environment: qemu or bsp') @@ -89,12 +100,16 @@ class MugenTest(OebuildCommand): self.setup_mugen_environment(mugen_path) if not self.is_mugen_installed(mugen_path): - print(f"Mugen not found at {mugen_path}. Please install Mugen first " - f"or specify the correct path.") + print( + f'Mugen not found at {mugen_path}. Please install Mugen first ' + f'or specify the correct path.' + ) sys.exit(1) if not args.ip or not args.user or not args.password: - logger.error("For remote testing, --ip, --user, and --password are required.") + logger.error( + 'For remote testing, --ip, --user, and --password are required.' + ) return self.select_test_suite(mugen_path, args) @@ -105,9 +120,9 @@ class MugenTest(OebuildCommand): """ try: subprocess.run(['lshw', '-version'], check=True) - print("lshw is already installed.") + print('lshw is already installed.') except subprocess.CalledProcessError: - print("lshw is not installed. Installing lshw...") + print('lshw is not installed. Installing lshw...') sys.exit(0) def setup_mugen_environment(self, mugen_path): @@ -118,10 +133,10 @@ class MugenTest(OebuildCommand): os.chdir(mugen_path) env_file = os.path.join(mugen_path, 'conf', 'env.json') if os.path.exists(env_file): - print(f"Removing existing {env_file}") + print(f'Removing existing {env_file}') os.remove(env_file) else: - print(f"No env.json found in {mugen_path}/conf.") + print(f'No env.json found in {mugen_path}/conf.') def get_mugen_path(self, custom_path=None): """ @@ -129,7 +144,7 @@ class MugenTest(OebuildCommand): """ if custom_path: return custom_path - return os.getenv('MUGEN_HOME', os.path.expanduser("~/.local/mugen")) + return os.getenv('MUGEN_HOME', os.path.expanduser('~/.local/mugen')) def is_mugen_installed(self, mugen_path): """ @@ -142,21 +157,24 @@ class MugenTest(OebuildCommand): Allows the user to select and run a test suite. """ test_suites = { - 1: "embedded_tiny_image_test", - 2: "embedded_os_basic_test", - 3: "embedded_security_config_test", - 4: "embedded_application_develop_tests" + 1: 'embedded_tiny_image_test', + 2: 'embedded_os_basic_test', + 3: 'embedded_security_config_test', + 4: 'embedded_application_develop_tests', } - print("Select a test suite to run:") + print('Select a test suite to run:') for i, suite in test_suites.items(): - print(f"{i} -- {suite.replace('_', ' ').capitalize()}") + print(f'{i} -- {suite.replace("_", " ").capitalize()}') - choice = int(input(f"Enter the number of the test suite to run " - f"(1-{len(test_suites)}): ")) + choice = int( + input( + f'Enter the number of the test suite to run (1-{len(test_suites)}): ' + ) + ) if choice not in test_suites: - print("Invalid choice. Exiting.") + print('Invalid choice. Exiting.') return selected_suite = test_suites[choice] @@ -168,67 +186,73 @@ class MugenTest(OebuildCommand): """ cmd = None try: - print(f"Running {suite} with Mugen...") - -# if args.env == "qemu": -# if suite == "embedded_tiny_image_test": -# cmd = ( -# f"bash {mugen_path}/mugen.sh -c --ip {args.ip} --password {args.password} " -# f"--user {args.user} --port {args.port} --put_all --run_remote" -# ) -# else: -# if not args.kernal_img_path or not args.initrd_path: -# logger.error( -# "For this test, --kernal_img_path and --initrd_path are required." -# ) -# return -# qemu_start_cmd = ( -# f"sh qemu_ctl.sh start --put_all --kernal_img_path {args.kernal_img_path} " -# f"--initrd_path {args.initrd_path}" -# ) -# if suite in { -# "embedded_os_basic_test", "embedded_security_config_test", -# "embedded_application_develop_tests" -# }: -# qemu_start_cmd += " --qemu_type arm" -# subprocess.run(qemu_start_cmd, shell=True, check=True) -# -# if suite == "embedded_application_develop_tests": -# compile_cmd = f"bash {mugen_path}/mugen.sh -b {suite}" -# subprocess.run(compile_cmd, shell=True, check=True) -# -# cmd = f"bash {mugen_path}/mugen.sh -f {suite} -s" -# -# elif args.env == "bsp": -# cmd = f"bash {mugen_path}/mugen.sh -f {suite} -s" -# -# if cmd: -# subprocess.run(cmd, shell=True, check=True) + print(f'Running {suite} with Mugen...') + + # if args.env == "qemu": + # if suite == "embedded_tiny_image_test": + # cmd = ( + # f"bash {mugen_path}/mugen.sh -c --ip {args.ip} --password {args.password} " + # f"--user {args.user} --port {args.port} --put_all --run_remote" + # ) + # else: + # if not args.kernal_img_path or not args.initrd_path: + # logger.error( + # "For this test, --kernal_img_path and --initrd_path are required." + # ) + # return + # qemu_start_cmd = ( + # f"sh qemu_ctl.sh start --put_all --kernal_img_path {args.kernal_img_path} " + # f"--initrd_path {args.initrd_path}" + # ) + # if suite in { + # "embedded_os_basic_test", "embedded_security_config_test", + # "embedded_application_develop_tests" + # }: + # qemu_start_cmd += " --qemu_type arm" + # subprocess.run(qemu_start_cmd, shell=True, check=True) + # + # if suite == "embedded_application_develop_tests": + # compile_cmd = f"bash {mugen_path}/mugen.sh -b {suite}" + # subprocess.run(compile_cmd, shell=True, check=True) + # + # cmd = f"bash {mugen_path}/mugen.sh -f {suite} -s" + # + # elif args.env == "bsp": + # cmd = f"bash {mugen_path}/mugen.sh -f {suite} -s" + # + # if cmd: + # subprocess.run(cmd, shell=True, check=True) os.chdir(mugen_path) # Constructing the remote environment setup command cmd = ( - f"bash mugen.sh -c --ip {args.ip} --password {args.password} " - f"--user {args.user} --port {args.port}" + f'bash mugen.sh -c --ip {args.ip} --password {args.password} ' + f'--user {args.user} --port {args.port}' ) result = subprocess.run(cmd, shell=True, check=True) if result.returncode == 0: - print("Successfully configured and connected to the remote environment.") + print( + 'Successfully configured and connected to the remote environment.' + ) else: - print(f"Failed to configure the remote environment." - f"Return code: {result.returncode}.") + print( + f'Failed to configure the remote environment.' + f'Return code: {result.returncode}.' + ) sys.exit(result.returncode) # Running the selected test suite - cmd = f"bash mugen.sh -f {suite} -s" + cmd = f'bash mugen.sh -f {suite} -s' result = subprocess.run(cmd, shell=True, check=True) if result.returncode == 0: - print(f"Test suite {suite} completed successfully.") + print(f'Test suite {suite} completed successfully.') else: - print(f"Test suite {suite} failed with return code {result.returncode}.") + print( + f'Test suite {suite} failed with return code {result.returncode}.' + ) except subprocess.CalledProcessError as e: - logger.error("Failed to run test suite %s: %s", suite, e) + logger.error('Failed to run test suite %s: %s', suite, e) sys.exit(1) diff --git a/src/oebuild/app/plugins/neo-generate/__init__.py b/src/oebuild/app/plugins/neo-generate/__init__.py new file mode 100644 index 0000000..54a8858 --- /dev/null +++ b/src/oebuild/app/plugins/neo-generate/__init__.py @@ -0,0 +1,3 @@ +""" +empty package initializer for neo-generate command +""" diff --git a/src/oebuild/app/plugins/run_qemu/run_qemu.py b/src/oebuild/app/plugins/run_qemu/run_qemu.py index fb75bce..4715de1 100644 --- a/src/oebuild/app/plugins/run_qemu/run_qemu.py +++ b/src/oebuild/app/plugins/run_qemu/run_qemu.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import textwrap @@ -28,14 +28,14 @@ from oebuild.bashrc import Bashrc class RunQemu(OebuildCommand): - ''' + """ The command for run in qemu platform. - ''' + """ help_msg = 'run in qemu platform' - description = textwrap.dedent(''' + description = textwrap.dedent(""" The command for run in qemu platform. - ''') + """) def __init__(self): self.configure = Configure() @@ -50,8 +50,9 @@ class RunQemu(OebuildCommand): if self.client is not None: try: container = self.client.get_container(self.container_id) - self.client.delete_container(container=container, - is_force=True) + self.client.delete_container( + container=container, is_force=True + ) except DockerException: print(f""" the container {self.container_id} failed to be destroyed, please run @@ -61,14 +62,18 @@ the container {self.container_id} failed to be destroyed, please run """) def do_add_parser(self, parser_adder): - parser = self._parser(parser_adder, usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [command] -''') +""", + ) parser_adder.add_argument( 'command', nargs='?', default=None, - help='''The name of the directory that will be initialized''') + help="""The name of the directory that will be initialized""", + ) return parser @@ -76,7 +81,7 @@ the container {self.container_id} failed to be destroyed, please run try: self.client = DockerProxy() except DockerException: - logger.error("Please install docker first!!!") + logger.error('Please install docker first!!!') sys.exit(-1) logger.info('Run QEMU......') @@ -84,32 +89,33 @@ the container {self.container_id} failed to be destroyed, please run self._check_qemu_ifup() self.deal_env_container(docker_image=docker_image) self.bashrc.set_container( - container=self.client.get_container(container_id=self.container_id)) - self.bashrc.set_user("root") + container=self.client.get_container(container_id=self.container_id) + ) + self.bashrc.set_user('root') for index, param in enumerate(unknown): - if param.startswith("qemuparams"): - unknown[index] = "qemuparams=\"" + param.split("=")[1] + "\"" - if param.startswith("bootparams"): - unknown[index] = "bootparams=\"" + param.split("=")[1] + "\"" + if param.startswith('qemuparams'): + unknown[index] = 'qemuparams="' + param.split('=')[1] + '"' + if param.startswith('bootparams'): + unknown[index] = 'bootparams="' + param.split('=')[1] + '"' self.exec_qemu(' '.join(unknown)) def exec_qemu(self, params): - ''' + """ exec qemu - ''' - container: Container = self.client.get_container( - self.container_id) # type: ignore + """ + container: Container = self.client.get_container(self.container_id) # type: ignore # self.bak_bash(container=container) self.init_bash(container=container) content = self.bashrc.get_bashrc_content() qemu_helper_usr = os.path.join( oebuild_const.CONTAINER_BUILD, - "tmp/work/x86_64-linux/qemu-helper-native/1.0-r1/recipe-sysroot-native/usr" + 'tmp/work/x86_64-linux/qemu-helper-native/1.0-r1/recipe-sysroot-native/usr', ) qemu_helper_dir = os.path.join( oebuild_const.CONTAINER_BUILD, - "tmp/work/x86_64-linux/qemu-helper-native") + 'tmp/work/x86_64-linux/qemu-helper-native', + ) staging_bindir_native = f""" if [ ! -d {qemu_helper_usr} ];then mkdir -p {qemu_helper_usr} @@ -117,19 +123,24 @@ if [ ! -d {qemu_helper_usr} ];then ln -s /opt/buildtools/nativesdk/sysroots/x86_64-openeulersdk-linux/usr/bin {qemu_helper_usr} fi """ - content = self.bashrc.add_bashrc(content=content, line=staging_bindir_native) + content = self.bashrc.add_bashrc( + content=content, line=staging_bindir_native + ) # content = self.bashrc.add_bashrc( # content=content, # line=f"mv -f /root/{self.old_bashrc} /root/.bashrc") - content = self.bashrc.add_bashrc(content=content, - line=f"runqemu {params} && exit") + content = self.bashrc.add_bashrc( + content=content, line=f'runqemu {params} && exit' + ) self.bashrc.update_bashrc(content=content) - os.system(f"docker exec -it -u root {container.short_id} bash") + os.system(f'docker exec -it -u root {container.short_id} bash') # self.bashrc.restore_bashrc() - def _check_qemu_ifup(self, ): - if not os.path.exists("/etc/qemu-ifup"): + def _check_qemu_ifup( + self, + ): + if not os.path.exists('/etc/qemu-ifup'): print("""please create a virtual network interface as follows: 1, open /etc/qemu-ifup in vim or vi 2, add content to qemu-ifup @@ -142,59 +153,68 @@ now, you can continue run `oebuild runqemu` in compile directory sys.exit(0) def deal_env_container(self, docker_image): - ''' + """ This operation realizes the processing of the container, controls how the container is processed by parsing the env variable, if the container does not exist, or the original environment and the current environment that needs to be set are inconsistent, you need to create a new container, otherwise directly enable the sleeping container - ''' + """ volumns = [] - volumns.append("/dev/net/tun:/dev/net/tun") - volumns.append("/etc/qemu-ifup:/etc/qemu-ifup") + volumns.append('/dev/net/tun:/dev/net/tun') + volumns.append('/etc/qemu-ifup:/etc/qemu-ifup') volumns.append(self.work_dir + ':' + oebuild_const.CONTAINER_BUILD) - volumns.append(self.configure.source_dir() + ':' + - oebuild_const.CONTAINER_SRC) + volumns.append( + self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC + ) - parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS + " --privileged" + parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS + ' --privileged' container: Container = self.client.create_container( image=docker_image, parameters=parameters, volumes=volumns, - command="bash") + command='bash', + ) self.container_id = container.short_id container: Container = self.client.get_container(self.container_id) if not self.client.is_container_running(container): self.client.start_container(container) # set sudo.conf be root - cmd = "chown -R root:root /etc/sudo.conf /etc/sudoers /etc/sudoers.d" - self.client.container_exec_command(container=container, - command=cmd, - user="root", - params={"stream": False}) + cmd = 'chown -R root:root /etc/sudo.conf /etc/sudoers /etc/sudoers.d' + self.client.container_exec_command( + container=container, + command=cmd, + user='root', + params={'stream': False}, + ) def get_docker_image(self): - ''' + """ this is function is to get openeuler docker image automatic - ''' + """ return oebuild_const.DEFAULT_DOCKER def init_bash(self, container: Container): - ''' + """ Bitbake will initialize the compilation environment by reading the user initialization script first, then making directional substitutions, and finally writing the initialization script - ''' + """ # read container default user .bashrc content content = self.bashrc.get_bashrc_content() # get nativesdk environment path automatic for next step sdk_env_path = oebuild_util.get_nativesdk_environment( - container=container) + container=container + ) init_sdk_command = f'. {oebuild_const.NATIVESDK_DIR}/{sdk_env_path}' - content = self.bashrc.add_bashrc(content=content, line=init_sdk_command) - init_oe_command = f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ + content = self.bashrc.add_bashrc( + content=content, line=init_sdk_command + ) + init_oe_command = ( + f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ {oebuild_const.CONTAINER_BUILD}' + ) content = self.bashrc.add_bashrc(content=content, line=init_oe_command) self.bashrc.update_bashrc(content=content) diff --git a/src/oebuild/app/plugins/samples/samples.py b/src/oebuild/app/plugins/samples/samples.py index 7f2f4b3..08bdc24 100644 --- a/src/oebuild/app/plugins/samples/samples.py +++ b/src/oebuild/app/plugins/samples/samples.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import sys @@ -21,28 +21,30 @@ from oebuild.m_log import logger class Samples(OebuildCommand): - ''' + """ The 'samples' command is used to efficiently manage the existing compilation templates in yocto-meta-openeuler. These templates are located in the yocto-meta-openeuler/.oebuild/samples directory, and this command allows for rapid implementation of the build process. - ''' + """ help_msg = "manage the yocto-meta-openeuler's samples compile files" - description = textwrap.dedent(''' + description = textwrap.dedent(""" The 'samples' command is used to efficiently manage the existing compilation templates in yocto-meta-openeuler. These templates are located in the yocto-meta-openeuler/.oebuild/samples directory, and this command allows for rapid implementation of the build process. - ''') + """) def __init__(self): self.configure = Configure() super().__init__('samples', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [list] -''') +""", + ) # Secondary command return parser @@ -59,7 +61,7 @@ class Samples(OebuildCommand): samples = self._get_samples() if len(unknown) > 0 and 'list' == unknown[0]: for key, value in samples.items(): - print(f"{key}: {value}") + print(f'{key}: {value}') return self.do_exec(samples=samples) @@ -75,11 +77,14 @@ class Samples(OebuildCommand): list_samples.append(file_path) if os.path.isdir(file_path): recursive_listdir(file_path) + recursive_listdir(self.configure.yocto_samples_dir()) res = {} for index, sample in enumerate(list_samples): - res[str(index + 1)] = sample.replace(self.configure.yocto_samples_dir(), "").lstrip("/") + res[str(index + 1)] = sample.replace( + self.configure.yocto_samples_dir(), '' + ).lstrip('/') return res def do_exec(self, samples: dict): @@ -87,18 +92,20 @@ class Samples(OebuildCommand): we let the user select the sample num for next build task """ for key, value in samples.items(): - print(f"{key}: {value}") - select_num = "" + print(f'{key}: {value}') + select_num = '' while True: - res = input("please select what you want build, enter the num, q for exit: ") - if res not in samples and res != "q": - logger.info("please enter the valid num or q") + res = input( + 'please select what you want build, enter the num, q for exit: ' + ) + if res not in samples and res != 'q': + logger.info('please enter the valid num or q') continue - if res == "q": + if res == 'q': sys.exit(0) select_num = res break sample = samples[select_num] sample_path = os.path.join(self.configure.yocto_samples_dir(), sample) - os.system(f"oebuild {sample_path}") + os.system(f'oebuild {sample_path}') diff --git a/src/oebuild/app/plugins/toolchain/toolchain.py b/src/oebuild/app/plugins/toolchain/toolchain.py index 53c4e44..93bae08 100644 --- a/src/oebuild/app/plugins/toolchain/toolchain.py +++ b/src/oebuild/app/plugins/toolchain/toolchain.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import textwrap import argparse @@ -30,12 +30,12 @@ from oebuild.configure import Configure class Toolchain(OebuildCommand): - ''' + """ The toolchain provides the ability to build a cross-compilation chain. - ''' + """ help_msg = 'build openEuler cross-toolchain' - description = textwrap.dedent('''\ + description = textwrap.dedent("""\ The toolchain provides similar functionality to bitbake, allowing for the construction of an openEuler cross-toolchain. @@ -51,7 +51,7 @@ class Toolchain(OebuildCommand): 2, cd toolchain build directory 3, oebuild toolchain downsource(download relative repos) 4, oebuild toolchain aarch64 - ''') + """) def __init__(self): self._toolchain_yaml_path = os.path.join(os.getcwd(), 'toolchain.yaml') @@ -68,12 +68,14 @@ class Toolchain(OebuildCommand): self.bashrc.restore_bashrc() def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [auto | setlib | upenv | downsource | ] -''') +""", + ) return parser @@ -86,9 +88,15 @@ class Toolchain(OebuildCommand): self._check_support_toolchain() self._set_init_params() - if unknown is not None and len(unknown) >= 2 and unknown[0] == "setlib": + if ( + unknown is not None + and len(unknown) >= 2 + and unknown[0] == 'setlib' + ): self.toolchain_dict['llvm_lib'] = unknown[1] - oebuild_util.write_yaml(self._toolchain_yaml_path, self.toolchain_dict) + oebuild_util.write_yaml( + self._toolchain_yaml_path, self.toolchain_dict + ) self.toolchain_obj.llvm_lib = unknown[1] return # if toolchain is llvm, the docker volume should be cover llvm_lib @@ -106,23 +114,34 @@ class Toolchain(OebuildCommand): if unknown is None or len(unknown) == 0: content = self.bashrc.get_bashrc_content() for b_s in oebuild_const.TOOLCHAIN_BASH_BANNER.split('\n'): - b_s = f"echo {b_s}" + b_s = f'echo {b_s}' content = self.bashrc.add_bashrc(content=content, line=b_s) self.bashrc.update_bashrc(content=content) self.bashrc.clean_command_bash() # 调起容器环境 - build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) - docker_exec_list = ["docker", "exec", "-it", "-u", oebuild_const.CONTAINER_USER, - "-w", build_dir, self.container_id, "bash"] - os.system(" ".join(docker_exec_list)) - elif unknown[0] == "auto": + build_dir = os.path.join( + oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()) + ) + docker_exec_list = [ + 'docker', + 'exec', + '-it', + '-u', + oebuild_const.CONTAINER_USER, + '-w', + build_dir, + self.container_id, + 'bash', + ] + os.system(' '.join(docker_exec_list)) + elif unknown[0] == 'auto': if self.toolchain_obj.kind == oebuild_const.GCC_TOOLCHAIN: self.auto_build_gcc(config_list=self.toolchain_obj.gcc_configs) else: self.auto_build_llvm() - elif unknown[0] == "upenv": + elif unknown[0] == 'upenv': self._run_upenv(kind=self.toolchain_obj.kind) - elif unknown[0] == "downsource": + elif unknown[0] == 'downsource': self._run_downcode(kind=self.toolchain_obj.kind) else: if self.toolchain_obj.kind == oebuild_const.GCC_TOOLCHAIN: @@ -131,22 +150,25 @@ class Toolchain(OebuildCommand): else: self._build_llvm() - def _set_init_params(self,): + def _set_init_params( + self, + ): self.toolchain_dict = oebuild_util.read_yaml(self._toolchain_yaml_path) self.toolchain_obj = ParseToolchainParam().parse_to_obj( - toolchain_param_dict=self.toolchain_dict) + toolchain_param_dict=self.toolchain_dict + ) self.bashrc = Bashrc() self.client = DockerProxy() def _check_gcc_config(self, config_name: str): - config_list = os.listdir("configs") - if not config_name.startswith("config_"): - config_name = f"config_{config_name}" + config_list = os.listdir('configs') + if not config_name.startswith('config_'): + config_name = f'config_{config_name}' if config_name not in config_list: - logger.error("please enter valid toolchain name") - print("the valid toolchain list:") + logger.error('please enter valid toolchain name') + print('the valid toolchain list:') for config in config_list: - if config.startswith("config_"): + if config.startswith('config_'): print(config) print(""" you can run oebuild toolchain aarch64 or oebuild toolchain config_aarch64""") @@ -155,21 +177,35 @@ you can run oebuild toolchain aarch64 or oebuild toolchain config_aarch64""") def _deal_container(self, parse_env): # check docker image openeuler-sdk if exists, if not down it - if not self.client.is_image_exists(self.toolchain_obj.docker_param.image): - print(f"the {self.toolchain_obj.docker_param.image} not exists, now pull it") - self.client.pull_image_with_progress(self.toolchain_obj.docker_param.image) + if not self.client.is_image_exists( + self.toolchain_obj.docker_param.image + ): + print( + f'the {self.toolchain_obj.docker_param.image} not exists, now pull it' + ) + self.client.pull_image_with_progress( + self.toolchain_obj.docker_param.image + ) self.container_id = oebuild_util.deal_env_container( - env=parse_env, docker_param=self.toolchain_obj.docker_param) + env=parse_env, docker_param=self.toolchain_obj.docker_param + ) self.bashrc.set_container( - container=self.client.get_container(container_id=self.container_id)) + container=self.client.get_container(container_id=self.container_id) + ) self.client.check_change_ugid( - container=self.client.get_container(container_id=self.container_id), - container_user=oebuild_const.CONTAINER_USER) + container=self.client.get_container( + container_id=self.container_id + ), + container_user=oebuild_const.CONTAINER_USER, + ) def _set_llvm_pre(self): if self.toolchain_obj.kind != oebuild_const.LLVM_TOOLCHAIN: return - if self.toolchain_obj.llvm_lib is None or self.toolchain_obj.llvm_lib == "": + if ( + self.toolchain_obj.llvm_lib is None + or self.toolchain_obj.llvm_lib == '' + ): logger.error(""" compile llvm toolchain need aarch64 lib, please run: @@ -187,23 +223,25 @@ pointed it first""") check_vol = True if not check_vol: self.toolchain_obj.docker_param.volumns.append( - self.toolchain_obj.llvm_lib + ":" + oebuild_const.CONTAINER_LLVM_LIB + self.toolchain_obj.llvm_lib + + ':' + + oebuild_const.CONTAINER_LLVM_LIB ) def auto_build_gcc(self, config_list): - ''' + """ is for auto build gcc toolchains - ''' + """ # if exists open_source, do nothing - if not os.path.exists("open_source"): + if not os.path.exists('open_source'): self._run_downcode(kind=oebuild_const.GCC_TOOLCHAIN) for config in config_list: self._build_gcc(config_name=config) def auto_build_llvm(self): - ''' + """ if for auto build llvm toolchains - ''' + """ self._run_downcode(kind=oebuild_const.LLVM_TOOLCHAIN) self._build_llvm() @@ -212,75 +250,85 @@ pointed it first""") if kind == oebuild_const.LLVM_TOOLCHAIN: return content = self.bashrc.get_bashrc_content() - build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) + build_dir = os.path.join( + oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()) + ) open_source_cmd = f'export CROSS_SOURCE="{build_dir}/open_source/."' content = self.bashrc.add_bashrc(content=content, line=open_source_cmd) - mk_x_tools = f"mkdir -p {build_dir}/x-tools" + mk_x_tools = f'mkdir -p {build_dir}/x-tools' content = self.bashrc.add_bashrc(content=content, line=mk_x_tools) - ln_tools_cmd = f'cd ~ && rm -f x-tools && ln -fs {build_dir}/x-tools x-tools' + ln_tools_cmd = ( + f'cd ~ && rm -f x-tools && ln -fs {build_dir}/x-tools x-tools' + ) content = self.bashrc.add_bashrc(content=content, line=ln_tools_cmd) - content = self.bashrc.add_bashrc(content=content, line=f"cd {build_dir}") + content = self.bashrc.add_bashrc( + content=content, line=f'cd {build_dir}' + ) self.bashrc.update_bashrc(content=content) def _run_downcode(self, kind): - build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) + build_dir = os.path.join( + oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()) + ) res: ExecResult = self.client.container_exec_command( container=self.client.get_container(self.container_id), - command="./prepare.sh ./", + command='./prepare.sh ./', user=oebuild_const.CONTAINER_USER, - params={ - "work_space": build_dir - }) + params={'work_space': build_dir}, + ) for line in res.output: logger.info(line.decode().strip('\n')) if kind == oebuild_const.GCC_TOOLCHAIN: res: ExecResult = self.client.container_exec_command( container=self.client.get_container(self.container_id), - command="./update.sh", + command='./update.sh', user=oebuild_const.CONTAINER_USER, - params={ - "work_space": build_dir - }) + params={'work_space': build_dir}, + ) for line in res.output: logger.info(line.decode().strip('\n')) def _build_gcc(self, config_name: str): - ''' + """ build gcc with config - ''' - build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) + """ + build_dir = os.path.join( + oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()) + ) container = self.client.get_container(self.container_id) self.client.container_exec_command( container=container, - command=f"cp {config_name} .config", + command=f'cp {config_name} .config', user=oebuild_const.CONTAINER_USER, - params={ - "work_space": build_dir, - "stream": False}) + params={'work_space': build_dir, 'stream': False}, + ) content = self.bashrc.get_bashrc_content() - content = self.bashrc.add_bashrc(content=content, line="ct-ng build") + content = self.bashrc.add_bashrc(content=content, line='ct-ng build') self.bashrc.update_bashrc(content=content) self.bashrc.clean_command_bash() res: ExecResult = self.client.container_exec_command( container=container, - command=f"bash /home/{oebuild_const.CONTAINER_USER}/.bashrc", + command=f'bash /home/{oebuild_const.CONTAINER_USER}/.bashrc', user=oebuild_const.CONTAINER_USER, - params={"work_space": build_dir}) + params={'work_space': build_dir}, + ) for line in res.output: logger.info(line.decode().strip('\n')) def _build_llvm(self): - ''' + """ build gcc with config - ''' - build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) + """ + build_dir = os.path.join( + oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()) + ) container = self.client.get_container(self.container_id) content = self.bashrc.get_bashrc_content() lib_gcc_dir = f'{oebuild_const.CONTAINER_LLVM_LIB}/lib64/gcc' lib_include_dir = f'{oebuild_const.CONTAINER_LLVM_LIB}/aarch64-openeuler-linux-gnu/include' lib_sysroot_dir = f'{oebuild_const.CONTAINER_LLVM_LIB}/aarch64-openeuler-linux-gnu/sysroot' - init_cmd = f''' + init_cmd = f""" cd ./open_source/llvm-project ./build.sh -e -o -s -i -b release -I clang-llvm-17.0.6 cd ./clang-llvm-17.0.6 @@ -290,48 +338,60 @@ cp -rf {lib_include_dir} aarch64-openeuler-linux-gnu/ cp -rf {lib_sysroot_dir} aarch64-openeuler-linux-gnu/ cd ./bin ln -sf ld.lld aarch64-openeuler-linux-gnu-ld -''' +""" content = self.bashrc.add_bashrc(content=content, line=init_cmd) self.bashrc.update_bashrc(content=content) self.bashrc.clean_command_bash() res: ExecResult = self.client.container_exec_command( container=container, - command=f"bash /home/{oebuild_const.CONTAINER_USER}/.bashrc", + command=f'bash /home/{oebuild_const.CONTAINER_USER}/.bashrc', user=oebuild_const.CONTAINER_USER, - params={"work_space": build_dir}) + params={'work_space': build_dir}, + ) for line in res.output: logger.info(line.decode().strip('\n')) def _check_support_toolchain(self): if not os.path.exists(self._toolchain_yaml_path): logger.error( - "Please do it in compile workspace which contain toolchain.yaml") + 'Please do it in compile workspace which contain toolchain.yaml' + ) sys.exit(-1) def _check_env_and_upenv(self): # we check env if prepared only detect the configs and patchs directory exists kind = self.toolchain_obj.kind if kind == oebuild_const.GCC_TOOLCHAIN: - if not (os.path.exists("configs") and os.path.exists("patches")): + if not (os.path.exists('configs') and os.path.exists('patches')): self._run_upenv(kind=oebuild_const.GCC_TOOLCHAIN) return - if not os.path.exists("configs"): + if not os.path.exists('configs'): self._run_upenv(kind=oebuild_const.LLVM_TOOLCHAIN) def _run_upenv(self, kind): if kind == oebuild_const.GCC_TOOLCHAIN: # cp all cross-tools files to build_dir - logger.info("cp cross-tools data to ./") - src_cross_dir = os.path.join(Configure().source_yocto_dir(), ".oebuild/cross-tools") - subprocess.run(f'cp -ru {src_cross_dir}/* ./', shell=True, check=False) + logger.info('cp cross-tools data to ./') + src_cross_dir = os.path.join( + Configure().source_yocto_dir(), '.oebuild/cross-tools' + ) + subprocess.run( + f'cp -ru {src_cross_dir}/* ./', shell=True, check=False + ) # replace MANIFEST param in configs/config.xml - replace_manifest = ('sed -i "`grep -n \'MANIFEST=\' configs/config.xml ' - '| awk -F \':\' \'{print $1}\'`c MANIFEST=/usr1/openeuler' - '/src/yocto-meta-openeuler/.oebuild/manifest.yaml"' - 'configs/config.xml') + replace_manifest = ( + "sed -i \"`grep -n 'MANIFEST=' configs/config.xml " + "| awk -F ':' '{print $1}'`c MANIFEST=/usr1/openeuler" + '/src/yocto-meta-openeuler/.oebuild/manifest.yaml"' + 'configs/config.xml' + ) subprocess.run(replace_manifest, shell=True, check=False) else: # cp all llvm-toolchain files to build_dir - logger.info("cp llvm-toolchain data to ./") - src_llvm_dir = os.path.join(Configure().source_yocto_dir(), ".oebuild/llvm-toolchain") - subprocess.run(f'cp -ru {src_llvm_dir}/* ./', shell=True, check=False) + logger.info('cp llvm-toolchain data to ./') + src_llvm_dir = os.path.join( + Configure().source_yocto_dir(), '.oebuild/llvm-toolchain' + ) + subprocess.run( + f'cp -ru {src_llvm_dir}/* ./', shell=True, check=False + ) diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index 4f1a948..3bc5de1 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import os @@ -29,13 +29,13 @@ from oebuild.m_log import logger class Update(OebuildCommand): - ''' + """ The update command will prepare the basic environment related to the build, such as container images, build base repositories, etc - ''' + """ help_msg = 'Update the basic environment required for the build' - description = textwrap.dedent(''' + description = textwrap.dedent(""" The update command will involve three areas, namely the build container, yocto-meta-openeuler and the corresponding layers, if there are no parameters after the update, it will be updated in order yocto-meta-openeuler, build @@ -52,7 +52,7 @@ class Update(OebuildCommand): the build directory will be parsed yocto-meta-openeuler/.oebuild/ common.yaml to get the information that needs to be updated, and if it is in the build directory, it will parse compile.yaml to get the updated information - ''') + """) def __init__(self): self.configure = Configure() @@ -60,29 +60,34 @@ class Update(OebuildCommand): super().__init__('update', self.help_msg, self.description) def do_add_parser(self, parser_adder): - parser = self._parser(parser_adder, - usage=''' + parser = self._parser( + parser_adder, + usage=""" %(prog)s [yocto docker layer] [-tag] -''') - parser.add_argument('-tag', - '--tag', - dest='docker_tag', - help=''' +""", + ) + parser.add_argument( + '-tag', + '--tag', + dest='docker_tag', + help=""" with platform will list support archs, with feature will list support features - ''') + """, + ) parser.add_argument( 'item', nargs='?', default=None, - help='''The name of the directory that will be initialized''') + help="""The name of the directory that will be initialized""", + ) return parser def do_run(self, args: argparse.Namespace, unknown=None): - ''' + """ update action rely on directory which has initd, so check it first - ''' + """ # perpare parse help command if self.pre_parse_help(args, unknown): sys.exit(0) @@ -96,11 +101,11 @@ class Update(OebuildCommand): update_yocto, update_docker, update_layer = False, False, False if args.item is None: update_yocto, update_docker, update_layer = True, True, True - elif args.item == "yocto": + elif args.item == 'yocto': update_yocto = True - elif args.item == "docker": + elif args.item == 'docker': update_docker = True - elif args.item == "layer": + elif args.item == 'layer': update_layer = True else: logger.error('Please run oebuild update [yocto docker layer]') @@ -113,11 +118,13 @@ class Update(OebuildCommand): try: # check if yocto exists, if not exists, give a notice if not os.path.exists(self.configure.source_yocto_dir()): - logger.error(textwrap.dedent( - "The container's update depends on yocto-meta-openeuler." - " Please either run 'oebuild update yocto' or manually " - "download yocto-meta-openeuler in the src directory." - )) + logger.error( + textwrap.dedent( + "The container's update depends on yocto-meta-openeuler." + " Please either run 'oebuild update yocto' or manually " + 'download yocto-meta-openeuler in the src directory.' + ) + ) sys.exit(-1) oebuild_util.check_docker() # check yocto/oebuild/env.yaml，get container_tag and update docker image @@ -129,28 +136,35 @@ class Update(OebuildCommand): if update_layer: self.get_layer_repo() - def get_layer_repo(self, ): - ''' + def get_layer_repo( + self, + ): + """ download or update layers that will be needed - ''' + """ # check the main layer if exists - yocto_dir = os.path.join(self.configure.source_dir(), - "yocto-meta-openeuler") + yocto_dir = os.path.join( + self.configure.source_dir(), 'yocto-meta-openeuler' + ) if not os.path.exists(yocto_dir): # update main layer self.get_basic_repo() # get rely layers from yocto-meta-openeuler/.oebuild/common.yaml when not in build directory # or /compile.yaml where in build directory repos = None - if os.path.exists(os.path.join(os.getcwd(), "compile.yaml")): - compile_param_dict = oebuild_util.read_yaml(os.path.join(os.getcwd(), "compile.yaml")) + if os.path.exists(os.path.join(os.getcwd(), 'compile.yaml')): + compile_param_dict = oebuild_util.read_yaml( + os.path.join(os.getcwd(), 'compile.yaml') + ) compile_param = ParseCompileParam().parse_to_obj( - compile_param_dict=compile_param_dict) + compile_param_dict=compile_param_dict + ) repos = compile_param.repos else: - common_path = os.path.join(yocto_dir, ".oebuild/common.yaml") + common_path = os.path.join(yocto_dir, '.oebuild/common.yaml') repos = oebuild_util.trans_dict_key_to_list( - oebuild_util.read_yaml(yaml_path=common_path)['repos']) + oebuild_util.read_yaml(yaml_path=common_path)['repos'] + ) if repos is None: sys.exit(0) @@ -158,10 +172,13 @@ class Update(OebuildCommand): oebuild_util.download_repo_from_manifest( repo_list=repos, src_dir=self.configure.source_dir(), - manifest_path=self.configure.yocto_manifest_dir()) + manifest_path=self.configure.yocto_manifest_dir(), + ) - def get_basic_repo(self, ): - ''' + def get_basic_repo( + self, + ): + """ note: get_basic_repo is to download or update basic repo in config which set in keys basic_repo, the rule is that when the embedded/src/yocto-meta-openeuler exists, so check whether its @@ -169,53 +186,65 @@ class Update(OebuildCommand): pull else mv yocto-meta-openeuler to embedded/bak/yocto-meta-openeuler and rename yocto-meta-openeuler with a random string suffix. if embedded/src/yocto-meta-openeuler not exists, so just clone from config setting. - ''' + """ oebuild_config = self.configure.parse_oebuild_config() - yocto_config: ConfigBasicRepo = \ - oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] - - local_dir = os.path.join(self.configure.source_dir(), - yocto_config.path) - yocto_repo = OGit(repo_dir=local_dir, - remote_url=yocto_config.remote_url, - branch=yocto_config.branch) + yocto_config: ConfigBasicRepo = oebuild_config.basic_repo[ + oebuild_const.YOCTO_META_OPENEULER + ] + + local_dir = os.path.join( + self.configure.source_dir(), yocto_config.path + ) + yocto_repo = OGit( + repo_dir=local_dir, + remote_url=yocto_config.remote_url, + branch=yocto_config.branch, + ) yocto_repo.clone_or_pull_repo() def docker_image_update(self, docker_tag=None): - ''' + """ The container update logic is to update the corresponding tag container image if tag is specified, otherwise it is determined according to the yocto-meta-openeuler version branch in config, and if the version branch does not correspond to it, it will enter interactive mode, which is selected by the user - ''' + """ oebuild_config = self.configure.parse_oebuild_config() docker_config = oebuild_config.docker - check_docker_tag = CheckDockerTag(docker_tag=docker_tag, - configure=self.configure) + check_docker_tag = CheckDockerTag( + docker_tag=docker_tag, configure=self.configure + ) if docker_tag is not None: - if check_docker_tag.get_tag() is None or check_docker_tag.get_tag( - ) == "": + if ( + check_docker_tag.get_tag() is None + or check_docker_tag.get_tag() == '' + ): check_docker_tag.list_image_tag() return - docker_image = docker_config.repo_url + ":" + check_docker_tag.get_tag( + docker_image = ( + docker_config.repo_url + ':' + check_docker_tag.get_tag() ) else: docker_image = oebuild_util.get_docker_image_from_yocto( - self.configure.source_yocto_dir()) - if docker_image is None or docker_image == "": - if check_docker_tag.get_tag( - ) is None or check_docker_tag.get_tag() == "": + self.configure.source_yocto_dir() + ) + if docker_image is None or docker_image == '': + if ( + check_docker_tag.get_tag() is None + or check_docker_tag.get_tag() == '' + ): check_docker_tag.list_image_tag() return - docker_image = docker_config.repo_url + ":" + check_docker_tag.get_tag( + docker_image = ( + docker_config.repo_url + ':' + check_docker_tag.get_tag() ) client = DockerProxy() - logger.info("Pull %s ...", docker_image) + logger.info('Pull %s ...', docker_image) client.pull_image_with_progress(docker_image) # check if docker image had download successful if not client.is_image_exists(docker_image): - logger.error("docker pull %s failed", docker_image) + logger.error('docker pull %s failed', docker_image) sys.exit(-1) - logger.info("finishd pull %s ...", docker_image) + logger.info('finishd pull %s ...', docker_image) diff --git a/src/oebuild/auto_completion.py b/src/oebuild/auto_completion.py index c910409..97076c2 100644 --- a/src/oebuild/auto_completion.py +++ b/src/oebuild/auto_completion.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,8 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" + import hashlib import re import sys @@ -19,9 +20,9 @@ from os.path import abspath, expanduser, expandvars, join logger = logging.getLogger() -class AutoCompletion(): +class AutoCompletion: """ - code hint + code hint """ def run(self): @@ -31,34 +32,44 @@ class AutoCompletion(): """ oebuild_rc = textwrap.dedent(""" - ###!###>>>>>>>>>>>oebuild_complete>>>>>>>>>>>>>>> + ###!###>>>>>>>>>>>oebuild_complete>>>>>>>>>>>>>>> if pip list | grep oebuild &> /dev/null ; then - export oebuild_sh=$(pip show oebuild | grep Location | awk -F" " '{print $2}')/oebuild/app/conf/oebuild.sh + export oebuild_sh=$(pip show oebuild | grep Location | \ + awk -F" " '{print $2}')/oebuild/app/conf/oebuild.sh if [ -f $oebuild_sh ] ; then . $oebuild_sh fi fi ###!###<<<<<<<<<<>>>>>>>>>>oebuild_complete>>>>>>>>>>>>>>>)[\W\w]+(?<=###!###<<<<<<<<<<<' - 'oebuild_complete<<<<<<<<<<<<<<<)') + 'oebuild_complete<<<<<<<<<<<<<<<)' + ) re_info = re.search(pattern_bashrc, rc_content) if re_info is None: - rc_content += f"\n{oebuild_rc}\n" + rc_content += f'\n{oebuild_rc}\n' with open(bashrc_path, 'w', encoding='utf-8') as fh: fh.write(rc_content) else: @@ -66,7 +77,9 @@ class AutoCompletion(): bashrc_ma5 = self.md5_string(bashrc_data) rc_content_md5 = self.md5_string(oebuild_rc) if bashrc_ma5 != rc_content_md5: - rc_content = re.sub(pattern_bashrc, f"\n{oebuild_rc}\n", rc_content) + rc_content = re.sub( + pattern_bashrc, f'\n{oebuild_rc}\n', rc_content + ) with open(bashrc_path, 'w', encoding='utf-8') as fh: fh.write(rc_content) @@ -80,6 +93,6 @@ class AutoCompletion(): """ md5 = hashlib.md5() - md5.update(in_str.encode("utf8")) + md5.update(in_str.encode('utf8')) result = md5.hexdigest() return result diff --git a/src/oebuild/bashrc.py b/src/oebuild/bashrc.py index 131a620..694ecae 100644 --- a/src/oebuild/bashrc.py +++ b/src/oebuild/bashrc.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os import sys @@ -24,36 +24,39 @@ from oebuild.m_log import logger class Bashrc: - ''' + """ is for modify ${HOME}/.bashrc - ''' - def __init__(self,) -> None: + """ + + def __init__( + self, + ) -> None: self.container = None self.client = None self.user = oebuild_const.CONTAINER_USER - self.home_dir = f"/home/{oebuild_const.CONTAINER_USER}" + self.home_dir = f'/home/{oebuild_const.CONTAINER_USER}' def set_user(self, user): - ''' + """ set user param, so the next steps will be in the user pointed - ''' + """ self.user = user - if user == "root": - self.home_dir = "/root/" + if user == 'root': + self.home_dir = '/root/' def set_container(self, container: Container): - ''' + """ After setting the container parameters, all operations on bashrc will be based on the container. - ''' + """ self.container = container self.client = DockerProxy() def update_bashrc(self, content: str): - ''' + """ update user initialization script by replace file, first create a file and writed content and move it to .bashrc - ''' + """ tmp_file = self._set_tmpfile_content(content) # deal host bashrc if self.container is None: @@ -63,48 +66,53 @@ class Bashrc: self.client.copy_to_container( container=self.container, source_path=tmp_file, - to_path=self.home_dir) + to_path=self.home_dir, + ) self.container.exec_run( - cmd=f''' + cmd=f""" mv {self.home_dir}/{tmp_file} {self.home_dir}/.bashrc - ''', - user="root" + """, + user='root', ) os.remove(tmp_file) def clean_command_bash(self): - ''' + """ this is for finished .bashrc then clean command auto - ''' + """ old_content = self.get_bashrc_content() - clean_command = f"sed -i '/{oebuild_const.BASH_END_FLAG.strip()}$/d' ~/.bashrc" + clean_command = ( + f"sed -i '/{oebuild_const.BASH_END_FLAG.strip()}$/d' ~/.bashrc" + ) content = Bashrc().add_bashrc(old_content, clean_command) self.update_bashrc(content=content) def restore_bashrc(self): - ''' + """ Restoring .bashrc will strip out the command line content added during bitbake initialization - ''' + """ old_content = self.get_bashrc_content() - self.update_bashrc(content=self.restore_bashrc_content(old_content=old_content)) + self.update_bashrc( + content=self.restore_bashrc_content(old_content=old_content) + ) - def get_bashrc_content(self,): - ''' + def get_bashrc_content( + self, + ): + """ get bashrc shell content - ''' + """ # return host bashrc if self.container is None: return subprocess.getoutput('cat $HOME/.bashrc') # deal container bashrc res = self.client.container_exec_command( container=self.container, - command=f"cat {self.home_dir}/.bashrc", - user="root", - params={ - "work_space": None, - "stream": False - }) + command=f'cat {self.home_dir}/.bashrc', + user='root', + params={'work_space': None, 'stream': False}, + ) if res.exit_code != 0: logger.error(res.output) @@ -114,38 +122,43 @@ mv {self.home_dir}/{tmp_file} {self.home_dir}/.bashrc @staticmethod def restore_bashrc_content(old_content): - ''' + """ restore bashrc content, it will delete line with oebuild_const.BASH_END_FLAG - ''' + """ new_content = '' for line in old_content.split('\n'): line: str = line - if line.endswith(oebuild_const.BASH_END_FLAG) or line.replace(" ", '') == '': + if ( + line.endswith(oebuild_const.BASH_END_FLAG) + or line.replace(' ', '') == '' + ): continue new_content = new_content + line + '\n' return new_content @staticmethod def add_bashrc(content: str, line: str): - ''' + """ add new line to bashrc with oebuild_const.BASH_END_FLAG - ''' + """ if not content.endswith('\n'): content = content + '\n' - for split in line.split("\n"): + for split in line.split('\n'): content = content + split + oebuild_const.BASH_END_FLAG + '\n' return content @staticmethod def init_bashrc_content(old_content, init_command: list): - ''' + """ add init command line to bashrc shell - ''' + """ new_content = Bashrc().restore_bashrc_content(old_content=old_content) for command in init_command: - new_content = new_content + command + oebuild_const.BASH_END_FLAG + '\n' + new_content = ( + new_content + command + oebuild_const.BASH_END_FLAG + '\n' + ) return new_content @@ -154,7 +167,7 @@ mv {self.home_dir}/{tmp_file} {self.home_dir}/.bashrc tmp_file = oebuild_util.generate_random_str(6) if os.path.exists(tmp_file): continue - with open(tmp_file, 'w', encoding="utf-8") as w_f: + with open(tmp_file, 'w', encoding='utf-8') as w_f: w_f.write(content) break return tmp_file diff --git a/src/oebuild/bb/utils.py b/src/oebuild/bb/utils.py index 791f385..f84c3d0 100644 --- a/src/oebuild/bb/utils.py +++ b/src/oebuild/bb/utils.py @@ -110,10 +110,14 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): for var in variables: if var.endswith('()'): var_res[var] = re.compile( - r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re)) + r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' + % (var[:-2].rstrip(), override_re) + ) else: var_res[var] = re.compile( - r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re)) + r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' + % (var, override_re) + ) updated = False varset_start = '' @@ -126,8 +130,10 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): def handle_var_end(): prerun_newlines = newlines[:] op = varset_start[len(in_var):].strip() - (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines) - changed = (prerun_newlines != newlines) + (newvalue, newop, indent, minbreak) = varfunc( + in_var, full_value, op, newlines + ) + changed = prerun_newlines != newlines if newvalue is None: # Drop the value @@ -135,7 +141,7 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): elif newvalue != full_value or (newop not in [None, op]): if newop not in [None, op]: # Callback changed the operator - varset_new = "%s %s" % (in_var, newop) + varset_new = '%s %s' % (in_var, newop) else: varset_new = varset_start @@ -149,8 +155,14 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): if in_var.endswith('()'): # A function definition if isinstance(newvalue, list): - newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, - ('\n%s' % indentspc).join(newvalue))) + newlines.append( + '%s {\n%s%s\n}\n' + % ( + varset_new, + indentspc, + ('\n%s' % indentspc).join(newvalue), + ) + ) else: if not newvalue.startswith('\n'): newvalue = '\n' + newvalue @@ -166,11 +178,17 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): elif minbreak: # First item on first line if len(newvalue) == 1: - newlines.append('%s "%s"\n' % (varset_new, newvalue[0])) + newlines.append( + '%s "%s"\n' % (varset_new, newvalue[0]) + ) else: - newlines.append('%s "%s \\\n' % (varset_new, newvalue[0])) + newlines.append( + '%s "%s \\\n' % (varset_new, newvalue[0]) + ) for item in newvalue[1:]: - newlines.append('%s%s \\\n' % (indentspc, item)) + newlines.append( + '%s%s \\\n' % (indentspc, item) + ) newlines.append('%s"\n' % indentspc) else: # No item on first line @@ -208,7 +226,7 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): in_var = None else: skip = False - for (varname, var_re) in var_res.items(): + for varname, var_re in var_res.items(): res = var_re.match(line) if res: isfunc = varname.endswith('()') @@ -307,7 +325,10 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): if removelayers: for removelayer in removelayers: for layer in bblayers: - if fnmatch.fnmatch(canonicalise_path(layer), canonicalise_path(removelayer)): + if fnmatch.fnmatch( + canonicalise_path(layer), + canonicalise_path(removelayer), + ): updated = True bblayers.remove(layer) removed.append(removelayer) @@ -338,7 +359,9 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): return (origvalue, None, 2, False) with open(bblayers_conf, 'r') as f: - (_, newlines) = edit_metadata(f, ['BBLAYERS'], handle_bblayers_firstpass) + (_, newlines) = edit_metadata( + f, ['BBLAYERS'], handle_bblayers_firstpass + ) if not bblayercalls: raise Exception('Unable to find BBLAYERS in %s' % bblayers_conf) @@ -352,12 +375,21 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): notadded = [] for layer in addlayers: layer_canon = canonicalise_path(layer) - if layer_canon in orig_bblayers and not layer_canon in removelayers_canon: + if ( + layer_canon in orig_bblayers + and layer_canon not in removelayers_canon + ): notadded.append(layer) notadded_canon = [canonicalise_path(layer) for layer in notadded] - addlayers[:] = [layer for layer in addlayers if canonicalise_path(layer) not in notadded_canon] + addlayers[:] = [ + layer + for layer in addlayers + if canonicalise_path(layer) not in notadded_canon + ] - (updated, newlines) = edit_metadata(newlines, ['BBLAYERS'], handle_bblayers) + (updated, newlines) = edit_metadata( + newlines, ['BBLAYERS'], handle_bblayers + ) if addlayers: # Still need to add these for addlayer in addlayers: diff --git a/src/oebuild/bblayers.py b/src/oebuild/bblayers.py index 92df74a..fd57527 100644 --- a/src/oebuild/bblayers.py +++ b/src/oebuild/bblayers.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os @@ -16,11 +16,11 @@ import oebuild.bb.utils as bb_utils class BBLayers: - ''' + """ The BBlayer class implements the layer added in the container environment in the physical environment, and the add operation references bitbake-related code - ''' + """ def __init__(self, bblayers_dir: str, base_dir: str): self._base_dir = base_dir @@ -28,20 +28,20 @@ class BBLayers: @property def base_dir(self): - ''' + """ Returns base_dir value - ''' + """ return self._base_dir @property def bblayers_dir(self): - ''' + """ Returns bblayers_dir value - ''' + """ return self._bblayers_dir def add_layer(self, pre_dir: str, layers): - ''' + """ Add a layer layer to bblayers.conf, but our layer layer verification is done on the host, and the added path is written as a path in the container @@ -49,7 +49,7 @@ class BBLayers: pre_dir (str): when added layer with path, for example pre_dir/layer layers (str or list): needed to add to bblayers.conf - ''' + """ try: self.check_layer_exist(layers=layers) except Exception as e_p: @@ -62,15 +62,17 @@ class BBLayers: for layer in layers: bblayers.append(os.path.join(pre_dir, layer)) - bb_utils.edit_bblayers_conf(self.bblayers_dir, add=bblayers, remove=None) + bb_utils.edit_bblayers_conf( + self.bblayers_dir, add=bblayers, remove=None + ) def check_layer_exist(self, layers): - ''' + """ To check if it is legitimate to add a layer, the main thing is to verify the existence of layer.conf args: layers (str or list): needed to add to bblayers.conf - ''' + """ bblayers = [] if isinstance(layers, str): bblayers.append(layers) @@ -80,9 +82,9 @@ class BBLayers: for layer in bblayers: layer_dir = os.path.join(self.base_dir, layer) if not os.path.exists(layer_dir): - raise ValueError("layer does not exists") + raise ValueError('layer does not exists') layer_conf_dir = os.path.join(layer_dir, 'conf', 'layer.conf') if not os.path.exists(layer_conf_dir): print(layer_conf_dir) - raise ValueError("invalid layer") + raise ValueError('invalid layer') diff --git a/src/oebuild/check_docker_tag.py b/src/oebuild/check_docker_tag.py index a40979a..dc599b3 100644 --- a/src/oebuild/check_docker_tag.py +++ b/src/oebuild/check_docker_tag.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os import pathlib @@ -20,14 +20,14 @@ import oebuild.util as oebuild_util class CheckDockerTag: - ''' + """ This class is used to synthesize the build environment parameters to obtain the docker image version that should be updated when building with OEBUILD, and these environment parameters are the docker_tag entered by the user, the env configuration file, the yocto branch name, etc. Judge down in turn, and finally return a suitable docker image version - ''' + """ def __init__(self, docker_tag: str, configure: Configure): self.docker_tag = docker_tag @@ -35,43 +35,50 @@ class CheckDockerTag: self.tag_list = [] self._parse_docker_tag() - def _parse_docker_tag(self,): + def _parse_docker_tag( + self, + ): oebuild_config = self.configure.parse_oebuild_config() docker_config = oebuild_config.docker - tags = {} self.tag_list = list(set(docker_config.tag_map.values())) - def get_tags(self,): - ''' + def get_tags( + self, + ): + """ return docker image tag list - ''' + """ return self.tag_list - def list_image_tag(self,): - ''' + def list_image_tag( + self, + ): + """ print compile docker image tag list - ''' + """ oebuild_config = self.configure.parse_oebuild_config() docker_config = oebuild_config.docker - log = f'''the openeuler embedded docker image repo url: + log = f"""the openeuler embedded docker image repo url: {docker_config.repo_url} the openeuler embedded docker tag can be selected list: -''' +""" for tag in self.tag_list: - log += f"{tag}\n" + log += f'{tag}\n' print(log) - def get_tag(self,) -> str: - ''' + def get_tag( + self, + ) -> str: + """ return docker instance tag - ''' - if self.docker_tag is not None and self.docker_tag != "": + """ + if self.docker_tag is not None and self.docker_tag != '': if self.docker_tag not in self.tag_list: return None return str(self.docker_tag) yocto_dir = self.configure.source_yocto_dir() - env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") + env_path = os.path.join(yocto_dir, '.oebuild/env.yaml') if os.path.exists(env_path): env_parse = oebuild_util.read_yaml(pathlib.Path(env_path)) return str(env_parse['docker_tag']) diff --git a/src/oebuild/command.py b/src/oebuild/command.py index cb2b0f8..d9d6939 100644 --- a/src/oebuild/command.py +++ b/src/oebuild/command.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" from abc import ABC, abstractmethod import argparse @@ -16,7 +16,7 @@ from typing import List class OebuildCommand(ABC): - '''Abstract superclass for a oebuild command.''' + """Abstract superclass for a oebuild command.""" def __init__(self, name, help_msg, description): self.name = name @@ -25,20 +25,20 @@ class OebuildCommand(ABC): self.parser: argparse.ArgumentParser = None def run(self, args: argparse.ArgumentParser, unknown: List): - ''' + """ The executing body, each inherited class will register the executor with the executor body for execution - ''' + """ self.do_run(args=args, unknown=unknown) def pre_parse_help(self, args: argparse.ArgumentParser, unknown: List): - ''' + """ Whether to parse the help command in advance, designed to adapt to some extended scenarios that do not require command resolution, generally the function is placed in the front of the do_run to execute, if it returns true, it means that it is a help command, then there is no need to continue to execute, otherwise the specific function content is executed - ''' + """ pars = args.parse_args(unknown) if pars.help: self.print_help_msg() @@ -46,41 +46,46 @@ class OebuildCommand(ABC): return False def add_parser(self, parser_adder: argparse.ArgumentParser): - ''' + """ Registers a parser for this command, and returns it. The parser object is stored in a ``parser`` attribute. :param parser_adder: The return value of a call to ``argparse.ArgumentParser.add_subparsers()`` - ''' + """ self.parser: argparse.ArgumentParser = self.do_add_parser(parser_adder) if self.parser is None: raise ValueError('do_add_parser did not return a value') - self.parser.add_argument('-h', '--help', dest="help", action="store_true", - help='get help for oebuild or a command') + self.parser.add_argument( + '-h', + '--help', + dest='help', + action='store_true', + help='get help for oebuild or a command', + ) return self.parser @abstractmethod def do_add_parser(self, parser_adder: argparse.ArgumentParser): - ''' + """ The directive registers the interface, which the successor needs to implement - ''' + """ @abstractmethod def do_run(self, args: argparse.Namespace, unknown: List): - ''' + """ Subclasses must implement; called to run the command. :param args: ``argparse.Namespace`` of parsed arguments :param unknown: If ``accepts_unknown_args`` is true, a sequence of un-parsed argument strings. - ''' + """ def print_help_msg(self): - ''' + """ print help message - ''' + """ self.parser.print_help() def _parser(self, parser: argparse.ArgumentParser, **kwargs): diff --git a/src/oebuild/configure.py b/src/oebuild/configure.py index 109f62b..95e5a1e 100644 --- a/src/oebuild/configure.py +++ b/src/oebuild/configure.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os from typing import Dict, Optional, Union @@ -22,14 +22,15 @@ PathType = Union[str, os.PathLike] class OebuildNotFound(RuntimeError): - '''Neither the current directory nor any parent has a oebuild workspace.''' + """Neither the current directory nor any parent has a oebuild workspace.""" @dataclass class ConfigContainer: - ''' + """ container object in config - ''' + """ + # repo_url is for container's repo url repo_url: str @@ -39,9 +40,10 @@ class ConfigContainer: @dataclass class ConfigBasicRepo: - ''' + """ basic repo object in config - ''' + """ + # path is for repo's path that will downloaded path: str @@ -54,26 +56,28 @@ class ConfigBasicRepo: @dataclass class Config: - ''' + """ config object container docker and basic_repo - ''' + """ + docker: ConfigContainer basic_repo: dict class Configure: - ''' + """ Configure object is to contain some generally param or function about oebuild - ''' + """ @staticmethod - def oebuild_topdir(start: Optional[PathType] = None, - fall_back: bool = True): - ''' + def oebuild_topdir( + start: Optional[PathType] = None, fall_back: bool = True + ): + """ Like oebuild_dir(), but returns the path to the parent directory of the .oebuild/ directory instead, where project repositories are stored - ''' + """ cur_dir = pathlib.Path(start or os.getcwd()) while True: @@ -86,27 +90,28 @@ class Configure: if fall_back: return Configure.oebuild_topdir(fall_back=False) - raise OebuildNotFound('Could not find a oebuild workspace ' - 'in this or any parent directory') + raise OebuildNotFound( + 'Could not find a oebuild workspace in this or any parent directory' + ) cur_dir = parent_dir @staticmethod def oebuild_dir(start: Optional[PathType] = None): - '''Returns the absolute path of the workspace's .oebuild directory. + """Returns the absolute path of the workspace's .oebuild directory. Starts the search from the start directory, and goes to its parents. If the start directory is not specified, the current directory is used. Raises OebuildNotFound if no .oebuild directory is found. - ''' + """ return os.path.join(Configure.oebuild_topdir(start), '.oebuild') @staticmethod def is_oebuild_dir(start: Optional[PathType] = None): - ''' + """ Determine whether OEBuild is initialized - ''' + """ try: Configure.oebuild_topdir(start) return True @@ -115,93 +120,104 @@ class Configure: @staticmethod def source_dir(): - ''' + """ returns src directory base on topdir, the openEuler Embedded meta layers will be in here when you run oebuild update - ''' + """ return os.path.join(Configure.oebuild_topdir(), 'src') @staticmethod def source_yocto_dir(): - ''' + """ return src/yocto-meta-openeuler path - ''' + """ config = Configure.parse_oebuild_config() basic_config = config.basic_repo - yocto_config: ConfigBasicRepo = basic_config[oebuild_const.YOCTO_META_OPENEULER] + yocto_config: ConfigBasicRepo = basic_config[ + oebuild_const.YOCTO_META_OPENEULER + ] yocto_dir = yocto_config.path return os.path.join(Configure.source_dir(), yocto_dir) @staticmethod def yocto_manifest_dir(): - ''' + """ return .oebuild/manifest.yaml - ''' - return os.path.join(Configure.source_yocto_dir(), ".oebuild/manifest.yaml") + """ + return os.path.join( + Configure.source_yocto_dir(), '.oebuild/manifest.yaml' + ) @staticmethod def yocto_samples_dir(): - ''' + """ return .oebuild/samples - ''' - return os.path.join(Configure.source_yocto_dir(), ".oebuild/samples") + """ + return os.path.join(Configure.source_yocto_dir(), '.oebuild/samples') @staticmethod def source_poky_dir(): - ''' + """ return src/yocto-poky path - ''' + """ return os.path.join(Configure.source_dir(), oebuild_const.YOCTO_POKY) @staticmethod def yocto_bak_dir(): - ''' + """ returns yocto_bak directory base on topdir, the openEuler Embedded meta layers will be in here when you run oebuild update - ''' + """ return os.path.join(Configure.oebuild_topdir(), 'yocto_bak') @staticmethod def build_dir(): - ''' + """ returns build absolute path which the build result will be in - ''' + """ return os.path.join(Configure.oebuild_topdir(), 'build') @staticmethod def source_nativesdk_dir(): - ''' + """ returns yocto-meta-openeuler/.oebuild/nativesdk - ''' + """ return os.path.join(Configure.source_yocto_dir(), '.oebuild/nativesdk') @staticmethod def env_dir(): - ''' + """ returns env path - ''' + """ return os.path.join(Configure.build_dir(), '.env') @staticmethod def parse_oebuild_config(): - ''' + """ just parse oebuild config and return a json object, the file path is {WORKSPACE}.oebuild/config - ''' + """ config = oebuild_util.read_yaml( - yaml_path=os.path.join(Configure.oebuild_dir(), oebuild_const.CONFIG)) + yaml_path=os.path.join( + Configure.oebuild_dir(), oebuild_const.CONFIG + ) + ) tag_map = {} for key, value in config['docker']['tag_map'].items(): tag_map[key] = value - docker_config = ConfigContainer(repo_url=config['docker']['repo_url'], tag_map=tag_map) + docker_config = ConfigContainer( + repo_url=config['docker']['repo_url'], tag_map=tag_map + ) basic_config = {} for key, repo in config['basic_repo'].items(): - basic_config[key] = ConfigBasicRepo(path=repo['path'], - remote_url=repo['remote_url'], - branch=repo['branch']) + basic_config[key] = ConfigBasicRepo( + path=repo['path'], + remote_url=repo['remote_url'], + branch=repo['branch'], + ) config = Config(docker=docker_config, basic_repo=basic_config) @@ -209,9 +225,9 @@ class Configure: @staticmethod def update_oebuild_config(config: Config): - ''' + """ update {WORKSPACE}/.oebuild/config - ''' + """ data = {} docker_config = config.docker @@ -226,14 +242,19 @@ class Configure: data['basic_repo'] = {} for key, repo in basic_config.items(): repo: ConfigBasicRepo = repo - data['basic_repo'][key] = {'path': repo.path, - 'remote_url': repo.remote_url, - 'branch': repo.branch} + data['basic_repo'][key] = { + 'path': repo.path, + 'remote_url': repo.remote_url, + 'branch': repo.branch, + } try: oebuild_util.write_yaml( - yaml_path=pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG), - data=data) + yaml_path=pathlib.Path( + Configure.oebuild_dir(), oebuild_const.CONFIG + ), + data=data, + ) return True except TypeError: return False diff --git a/src/oebuild/const.py b/src/oebuild/const.py index acd160e..c36e333 100644 --- a/src/oebuild/const.py +++ b/src/oebuild/const.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,30 +8,32 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" # used for util.py CONFIG_YAML = 'config.yaml' PLUGINS_YAML = 'plugins.yaml' UPGRADE_YAML = 'upgrade.yaml' COMPILE_YAML = 'compile.yaml.sample' -BASH_END_FLAG = " ###!!!###" -CONTAINER_USER = "openeuler" +BASH_END_FLAG = ' ###!!!###' +CONTAINER_USER = 'openeuler' CONTAINER_BUILD = '/home/openeuler/build' CONTAINER_LLVM_LIB = '/home/openeuler/llvm-lib' -DEFAULT_DOCKER = "swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container:latest" -DEFAULT_SDK_DOCKER = "swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-sdk:latest" +DEFAULT_DOCKER = 'swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container:latest' +DEFAULT_SDK_DOCKER = ( + 'swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-sdk:latest' +) CONTAINER_SRC = '/usr1/openeuler/src' -CONTAINER_USER = "openeuler" -NATIVESDK_DIR = "/opt/buildtools/nativesdk" +CONTAINER_USER = 'openeuler' +NATIVESDK_DIR = '/opt/buildtools/nativesdk' PROXY_LIST = ['http_proxy', 'https_proxy'] # used for local_conf -NATIVESDK_DIR_NAME = "OPENEULER_NATIVESDK_SYSROOT" -OPENEULER_SP_DIR = "OPENEULER_SP_DIR" -SSTATE_MIRRORS = "SSTATE_MIRRORS" -SSTATE_DIR = "SSTATE_DIR" -TMP_DIR = "TMPDIR" +NATIVESDK_DIR_NAME = 'OPENEULER_NATIVESDK_SYSROOT' +OPENEULER_SP_DIR = 'OPENEULER_SP_DIR' +SSTATE_MIRRORS = 'SSTATE_MIRRORS' +SSTATE_DIR = 'SSTATE_DIR' +TMP_DIR = 'TMPDIR' NATIVE_GCC_MAP = '/usr1/openeuler/native_gcc' NATIVE_LLVM_MAP = '/usr1/openeuler/native_llvm' @@ -39,20 +41,20 @@ SSTATE_MIRRORS_MAP = '/usr1/openeuler/sstate-mirrors' SSTATE_DIR_MAP = '/usr1/openeuler/sstate-dir' CACHE_SRC_DIR_MAP = '/usr1/openeuler/cache_src' -EXTERNAL_LLVM = "EXTERNAL_TOOLCHAIN_LLVM" -EXTERNAL_GCC = "EXTERNAL_TOOLCHAIN_GCC" -EXTERNAL = "EXTERNAL_TOOLCHAIN" -CACHE_SRC_DIR = "CACHE_SRC_DIR" +EXTERNAL_LLVM = 'EXTERNAL_TOOLCHAIN_LLVM' +EXTERNAL_GCC = 'EXTERNAL_TOOLCHAIN_GCC' +EXTERNAL = 'EXTERNAL_TOOLCHAIN' +CACHE_SRC_DIR = 'CACHE_SRC_DIR' # used for bitbake/in_container.py -BASH_BANNER = ''' +BASH_BANNER = """ Welcome to the openEuler Embedded build environment, where you can run [bitbake recipe] to build what you want, or you ran run [bitbake -h] for help -''' +""" # used for toolchain/toolchain.py -TOOLCHAIN_BASH_BANNER = ''' +TOOLCHAIN_BASH_BANNER = """ Welcome to the openEuler Embedded build environment, where you can create openEuler Embedded cross-chains tools by follows: "./cross-tools/prepare.sh ./" @@ -61,21 +63,21 @@ TOOLCHAIN_BASH_BANNER = ''' "cp config_arm32 .config && ct-ng build" "cp config_x86_64 .config && ct-ng build" "cp config_riscv64 .config && ct-ng build" -''' +""" # used for configure.py -YOCTO_META_OPENEULER = "yocto_meta_openeuler" -YOCTO_POKY = "yocto-poky" -CONFIG = "config" -COMPILE_YAML = "compile.yaml.sample" +YOCTO_META_OPENEULER = 'yocto_meta_openeuler' +YOCTO_POKY = 'yocto-poky' +CONFIG = 'config' +COMPILE_YAML = 'compile.yaml.sample' # used for parse_templete.py PLATFORM = 'platform' -BUILD_IN_DOCKER = "docker" -BUILD_IN_HOST = "host" +BUILD_IN_DOCKER = 'docker' +BUILD_IN_HOST = 'host' -DEFAULT_CONTAINER_PARAMS = "-itd --network host" +DEFAULT_CONTAINER_PARAMS = '-itd --network host' # used for toolchain type -GCC_TOOLCHAIN = "gcc" -LLVM_TOOLCHAIN = "llvm" +GCC_TOOLCHAIN = 'gcc' +LLVM_TOOLCHAIN = 'llvm' diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py index 1005fae..ca04a3c 100644 --- a/src/oebuild/docker_proxy.py +++ b/src/oebuild/docker_proxy.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os from io import BytesIO @@ -26,19 +26,19 @@ from oebuild.m_log import logger class DockerProxy: - ''' + """ a object just be wrapper again to run docker command easily - ''' + """ def __init__(self): self._docker = docker.from_env() def is_image_exists(self, image_name): - ''' + """ determize if image exist args: image_name (str): docker image name - ''' + """ try: self._docker.images.get(image_name) return True @@ -46,11 +46,11 @@ class DockerProxy: return False def get_container_img_id(self, container_id): - ''' + """ determize if container exists args: container_id (str): docker container short_id or id - ''' + """ try: container = self._docker.containers.get(container_id=container_id) return container.image.id @@ -58,11 +58,11 @@ class DockerProxy: return None def is_container_exists(self, container_id): - ''' + """ determize if container exists args: container_id (str): docker container short_id or id - ''' + """ try: self._docker.containers.get(container_id=container_id) return True @@ -70,97 +70,97 @@ class DockerProxy: return False def pull_image(self, image_name: str): - ''' + """ pull image like command 'docker pull' args: image_name (str): docker image name, if no tag, the default tag is latest - ''' + """ repository, tag = self._get_image_name_tag(image_name=image_name) self._docker.images.pull(repository=repository, tag=tag) def pull_image_with_progress(self, image_name: str): - ''' + """ pull docker image and print progress - ''' - os.system(f"docker pull {image_name}") + """ + os.system(f'docker pull {image_name}') def _get_image_name_tag(self, image_name: str): repository = image_name.split(':')[0] - tag = "latest" + tag = 'latest' if len(image_name.split(':')) == 2: tag = image_name.split(':')[1] return repository, tag def get_image(self, image_name): - ''' + """ get a docker image object args: image_name (str): docker image name - ''' + """ return self._docker.images.get(image_name) def get_container(self, container_id): - ''' + """ get a docker container object args: container_id (str): docker container short_id or id - ''' + """ return self._docker.containers.get(container_id=container_id) def get_all_container(self): - ''' + """ get all container like command 'docker ps -a' args: None - ''' + """ return self._docker.containers.list(all=True) @staticmethod def stop_container(container: Container): - ''' + """ stop a container if running like command 'docker stop' args: container (Container): container object - ''' + """ container.stop() @staticmethod def delete_container(container: Container, is_force: bool = False): - ''' + """ rm a container which not running like command 'docker rm' args: container (Container): container object - ''' + """ container.remove(force=is_force) @staticmethod def start_container(container: Container): - ''' + """ start a container like command 'docker start' args: container (Container): container object - ''' + """ container.start() @staticmethod def is_container_running(container: Container): - ''' + """ determize if a container in running state args: container (Container): container object - ''' - if container.status == "running": + """ + if container.status == 'running': return True return False @staticmethod def add_tar(path_dir): - ''' + """ add a path to tar args: path_dir (str): the directory that will added to a tar - ''' + """ if os.path.exists(path=path_dir): pw_tarstream = BytesIO() with tarfile.TarFile(fileobj=pw_tarstream, mode='w') as pw_tar: @@ -170,24 +170,24 @@ class DockerProxy: return None def copy_to_container(self, container: Container, source_path, to_path): - ''' + """ copy file that tar before to container args: container (Container): docker container object source_path (str): which copied file path to_path (str): will copy to docker container path - ''' + """ tar = self.add_tar(source_path) return container.put_archive(path=to_path, data=tar) def copy_from_container(self, container: Container, from_path, dst_path): - ''' + """ copy file from container to local args: container (Container): docker container object from_path (str): which copied file path dst_path (str): will copy from docker container path - ''' + """ pw_tarstream = BytesIO() bits, _ = container.get_archive(from_path) for trunk in bits: @@ -197,42 +197,43 @@ class DockerProxy: res = tar.extractall(path=dst_path) return res is None - def container_exec_command(self, container: Container, - command, - user: str = '', - params=None): - ''' + def container_exec_command( + self, container: Container, command, user: str = '', params=None + ): + """ run command like 'docker run exec', other param will be default and just use a little params returns a data stream - ''' + """ if params is None: params = {} res = container.exec_run( cmd=command, user=user, - workdir=None if "work_space" not in params else params['work_space'], + workdir=None + if 'work_space' not in params + else params['work_space'], stderr=True, stdout=True, - stream=True if "stream" not in params else params['stream'], - demux=False if "demux" not in params else params['demux'] + stream=True if 'stream' not in params else params['stream'], + demux=False if 'demux' not in params else params['demux'], ) return res - def create_container(self, - image: str, - parameters: str, - volumes: List, - command: str) -> Container: - ''' + def create_container( + self, image: str, parameters: str, volumes: List, command: str + ) -> Container: + """ create a new container - ''' - run_command = f"docker run {parameters}" + """ + run_command = f'docker run {parameters}' for volume in volumes: - run_command = f"{run_command} -v {volume}" - run_command = f"{run_command} {image} {command}" - res = subprocess.run(run_command, shell=True, capture_output=True, check=True, text=True) + run_command = f'{run_command} -v {volume}' + run_command = f'{run_command} {image} {command}' + res = subprocess.run( + run_command, shell=True, capture_output=True, check=True, text=True + ) if res.returncode != 0: logger.error(res.stderr.strip()) sys.exit(res.returncode) @@ -240,21 +241,19 @@ class DockerProxy: return self.get_container(container_id=container_id) def check_change_ugid(self, container: Container, container_user): - ''' + """ the function is to check and change container user uid and gid to same with host's, together also alter directory pointed uid and gid - ''' + """ res = self.container_exec_command( container=container, user='root', - command=f"id {container_user}", - params={ - "work_space": f"/home/{container_user}", - "stream": False - }) + command=f'id {container_user}', + params={'work_space': f'/home/{container_user}', 'stream': False}, + ) if res.exit_code != 0: - raise ValueError("check docker user id faild") + raise ValueError('check docker user id faild') res_cont = res.output.decode() @@ -265,14 +264,14 @@ class DockerProxy: if match_uid: cuid = match_uid.group() else: - raise ValueError(f"can not get container {container_user} uid") + raise ValueError(f'can not get container {container_user} uid') # get gid from container in default user pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + container_user + r'\))') match_gid = pattern.search(cuids[1]) if match_gid: cgid = match_gid.group() else: - raise ValueError(f"can not get container {container_user} gid") + raise ValueError(f'can not get container {container_user} gid') # judge host uid and gid are same with container uid and gid # if not same and change container uid and gid equal to host's uid and gid @@ -280,35 +279,37 @@ class DockerProxy: self.change_container_uid( container=container, uid=os.getuid(), - container_user=container_user) + container_user=container_user, + ) if os.getgid() != cgid: self.change_container_gid( container=container, gid=os.getgid(), - container_user=container_user) + container_user=container_user, + ) - def change_container_uid(self, container: Container, uid: int, container_user): - ''' + def change_container_uid( + self, container: Container, uid: int, container_user + ): + """ alter container user pointed uid - ''' + """ self.container_exec_command( container=container, user='root', - command=f"usermod -u {uid} {container_user}", - params={ - "work_space": f"/home/{container_user}", - "stream": False - }) - - def change_container_gid(self, container: Container, gid: int, container_user): - ''' + command=f'usermod -u {uid} {container_user}', + params={'work_space': f'/home/{container_user}', 'stream': False}, + ) + + def change_container_gid( + self, container: Container, gid: int, container_user + ): + """ alter container pointed gid - ''' + """ self.container_exec_command( container=container, user='root', - command=f"groupmod -g {gid} {container_user}", - params={ - "work_space": f"/home/{container_user}", - "stream": False - }) + command=f'groupmod -g {gid} {container_user}', + params={'work_space': f'/home/{container_user}', 'stream': False}, + ) diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index 0092e46..2a12bbb 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os import re @@ -21,51 +21,51 @@ from oebuild.struct import CompileParam class BaseLocalConf(ValueError): - ''' + """ basic error about parse_template - ''' + """ class NativesdkNotExist(BaseLocalConf): - ''' + """ nativesdk directory not exist - ''' + """ class NativesdkNotValid(BaseLocalConf): - ''' + """ nativesdk directory not valid - ''' + """ def get_nativesdk_sysroot(nativesdk_dir=oebuild_const.NATIVESDK_DIR): - ''' + """ return environment initialization shell, if nativesdk directory is not exists or can not find any initialization shell, raise error - ''' - sysroot_dir = os.path.join(nativesdk_dir, "sysroots") + """ + sysroot_dir = os.path.join(nativesdk_dir, 'sysroots') if not os.path.isdir(nativesdk_dir): - logger.error("the %s is not exists", nativesdk_dir) + logger.error('the %s is not exists', nativesdk_dir) sys.exit(1) if not os.path.isdir(sysroot_dir): - logger.error("the %s is not value", nativesdk_dir) + logger.error('the %s is not value', nativesdk_dir) sys.exit(1) # list items in nativesdk to find environment shell list_items = os.listdir(sysroot_dir) for item in list_items: - ret = re.match("^(x86_64-)[a-zA-Z0-9]{1,}(-linux)$", item) + ret = re.match('^(x86_64-)[a-zA-Z0-9]{1,}(-linux)$', item) if ret is not None: abs_path = os.path.join(sysroot_dir, item) if os.path.isdir(abs_path): - return os.path.join("sysroots", item) - logger.error("can not find any sysroots directory") + return os.path.join('sysroots', item) + logger.error('can not find any sysroots directory') sys.exit(1) def match_and_add(new_str: str, content: str): - ''' + """ math line in content when the new_str not exist and added - ''' + """ for line in content.split('\n'): if new_str.strip() != line.strip(): continue @@ -78,9 +78,9 @@ def match_and_add(new_str: str, content: str): def match_and_replace(pre: str, new_str: str, content: str): - ''' + """ math line in content when the new_str exist and replace - ''' + """ for line in content.split('\n'): ret = re.match(f'^({pre})', line) if ret is None: @@ -94,10 +94,10 @@ def match_and_replace(pre: str, new_str: str, content: str): class LocalConf: - ''' + """ LocalConf corresponds to the local.conf configuration file, which can be modified by specifying parameters - ''' + """ def __init__(self, local_conf_path: str): self.local_path = local_conf_path @@ -114,10 +114,12 @@ class LocalConf: os.system(rf"sed -i 's/^ \+//' {self.local_path}") def update(self, compile_param: CompileParam, src_dir=None): - ''' + """ update local.conf by ParseCompile - ''' - pre_content = self._deal_other_local_param(compile_param=compile_param, src_dir=src_dir) + """ + pre_content = self._deal_other_local_param( + compile_param=compile_param, src_dir=src_dir + ) compile_param.local_conf = f'{pre_content}\n{compile_param.local_conf}' self._add_content_to_local_conf(local_conf=compile_param.local_conf) @@ -133,7 +135,7 @@ class LocalConf: replace_toolchain_str = f''' {oebuild_const.EXTERNAL_LLVM} = "{compile_param.llvm_toolchain_dir}"''' return replace_toolchain_str - return "" + return '' def _deal_cache_src_dir(self, compile_param: CompileParam): if compile_param.cache_src_dir is not None: @@ -145,57 +147,71 @@ class LocalConf: replace_cache_src_str = f''' {oebuild_const.CACHE_SRC_DIR} = "{compile_param.cache_src_dir}"''' return replace_cache_src_str - return "" + return '' def _deal_sstate_mirrors(self, compile_param: CompileParam): # replace sstate_cache if compile_param.sstate_mirrors is not None: if os.path.islink(compile_param.sstate_mirrors): - new_str = f"file://.* {compile_param.sstate_mirrors}/PATH;downloadfilename=PATH" + new_str = f'file://.* {compile_param.sstate_mirrors}/PATH;downloadfilename=PATH' else: if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: - new_str = f"file://.* file://{oebuild_const.SSTATE_MIRRORS_MAP}/PATH" + new_str = f'file://.* file://{oebuild_const.SSTATE_MIRRORS_MAP}/PATH' else: - new_str = f"file://.* file://{compile_param.sstate_mirrors}/PATH" + new_str = ( + f'file://.* file://{compile_param.sstate_mirrors}/PATH' + ) return f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"' - return "" + return '' def _deal_other_local_param(self, compile_param: CompileParam, src_dir): - pre_content = "" + pre_content = '' # add MACHINE if compile_param.machine is not None: pre_content += f'MACHINE = "{compile_param.machine}"\n' pre_content = self._deal_toolchain_replace(compile_param, pre_content) - pre_content += self._deal_llvm_toolchain_dir(compile_param) + "\n" + pre_content += self._deal_llvm_toolchain_dir(compile_param) + '\n' - pre_content += self._deal_cache_src_dir(compile_param) + "\n" + pre_content += self._deal_cache_src_dir(compile_param) + '\n' - pre_content += self._deal_sstate_mirrors(compile_param) + "\n" + pre_content += self._deal_sstate_mirrors(compile_param) + '\n' # replace nativesdk OPENEULER_SP_DIR if compile_param.build_in == oebuild_const.BUILD_IN_HOST: self.check_nativesdk_valid(compile_param.nativesdk_dir) if compile_param.nativesdk_dir is None: - raise ValueError("please set nativesdk dir") - nativesdk_sysroot = get_nativesdk_sysroot(compile_param.nativesdk_dir) - nativesdk_sys_dir = os.path.join(compile_param.nativesdk_dir, nativesdk_sysroot) - - pre_content += f'{oebuild_const.NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"\n' + raise ValueError('please set nativesdk dir') + nativesdk_sysroot = get_nativesdk_sysroot( + compile_param.nativesdk_dir + ) + nativesdk_sys_dir = os.path.join( + compile_param.nativesdk_dir, nativesdk_sysroot + ) + + pre_content += ( + f'{oebuild_const.NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"\n' + ) pre_content += f'{oebuild_const.OPENEULER_SP_DIR} = "{src_dir}"\n' # replace sstate_dir if compile_param.sstate_dir is not None: - pre_content += f'{oebuild_const.SSTATE_DIR} = "{compile_param.sstate_dir}"\n' + pre_content += ( + f'{oebuild_const.SSTATE_DIR} = "{compile_param.sstate_dir}"\n' + ) # replace tmpdir if compile_param.tmp_dir is not None: - pre_content += f'{oebuild_const.TMP_DIR} = "{compile_param.tmp_dir}"\n' + pre_content += ( + f'{oebuild_const.TMP_DIR} = "{compile_param.tmp_dir}"\n' + ) return pre_content - def _deal_toolchain_replace(self, compile_param: CompileParam, pre_content): + def _deal_toolchain_replace( + self, compile_param: CompileParam, pre_content + ): # The newly added external compiler chain is named EXTERNAL_TOOLCHAIN_GCC, however, # the old version still uses EXTERNAL_TOOLCHAIN. Therefore, to maintain compatibility # with both new and old versions, we have made the following adjustment: @@ -204,64 +220,91 @@ class LocalConf: if compile_param.toolchain_dir is not None: if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: # check if exists TXTERNAL_TOOLCHAIN_GCC, replace it or replace TXTERNAL_TOOLCHAIN - if len(re.findall(f'^({oebuild_const.EXTERNAL_GCC})*', self.content)) > 0: + if ( + len( + re.findall( + f'^({oebuild_const.EXTERNAL_GCC})*', self.content + ) + ) + > 0 + ): toolchain_type = compile_param.toolchain_type.replace( - oebuild_const.EXTERNAL, - oebuild_const.EXTERNAL_GCC) + oebuild_const.EXTERNAL, oebuild_const.EXTERNAL_GCC + ) replace_toolchain_str = f''' {toolchain_type} = "{oebuild_const.NATIVE_GCC_MAP}"''' else: replace_toolchain_str = f''' {compile_param.toolchain_type} = "{oebuild_const.NATIVE_GCC_MAP}"''' else: - if len(re.findall(f'^({oebuild_const.EXTERNAL_GCC})*', self.content)) > 0: + if ( + len( + re.findall( + f'^({oebuild_const.EXTERNAL_GCC})*', self.content + ) + ) + > 0 + ): toolchain_type = compile_param.toolchain_type.replace( - oebuild_const.EXTERNAL, - oebuild_const.EXTERNAL_GCC) + oebuild_const.EXTERNAL, oebuild_const.EXTERNAL_GCC + ) replace_toolchain_str = f''' {toolchain_type} = "{compile_param.toolchain_dir}"''' else: replace_toolchain_str = f''' {compile_param.toolchain_type} = "{compile_param.toolchain_dir}"''' - pre_content += replace_toolchain_str + "\n" + pre_content += replace_toolchain_str + '\n' return pre_content def _add_content_to_local_conf(self, local_conf): - user_content_flag = "#===========the content is user added==================" - if user_content_flag not in self.content and local_conf is not None and local_conf != "": + user_content_flag = ( + '#===========the content is user added==================' + ) + if ( + user_content_flag not in self.content + and local_conf is not None + and local_conf != '' + ): # check if exists remark sysmbol, if exists and replace it - self.content += f"\n{user_content_flag}\n" + self.content += f'\n{user_content_flag}\n' for line in local_conf.split('\n'): - if line.startswith("#"): - r_line = line.lstrip("#").strip(" ") + if line.startswith('#'): + r_line = line.lstrip('#').strip(' ') self.content = self.content.replace(r_line, line) - if line.strip(" ") == "None": + if line.strip(' ') == 'None': continue - self.content += line + "\n" + self.content += line + '\n' - with open(self.local_path, 'w', encoding="utf-8") as r_f: + with open(self.local_path, 'w', encoding='utf-8') as r_f: r_f.write(self.content) def check_nativesdk_valid(self, nativesdk_dir): - ''' + """ Check whether the set nativesdk is valid, check whether the path exists, and then check whether the internal OECORE_NATIVE_SYSROOT variables are consistent with the set nativesdk - ''' + """ if not os.path.exists(nativesdk_dir): - raise NativesdkNotExist(f"nativesdk directory: {nativesdk_dir} not exist") + raise NativesdkNotExist( + f'nativesdk directory: {nativesdk_dir} not exist' + ) nativesdk_environment_path = os.path.join( nativesdk_dir, - oebuild_util.get_nativesdk_environment(nativesdk_dir)) + oebuild_util.get_nativesdk_environment(nativesdk_dir), + ) with open(nativesdk_environment_path, 'r', encoding='utf-8') as r_f: for line in r_f.readlines(): line = line.strip('\n') - if not line.startswith("export OECORE_NATIVE_SYSROOT="): + if not line.startswith('export OECORE_NATIVE_SYSROOT='): continue - oecore_sysroot_dir = line.lstrip('export OECORE_NATIVE_SYSROOT=').strip('"') + oecore_sysroot_dir = line.lstrip( + 'export OECORE_NATIVE_SYSROOT=' + ).strip('"') if not oecore_sysroot_dir.startswith(nativesdk_dir): - raise NativesdkNotValid(f"nativesdk directory: {nativesdk_dir} are not valid") + raise NativesdkNotValid( + f'nativesdk directory: {nativesdk_dir} are not valid' + ) return diff --git a/src/oebuild/m_log.py b/src/oebuild/m_log.py index dc32330..4d91d64 100644 --- a/src/oebuild/m_log.py +++ b/src/oebuild/m_log.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,8 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" + import sys import logging @@ -26,9 +27,9 @@ logger.addHandler(ch) def set_log_to_file(): - ''' + """ set log to log file - ''' + """ fh = logging.FileHandler('oebuild.log') fh.setLevel(logging.INFO) fh.setFormatter(formatter) diff --git a/src/oebuild/oebuild_parser.py b/src/oebuild/oebuild_parser.py index af6a11c..e6ec0cc 100644 --- a/src/oebuild/oebuild_parser.py +++ b/src/oebuild/oebuild_parser.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import argparse import sys @@ -18,9 +18,9 @@ import textwrap class OebuildHelpAction(argparse.Action): - ''' + """ set argparse help is true - ''' + """ def __call__(self, parser, namespace, values, option_string=None): # Just mark that help was requested. @@ -28,7 +28,7 @@ class OebuildHelpAction(argparse.Action): class OebuildArgumentParser(argparse.ArgumentParser): - ''' + """ The argparse module is infuriatingly coy about its parser and help formatting APIs, marking almost everything you need to customize help output an "implementation detail". Even accessing @@ -40,7 +40,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): possible headaches by overriding some "proper" argparse APIs here instead of monkey-patching the module or breaking abstraction barriers. This is duplicative but more future-proof. - ''' + """ def __init__(self, *args, **kwargs): # The super constructor calls add_argument(), so this has to @@ -50,8 +50,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): super(OebuildArgumentParser, self).__init__(*args, **kwargs) def print_help(self, file=None): - print(self.format_help(), end='', - file=file or sys.stdout) + print(self.format_help(), end='', file=file or sys.stdout) def format_help(self): # When top_level is True, we override the parent method to @@ -79,9 +78,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): for s_t in strings: print(s_t, file=sio) - append(self.format_usage(), - self.description, - '') + append(self.format_usage(), self.description, '') append('optional arguments:') for w_o in self.oebuild_optionals: @@ -100,7 +97,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): def _format_oebuild_optional(self, append, w_o, width): metavar = w_o['metavar'] options = w_o['options'] - help_msg = w_o.get('help', "") + help_msg = w_o.get('help', '') # Join the various options together as a comma-separated list, # with the metavar if there is one. That's our "thing". @@ -131,9 +128,12 @@ class OebuildArgumentParser(argparse.ArgumentParser): # Reflow the lines in help to the desired with, using # the help_offset as an initial indent. help_msg = ' '.join(help_msg.split()) - help_lines = textwrap.wrap(help_msg, width=width, - initial_indent=help_indent, - subsequent_indent=help_indent) + help_lines = textwrap.wrap( + help_msg, + width=width, + initial_indent=help_indent, + subsequent_indent=help_indent, + ) if thinglen > help_offset - 1: # If the "thing" (plus room for a space) is longer @@ -154,8 +154,12 @@ class OebuildArgumentParser(argparse.ArgumentParser): # module calls kwargs.pop(), so can't call super first without # losing data. optional = {'options': [], 'metavar': kwargs.get('metavar', None)} - need_metavar = (optional['metavar'] is None and - kwargs.get('action') in (None, 'store')) + need_metavar = optional['metavar'] is None and kwargs.get( + 'action' + ) in ( + None, + 'store', + ) for arg in args: if not arg.startswith('-'): break @@ -164,8 +168,9 @@ class OebuildArgumentParser(argparse.ArgumentParser): # used. By convention, long options go last, so this # matches the default argparse behavior. if need_metavar: - optional['metavar'] = arg.lstrip('-').translate( - {ord('-'): '_'}).upper() + optional['metavar'] = ( + arg.lstrip('-').translate({ord('-'): '_'}).upper() + ) optional['help'] = kwargs.get('help') self.oebuild_optionals.append(optional) diff --git a/src/oebuild/ogit.py b/src/oebuild/ogit.py index b67447e..dbacc51 100644 --- a/src/oebuild/ogit.py +++ b/src/oebuild/ogit.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import os @@ -20,9 +20,9 @@ from oebuild.m_log import logger class OGit: - ''' + """ owner git to print progress in clone action - ''' + """ def __init__(self, repo_dir, remote_url, branch=None) -> None: self._repo_dir = repo_dir @@ -35,49 +35,51 @@ class OGit: @property def repo_dir(self): - ''' + """ return repo dir - ''' + """ return self._repo_dir @property def remote_url(self): - ''' + """ return remote url - ''' + """ return self._remote_url @property def branch(self): - ''' + """ return branch - ''' + """ return self.branch def check_out_version(self, version): - ''' + """ check out version - ''' + """ return self._fetch_upstream(version=version) def clone_or_pull_repo(self): - ''' + """ clone or pull git repo - ''' + """ self._fetch_upstream() def _fetch_upstream(self, version=None): repo = Repo.init(self._repo_dir) remote = None for item in repo.remotes: - if self._remote_url.rstrip(".git") == item.url.rstrip(".git"): + if self._remote_url.rstrip('.git') == item.url.rstrip('.git'): remote = item else: continue if remote is None: - remote_name = "upstream" - remote = git.Remote.add(repo=repo, name=remote_name, url=self._remote_url) - logger.info("Fetching into %s ...", self._repo_dir) + remote_name = 'upstream' + remote = git.Remote.add( + repo=repo, name=remote_name, url=self._remote_url + ) + logger.info('Fetching into %s ...', self._repo_dir) try: if version is None: remote.fetch(self._branch, progress=CustomRemote(), depth=1) @@ -87,10 +89,10 @@ class OGit: try: remote.fetch(version, progress=CustomRemote(), depth=1) except GitCommandError: - logger.error("fetch failed") + logger.error('fetch failed') return False except GitCommandError: - logger.error("fetch failed") + logger.error('fetch failed') return False try: @@ -99,39 +101,39 @@ class OGit: else: repo.git.checkout(version) except GitCommandError: - logger.error("update faild") + logger.error('update faild') return False - logger.info("Fetching into %s successful\n", self._repo_dir) + logger.info('Fetching into %s successful\n', self._repo_dir) return True @staticmethod def get_repo_info(repo_dir: str): - ''' + """ return git repo info: remote_url, branch - ''' + """ try: repo = Repo(repo_dir) remote_url = repo.remote().url branch = repo.active_branch.name return remote_url, branch except TypeError: - return "", '' + return '', '' except git.GitError: - return "", "" + return '', '' except ValueError: - return "", "" + return '', '' class CustomRemote(git.RemoteProgress): - ''' + """ Rewrote RemoteProgress to show the process of code updates - ''' + """ def update(self, op_code, cur_count, max_count=None, message=''): - ''' + """ rewrote update function - ''' - end_str = "\r" + """ + end_str = '\r' if op_code & 2 == RemoteProgress.END: - end_str = "\r\n" + end_str = '\r\n' print(self._cur_line, end=end_str) diff --git a/src/oebuild/parse_env.py b/src/oebuild/parse_env.py index f6bd60f..cac6db3 100644 --- a/src/oebuild/parse_env.py +++ b/src/oebuild/parse_env.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" from typing import Optional from dataclasses import dataclass @@ -21,88 +21,90 @@ from oebuild.m_log import logger @dataclass class EnvContainer: - ''' + """ the container object in env object - ''' + """ + short_id: Optional[str] @dataclass class Env: - ''' + """ the env object - ''' + """ + container: Optional[EnvContainer] class ParseEnv: - ''' + """ This class is used to parse env.yaml and update env.yaml - ''' + """ def __init__(self, env_dir): - self.env_dir = pathlib.Path(env_dir) if isinstance(env_dir, str) else env_dir + self.env_dir = ( + pathlib.Path(env_dir) if isinstance(env_dir, str) else env_dir + ) self.env: Env = Env(container=None) self._parse_env() @property def container(self): - ''' + """ return container object - ''' + """ return self.env.container def _parse_env(self): - ''' + """ parse env.yaml to env object - ''' + """ data = oebuild_util.read_yaml(self.env_dir) if data is None: return - if "container" in data: + if 'container' in data: env_container = data['container'] try: self.env.container = EnvContainer( short_id=env_container['short_id'] ) except KeyError: - logger.error(".env parse error, please check it") + logger.error('.env parse error, please check it') sys.exit(-1) def set_env_container(self, env_container: EnvContainer): - ''' + """ set ParseEnv's container object - ''' + """ self.env.container = env_container def export_env(self): - ''' + """ export env object to env.yaml - ''' + """ data = {} if self.env.container is not None: container = self.env.container - data['container'] = { - 'short_id': container.short_id - } + data['container'] = {'short_id': container.short_id} oebuild_util.write_yaml(pathlib.Path(self.env_dir), data=data) @staticmethod def check_env_container(env_container): - ''' + """ Check that the env.yaml content is compliant - ''' - if "remote" not in env_container: - raise ValueError("the key remote is lack") + """ + if 'remote' not in env_container: + raise ValueError('the key remote is lack') - if "branch" not in env_container: - raise ValueError("the key branch is lack") + if 'branch' not in env_container: + raise ValueError('the key branch is lack') - if "short_id" not in env_container: - raise ValueError("the key short_id is lack") + if 'short_id' not in env_container: + raise ValueError('the key short_id is lack') - if "volumns" not in env_container: - raise ValueError("the key volumns is lack") + if 'volumns' not in env_container: + raise ValueError('the key volumns is lack') diff --git a/src/oebuild/parse_param.py b/src/oebuild/parse_param.py index de4ece7..8451774 100644 --- a/src/oebuild/parse_param.py +++ b/src/oebuild/parse_param.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" from typing import Dict @@ -18,67 +18,69 @@ import oebuild.const as oebuild_const class ParseRepoParam: - ''' + """ RepoParam: remote_url: str version: str - ''' + """ + @staticmethod def parse_to_obj(repo_param_dict: Dict[str, str]) -> RepoParam: - ''' + """ parse dict to RepoParam - ''' + """ return RepoParam( remote_url=repo_param_dict['remote_url'], - version=repo_param_dict['version'] + version=repo_param_dict['version'], ) @staticmethod def parse_to_dict(repo_param_obj: RepoParam) -> Dict[str, str]: - ''' + """ parse RepoParam to dict - ''' + """ return { - "remote_url": repo_param_obj.remote_url, - "version": repo_param_obj.version + 'remote_url': repo_param_obj.remote_url, + 'version': repo_param_obj.version, } class ParseDockerParam: - ''' + """ class DockerParam: image: str parameters: str volumns: list command: str - ''' + """ + @staticmethod def parse_to_obj(docker_param_dict: Dict) -> DockerParam: - ''' + """ parse dict to DockerParam - ''' + """ return DockerParam( image=docker_param_dict['image'], parameters=docker_param_dict['parameters'], volumns=docker_param_dict['volumns'], - command=docker_param_dict['command'] + command=docker_param_dict['command'], ) @staticmethod def parse_to_dict(docker_param_obj: DockerParam): - ''' + """ parse dict to DockerParam - ''' + """ return { 'image': docker_param_obj.image, 'parameters': docker_param_obj.parameters, 'volumns': docker_param_obj.volumns, - 'command': docker_param_obj.command + 'command': docker_param_obj.command, } class ParseCompileParam: - ''' + """ CompileParam: build_in: str machine: str @@ -99,23 +101,29 @@ class ParseCompileParam: nativesdk_dir: Optional[str] bitbake_cmds: Optional[list] - ''' + """ + @staticmethod def parse_to_obj(compile_param_dict) -> CompileParam: - ''' + """ The initialization operation is used to parse the compile.yaml file and perform a series of checks before parsing - ''' + """ docker_param: DockerParam = None - if "docker_param" in compile_param_dict and compile_param_dict['docker_param'] is not None: + if ( + 'docker_param' in compile_param_dict + and compile_param_dict['docker_param'] is not None + ): docker_param = ParseDockerParam.parse_to_obj( docker_param_dict=compile_param_dict['docker_param'] ) # for old version repos = [] - if "repos" in compile_param_dict: - repos = oebuild_util.trans_dict_key_to_list(compile_param_dict['repos']) + if 'repos' in compile_param_dict: + repos = oebuild_util.trans_dict_key_to_list( + compile_param_dict['repos'] + ) def get_value_from_dict(key, dictobj, default_value=None): if key not in dictobj: @@ -123,30 +131,49 @@ class ParseCompileParam: return dictobj[key] return CompileParam( - build_in=get_value_from_dict('build_in', - compile_param_dict, - oebuild_const.BUILD_IN_DOCKER), + build_in=get_value_from_dict( + 'build_in', compile_param_dict, oebuild_const.BUILD_IN_DOCKER + ), machine=get_value_from_dict('machine', compile_param_dict, None), - toolchain_type=get_value_from_dict('toolchain_type', compile_param_dict, None), + toolchain_type=get_value_from_dict( + 'toolchain_type', compile_param_dict, None + ), no_layer=get_value_from_dict('no_layer', compile_param_dict, None), - toolchain_dir=get_value_from_dict('toolchain_dir', compile_param_dict, None), - llvm_toolchain_dir=get_value_from_dict('llvm_toolchain_dir', compile_param_dict, None), - nativesdk_dir=get_value_from_dict('nativesdk_dir', compile_param_dict, None), - sstate_mirrors=get_value_from_dict('sstate_mirrors', compile_param_dict, None), - sstate_dir=get_value_from_dict('sstate_dir', compile_param_dict, None), + toolchain_dir=get_value_from_dict( + 'toolchain_dir', compile_param_dict, None + ), + llvm_toolchain_dir=get_value_from_dict( + 'llvm_toolchain_dir', compile_param_dict, None + ), + nativesdk_dir=get_value_from_dict( + 'nativesdk_dir', compile_param_dict, None + ), + sstate_mirrors=get_value_from_dict( + 'sstate_mirrors', compile_param_dict, None + ), + sstate_dir=get_value_from_dict( + 'sstate_dir', compile_param_dict, None + ), tmp_dir=get_value_from_dict('tmp_dir', compile_param_dict, None), - cache_src_dir=get_value_from_dict('cache_src_dir', compile_param_dict, None), + cache_src_dir=get_value_from_dict( + 'cache_src_dir', compile_param_dict, None + ), repos=None if len(repos) == 0 else repos, - local_conf=get_value_from_dict('local_conf', compile_param_dict, None), + local_conf=get_value_from_dict( + 'local_conf', compile_param_dict, None + ), layers=get_value_from_dict('layers', compile_param_dict, None), docker_param=docker_param, - bitbake_cmds=get_value_from_dict('bitbake_cmds', compile_param_dict, None)) + bitbake_cmds=get_value_from_dict( + 'bitbake_cmds', compile_param_dict, None + ), + ) @staticmethod def parse_to_dict(compile_param: CompileParam): - ''' + """ parse CompileParam obj to dict - ''' + """ compile_obj = {} compile_obj['build_in'] = compile_param.build_in compile_obj['machine'] = compile_param.machine @@ -158,7 +185,9 @@ class ParseCompileParam: if compile_param.toolchain_dir is not None: compile_obj['toolchain_dir'] = compile_param.toolchain_dir if compile_param.llvm_toolchain_dir is not None: - compile_obj['llvm_toolchain_dir'] = compile_param.llvm_toolchain_dir + compile_obj['llvm_toolchain_dir'] = ( + compile_param.llvm_toolchain_dir + ) if compile_param.nativesdk_dir is not None: compile_obj['nativesdk_dir'] = compile_param.nativesdk_dir if compile_param.sstate_mirrors is not None: @@ -173,7 +202,8 @@ class ParseCompileParam: compile_obj['layers'] = compile_param.layers if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: compile_obj['docker_param'] = ParseDockerParam().parse_to_dict( - compile_param.docker_param) + compile_param.docker_param + ) if compile_param.bitbake_cmds is not None: compile_obj['bitbake_cmds'] = compile_param.bitbake_cmds @@ -181,34 +211,41 @@ class ParseCompileParam: class ParseToolchainParam: - ''' + """ ToolchainParam: config_list: Optional[list] docker_param: DockerParam - ''' + """ + @staticmethod def parse_to_obj(toolchain_param_dict) -> ToolchainParam: - ''' + """ parse dict to RepoParam - ''' + """ return ToolchainParam( kind=toolchain_param_dict['kind'], - gcc_configs=None if 'gcc_configs' not in toolchain_param_dict + gcc_configs=None + if 'gcc_configs' not in toolchain_param_dict else toolchain_param_dict['gcc_configs'], - llvm_lib=None if 'llvm_lib' not in toolchain_param_dict + llvm_lib=None + if 'llvm_lib' not in toolchain_param_dict else toolchain_param_dict['llvm_lib'], - docker_param=ParseDockerParam.parse_to_obj(toolchain_param_dict['docker_param']) + docker_param=ParseDockerParam.parse_to_obj( + toolchain_param_dict['docker_param'] + ), ) @staticmethod def parse_to_dict(toolchain_param_obj: ToolchainParam): - ''' + """ parse ToolchainParam to dict - ''' - res = {"kind": toolchain_param_obj.kind} + """ + res = {'kind': toolchain_param_obj.kind} if toolchain_param_obj.kind == oebuild_const.GCC_TOOLCHAIN: res['gcc_configs'] = toolchain_param_obj.gcc_configs else: res['llvm_lib'] = toolchain_param_obj.llvm_lib - res['docker_param'] = ParseDockerParam.parse_to_dict(toolchain_param_obj.docker_param) + res['docker_param'] = ParseDockerParam.parse_to_dict( + toolchain_param_obj.docker_param + ) return res diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 8afc6f4..29f8ebb 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,8 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" + import logging import sys from dataclasses import dataclass @@ -24,9 +25,10 @@ import oebuild.const as oebuild_const @dataclass class Template: - ''' + """ basic template for paltform and feature - ''' + """ + repos: Optional[list] layers: Optional[list] @@ -36,9 +38,10 @@ class Template: @dataclass class PlatformTemplate(Template): - ''' + """ the object will be parsed by platform config - ''' + """ + machine: LiteralScalarString toolchain_type: LiteralScalarString @@ -46,48 +49,49 @@ class PlatformTemplate(Template): @dataclass class FeatureTemplate(Template): - ''' + """ the object will be parsed by feature config - ''' + """ + feature_name: LiteralScalarString support: list class BaseParseTemplate(ValueError): - ''' + """ basic error about parse_template - ''' + """ class ConfigPathNotExists(BaseParseTemplate): - ''' + """ config path not exists - ''' + """ class PlatformNotAdd(BaseParseTemplate): - ''' + """ platform not add first - ''' + """ class FeatureNotSupport(BaseParseTemplate): - ''' + """ feature not support - ''' + """ class CommonNotFound(BaseParseTemplate): - ''' + """ common param not found - ''' + """ class ParseTemplate: - ''' + """ ParseTemplate is to add platform template and feature template and export compile.yaml finially - ''' + """ def __init__(self, yocto_dir: str): self.yocto_dir = yocto_dir @@ -97,13 +101,13 @@ class ParseTemplate: self.platform = None def add_template(self, config_dir): - ''' + """ this method is to add Template, note: the template has two type for board and application, and the deal is difference, when adding board template it will set board_template as the board_template unset, or replace as the board_template had setted. but feature_template will append to feature_template anywhere, the feature_template adding must after board_templiate, else throw exception - ''' + """ if not isinstance(config_dir, pathlib.Path): config_dir = pathlib.Path(config_dir) if not os.path.exists(config_dir): @@ -116,71 +120,88 @@ class ParseTemplate: repo_list = self.parse_repos_list(data['repos']) layers = None if 'layers' not in data else data['layers'] - local_conf = None if 'local_conf' not in data else data['local_conf'] + local_conf = ( + None if 'local_conf' not in data else data['local_conf'] + ) config_type = data['type'] config_name = os.path.basename(config_dir) if config_type == oebuild_const.PLATFORM: if self.platform is None: - self.platform = os.path.splitext(config_name)[0].strip("\n") + self.platform = os.path.splitext(config_name)[0].strip( + '\n' + ) else: - logging.error("Only one platform is allowed") + logging.error('Only one platform is allowed') sys.exit(-1) self.platform_template = PlatformTemplate( machine=data['machine'], toolchain_type=data['toolchain_type'], repos=repo_list, - local_conf=None if local_conf is None else LiteralScalarString(local_conf), - layers=None if layers is None else layers) + local_conf=None + if local_conf is None + else LiteralScalarString(local_conf), + layers=None if layers is None else layers, + ) return if self.platform_template is None: raise PlatformNotAdd('please add platform template first') if self.platform is None: - logging.error("Platform not specified") + logging.error('Platform not specified') sys.exit(-1) support_arch = [] if 'support' in data: support_arch = data['support'].split('|') if self.platform not in support_arch: - raise FeatureNotSupport(f'your arch is {self.platform},' - ' the feature is not supported,' - 'please check your application ' - 'support archs') - - self.feature_template.append(FeatureTemplate( - feature_name=LiteralScalarString(os.path.splitext(config_name)[0]), - repos=repo_list, - support=support_arch, - local_conf=None if local_conf is None else LiteralScalarString(local_conf), - layers=None if layers is None else layers - )) + raise FeatureNotSupport( + f'your arch is {self.platform},' + ' the feature is not supported,' + 'please check your application ' + 'support archs' + ) + + self.feature_template.append( + FeatureTemplate( + feature_name=LiteralScalarString( + os.path.splitext(config_name)[0] + ), + repos=repo_list, + support=support_arch, + local_conf=None + if local_conf is None + else LiteralScalarString(local_conf), + layers=None if layers is None else layers, + ) + ) except Exception as e_p: raise e_p - def get_default_generate_compile_conf_param(self,): - ''' + def get_default_generate_compile_conf_param( + self, + ): + """ return default generate_compile_conf param - ''' + """ return { - "nativesdk_dir": None, - "toolchain_dir": None, - "build_in": oebuild_const.BUILD_IN_DOCKER, - "sstate_mirrors": None, - "tmp_dir": None, - "datetime": None, - "is_disable_fetch": False, - "docker_image": None, - "src_dir": None, - "compile_dir": None + 'nativesdk_dir': None, + 'toolchain_dir': None, + 'build_in': oebuild_const.BUILD_IN_DOCKER, + 'sstate_mirrors': None, + 'tmp_dir': None, + 'datetime': None, + 'is_disable_fetch': False, + 'docker_image': None, + 'src_dir': None, + 'compile_dir': None, } def generate_compile_conf(self, param): - ''' + """ param obj: nativesdk_dir=None, toolchain_dir=None, @@ -196,46 +217,68 @@ class ParseTemplate: src_dir: str = None, compile_dir: str = None, cache_src_dir: str = None - ''' + """ # first param common yaml if self.platform_template is None: raise PlatformNotAdd('please set platform template first') - common_yaml_path = os.path.join(self.yocto_dir, '.oebuild', 'common.yaml') - repos, layers, local_conf = parse_repos_layers_local_obj(common_yaml_path) + common_yaml_path = os.path.join( + self.yocto_dir, '.oebuild', 'common.yaml' + ) + repos, layers, local_conf = parse_repos_layers_local_obj( + common_yaml_path + ) if self.platform_template.repos is not None: - repos.extend(oebuild_util.trans_dict_key_to_list(self.platform_template.repos)) + repos.extend( + oebuild_util.trans_dict_key_to_list( + self.platform_template.repos + ) + ) if self.platform_template.layers is not None: layers.extend(self.platform_template.layers) if self.platform_template.local_conf is not None: - local_conf = LiteralScalarString(self.platform_template.local_conf + local_conf) + local_conf = LiteralScalarString( + self.platform_template.local_conf + local_conf + ) for feature in self.feature_template: feature: FeatureTemplate = feature if feature.repos is not None: - repos.extend(oebuild_util.trans_dict_key_to_list(feature.repos)) + repos.extend( + oebuild_util.trans_dict_key_to_list(feature.repos) + ) if feature.layers is not None: layers.extend(feature.layers) if feature.local_conf is not None: - local_conf = LiteralScalarString(feature.local_conf + '\n' + local_conf) + local_conf = LiteralScalarString( + feature.local_conf + '\n' + local_conf + ) if param['datetime'] is not None: - datetime_str = LiteralScalarString(f'DATETIME = "{param["datetime"]}"') + datetime_str = LiteralScalarString( + f'DATETIME = "{param["datetime"]}"' + ) local_conf = LiteralScalarString(local_conf + '\n' + datetime_str) if param['no_fetch']: - disable_fetch_str = LiteralScalarString('OPENEULER_FETCH = "disable"') - local_conf = LiteralScalarString(local_conf + '\n' + disable_fetch_str) + disable_fetch_str = LiteralScalarString( + 'OPENEULER_FETCH = "disable"' + ) + local_conf = LiteralScalarString( + local_conf + '\n' + disable_fetch_str + ) compile_conf = {} compile_conf['build_in'] = param['build_in'] compile_conf['machine'] = self.platform_template.machine compile_conf['toolchain_type'] = self.platform_template.toolchain_type compile_conf['cache_src_dir'] = param['cache_src_dir'] - compile_conf = self._deal_non_essential_compile_conf_param(param, compile_conf) + compile_conf = self._deal_non_essential_compile_conf_param( + param, compile_conf + ) compile_conf['no_layer'] = param['no_layer'] compile_conf['repos'] = repos compile_conf['local_conf'] = local_conf @@ -247,14 +290,14 @@ class ParseTemplate: compile_conf['docker_param'] = get_docker_param_dict( docker_image=param['docker_image'], dir_list={ - "src_dir": param['src_dir'], - "cache_src_dir": param['cache_src_dir'], - "compile_dir": param['compile_dir'], - "toolchain_dir": param['toolchain_dir'], - "llvm_toolchain_dir": param['llvm_toolchain_dir'], - "sstate_mirrors": param['sstate_mirrors'], - "sstate_dir": param['sstate_dir'] - } + 'src_dir': param['src_dir'], + 'cache_src_dir': param['cache_src_dir'], + 'compile_dir': param['compile_dir'], + 'toolchain_dir': param['toolchain_dir'], + 'llvm_toolchain_dir': param['llvm_toolchain_dir'], + 'sstate_mirrors': param['sstate_mirrors'], + 'sstate_dir': param['sstate_dir'], + }, ) return compile_conf @@ -276,9 +319,9 @@ class ParseTemplate: @staticmethod def parse_repos_list(repos): - ''' + """ parse repo json object to OebuildRepo - ''' + """ if isinstance(repos, list): return repos repo_list = [] @@ -289,7 +332,7 @@ class ParseTemplate: def get_docker_param_dict(docker_image, dir_list): - ''' + """ transfer docker param to dict dir_list: src_dir @@ -298,48 +341,69 @@ def get_docker_param_dict(docker_image, dir_list): llvm_toolchain_dir sstate_mirrors sstate_dir - ''' + """ parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS docker_param = {} docker_param['image'] = docker_image docker_param['parameters'] = parameters docker_param['volumns'] = get_docker_volumns(dir_list) - docker_param['command'] = "bash" + docker_param['command'] = 'bash' return docker_param def get_docker_volumns(dir_list): - ''' + """ translate the dir_dict to volumns list - ''' + """ volumns = [] - volumns.append("/dev/net/tun:/dev/net/tun") + volumns.append('/dev/net/tun:/dev/net/tun') if 'src_dir' in dir_list and dir_list['src_dir'] is not None: volumns.append(dir_list['src_dir'] + ':' + oebuild_const.CONTAINER_SRC) if 'compile_dir' in dir_list and dir_list['compile_dir'] is not None: - volumns.append(dir_list['compile_dir'] + ":" + os.path.join( - oebuild_const.CONTAINER_BUILD, os.path.basename(dir_list['compile_dir']))) + volumns.append( + dir_list['compile_dir'] + + ':' + + os.path.join( + oebuild_const.CONTAINER_BUILD, + os.path.basename(dir_list['compile_dir']), + ) + ) if 'toolchain_dir' in dir_list and dir_list['toolchain_dir'] is not None: - volumns.append(dir_list['toolchain_dir'] + ":" + oebuild_const.NATIVE_GCC_MAP) - if 'llvm_toolchain_dir' in dir_list and dir_list['llvm_toolchain_dir'] is not None: - volumns.append(dir_list['llvm_toolchain_dir'] + ":" + oebuild_const.NATIVE_LLVM_MAP) + volumns.append( + dir_list['toolchain_dir'] + ':' + oebuild_const.NATIVE_GCC_MAP + ) + if ( + 'llvm_toolchain_dir' in dir_list + and dir_list['llvm_toolchain_dir'] is not None + ): + volumns.append( + dir_list['llvm_toolchain_dir'] + + ':' + + oebuild_const.NATIVE_LLVM_MAP + ) if 'sstate_mirrors' in dir_list and dir_list['sstate_mirrors'] is not None: - volumns.append(dir_list['sstate_mirrors'] + ":" + oebuild_const.SSTATE_MIRRORS) + volumns.append( + dir_list['sstate_mirrors'] + ':' + oebuild_const.SSTATE_MIRRORS + ) if 'sstate_dir' in dir_list and dir_list['sstate_dir'] is not None: - volumns.append(dir_list['sstate_dir'] + ":" + oebuild_const.SSTATE_DIR) + volumns.append(dir_list['sstate_dir'] + ':' + oebuild_const.SSTATE_DIR) if 'cache_src_dir' in dir_list and dir_list['cache_src_dir'] is not None: - volumns.append(dir_list['cache_src_dir'] + ":" + oebuild_const.CACHE_SRC_DIR_MAP) + volumns.append( + dir_list['cache_src_dir'] + ':' + oebuild_const.CACHE_SRC_DIR_MAP + ) return volumns def parse_repos_layers_local_obj(common_yaml_path): - ''' + """ parse from yaml to repos, layers and local - ''' + """ if not os.path.exists(common_yaml_path): - logging.error('can not find .oebuild/common.yaml in yocto-meta-openeuler') + logging.error( + 'can not find .oebuild/common.yaml in yocto-meta-openeuler' + ) sys.exit(-1) data = oebuild_util.read_yaml(common_yaml_path) diff --git a/src/oebuild/spec.py b/src/oebuild/spec.py index cc7d882..6a9e18a 100644 --- a/src/oebuild/spec.py +++ b/src/oebuild/spec.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import importlib.util import os @@ -17,9 +17,9 @@ import sys class CommandError(RuntimeError): - ''' + """ Indicates that a command failed. - ''' + """ def __init__(self, returncode=1): super().__init__() @@ -27,12 +27,12 @@ class CommandError(RuntimeError): class CommandContextError(CommandError): - '''Indicates that a context-dependent command could not be run.''' + """Indicates that a context-dependent command could not be run.""" class ExtensionCommandError(CommandError): - '''Exception class indicating an extension command was badly - defined and could not be created.''' + """Exception class indicating an extension command was badly + defined and could not be created.""" def __init__(self, **kwargs): self.hint = kwargs.pop('hint', None) @@ -41,7 +41,6 @@ class ExtensionCommandError(CommandError): @dataclass class _CmdFactory: - py_file: str name: str attr: str @@ -58,21 +57,23 @@ class _CmdFactory: mod = _commands_module_from_file(self.py_file, self.attr) except ImportError as i_e: raise ExtensionCommandError( - hint=f'could not import {self.py_file}') from i_e + hint=f'could not import {self.py_file}' + ) from i_e # Get the attribute which provides the OebuildCommand subclass. try: return getattr(mod, self.attr) except AttributeError as a_e: raise ExtensionCommandError( - hint=f'no attribute {self.attr} in {self.py_file}') from a_e + hint=f'no attribute {self.attr} in {self.py_file}' + ) from a_e @dataclass class OebuildExtCommandSpec: - ''' + """ An object which allows instantiating a oebuild extension. - ''' + """ # Command name, as known to the user name: str @@ -89,9 +90,10 @@ class OebuildExtCommandSpec: @dataclass class _ExtCommand: - ''' + """ record extern command basic info, it's useful when app initialize - ''' + """ + name: str class_name: str @@ -100,31 +102,34 @@ class _ExtCommand: def get_spec(pre_dir, command_ext: _ExtCommand): - ''' + """ xxx - ''' + """ - py_file = os.path.join( - os.path.dirname(__file__), pre_dir, - command_ext.path) if ('/.local/oebuild_plugins/' - not in command_ext.path) else command_ext.path - factory = _CmdFactory(py_file=py_file, - name=command_ext.name, - attr=command_ext.class_name) + py_file = ( + os.path.join(os.path.dirname(__file__), pre_dir, command_ext.path) + if ('/.local/oebuild_plugins/' not in command_ext.path) + else command_ext.path + ) + factory = _CmdFactory( + py_file=py_file, name=command_ext.name, attr=command_ext.class_name + ) - return OebuildExtCommandSpec(name=command_ext.name, - description=factory().description, - help=factory().help_msg, - factory=factory) + return OebuildExtCommandSpec( + name=command_ext.name, + description=factory().description, + help=factory().help_msg, + factory=factory, + ) def _commands_module_from_file(file, mod_name): - ''' + """ Python magic for importing a module containing oebuild extension commands. To avoid polluting the sys.modules key space, we put these modules in an (otherwise unpopulated) oebuild.commands.ext package. - ''' + """ spec = importlib.util.spec_from_file_location(mod_name, file) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) diff --git a/src/oebuild/struct.py b/src/oebuild/struct.py index 6d60ce4..8763fdd 100644 --- a/src/oebuild/struct.py +++ b/src/oebuild/struct.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,8 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" + from dataclasses import dataclass from typing import Optional from ruamel.yaml.scalarstring import LiteralScalarString @@ -16,22 +17,24 @@ from ruamel.yaml.scalarstring import LiteralScalarString @dataclass class RepoParam: - ''' + """ object repo is to record template repo info, repo struct is: repo_name: remote_url: str version: str object repo transfer string to struct to use it next easily - ''' + """ + remote_url: str version: str @dataclass class DockerParam: - ''' + """ DockerParam defines the various parameters required for container startup - ''' + """ + # point out the docker image image: str # point out the parameter for create container @@ -44,9 +47,10 @@ class DockerParam: @dataclass class CompileLocalParam: - ''' + """ this is for parse to local.conf - ''' + """ + sstate_mirrors: Optional[str] sstate_dir: Optional[str] tmp_dir: Optional[str] @@ -55,9 +59,10 @@ class CompileLocalParam: @dataclass class CompileParamComm1: - ''' + """ this is for common param to compile.yaml - ''' + """ + build_in: str machine: str toolchain_type: str @@ -66,9 +71,10 @@ class CompileParamComm1: @dataclass class CompileParamComm2: - ''' + """ this is for common param to compile.yaml - ''' + """ + repos: Optional[list] layers: Optional[list] local_conf: Optional[LiteralScalarString] @@ -77,9 +83,10 @@ class CompileParamComm2: @dataclass class CompileParamSDK: - ''' + """ this is parse for host environment - ''' + """ + # gcc toolchain dir toolchain_dir: Optional[str] # llvm toolchain dir @@ -89,28 +96,32 @@ class CompileParamSDK: @dataclass class CompileParamBitbakeCmds: - ''' + """ this is for autobuild, oebuild need bitbake_cmds to run bitbake cmd automatic - ''' + """ + bitbake_cmds: Optional[list] @dataclass -class CompileParam(CompileParamComm1, - CompileParamComm2, - CompileLocalParam, - CompileParamSDK, - CompileParamBitbakeCmds): - ''' +class CompileParam( + CompileParamComm1, + CompileParamComm2, + CompileLocalParam, + CompileParamSDK, + CompileParamBitbakeCmds, +): + """ Compile is the parsed object of compile.yaml and is used to manipulate the build file - ''' + """ @dataclass class ToolchainParam: - ''' + """ this param is for oebuld toolchain - ''' + """ + kind: str # gcc_configs is for gcc toolchain type,for example: config_aarch64 gcc_configs: Optional[list] diff --git a/src/oebuild/util.py b/src/oebuild/util.py index 163917b..ba217fb 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" import pathlib import os @@ -34,58 +34,61 @@ from oebuild.parse_param import ParseRepoParam from oebuild.parse_env import EnvContainer, ParseEnv -def get_nativesdk_environment(nativesdk_dir=oebuild_const.NATIVESDK_DIR, - container: Container = None): - ''' +def get_nativesdk_environment( + nativesdk_dir=oebuild_const.NATIVESDK_DIR, container: Container = None +): + """ return environment initialization shell, if nativesdk directory is not exists or can not find any initialization shell, raise error - ''' + """ if container is None: if not os.path.isdir(nativesdk_dir): - logger.error("the %s directory is not exists", nativesdk_dir) + logger.error('the %s directory is not exists', nativesdk_dir) sys.exit(1) # list items in nativesdk to find environment shell list_items = os.listdir(nativesdk_dir) for item in list_items: - ret = re.match("^(environment-setup-)", item) + ret = re.match('^(environment-setup-)', item) if ret is not None: abs_path = os.path.join(nativesdk_dir, item) if os.path.isfile(abs_path) and not os.path.islink(abs_path): return item else: - res = container.exec_run("ls -al", - user=oebuild_const.CONTAINER_USER, - workdir=nativesdk_dir) + res = container.exec_run( + 'ls -al', user=oebuild_const.CONTAINER_USER, workdir=nativesdk_dir + ) if res.exit_code != 0: logger.error( - "can not find any nativesdk environment initialization shell") + 'can not find any nativesdk environment initialization shell' + ) sys.exit(res.exit_code) - list_items = res.output.decode("utf-8").split("\n") + list_items = res.output.decode('utf-8').split('\n') for item in list_items: item: str = item # notice: the item is like format with # "drwxr-xr-x 3 openeuler openeuler 4096 Nov 8 08:10 ." # so we must get the last clip from split with space - item_split = item.split(" ") + item_split = item.split(' ') if len(item_split) <= 0: continue - ret = re.match("^(environment-setup-)", - item_split[len(item_split) - 1]) - if ret is not None and item_split[0].startswith("-"): + ret = re.match( + '^(environment-setup-)', item_split[len(item_split) - 1] + ) + if ret is not None and item_split[0].startswith('-'): return item_split[len(item_split) - 1] - logger.error("can not find any nativesdk environment initialization shell") + logger.error('can not find any nativesdk environment initialization shell') sys.exit(1) def read_yaml(yaml_path): - ''' + """ read yaml file and parse it to object - ''' + """ if isinstance(yaml_path, str): yaml_path = pathlib.Path(yaml_path) if not os.path.exists(yaml_path.absolute()): - raise ValueError(f"yaml_dir can not find in :{yaml_path.absolute()}") + raise ValueError(f'yaml_dir can not find in :{yaml_path.absolute()}') try: with open(yaml_path, 'r', encoding='utf-8') as r_f: @@ -97,9 +100,9 @@ def read_yaml(yaml_path): def write_yaml(yaml_path, data): - ''' + """ write data to yaml file - ''' + """ if isinstance(yaml_path, str): yaml_path = pathlib.Path(yaml_path) if not os.path.exists(yaml_path.absolute()): @@ -113,66 +116,70 @@ def write_yaml(yaml_path, data): def get_git_repo_name(remote_url: str): - ''' + """ return repo name - ''' - url = remote_url.replace(".git", "") + """ + url = remote_url.replace('.git', '') return os.path.basename(url) def add_git_suffix(remote: str): - ''' + """ add .git suffix to remote if needed - ''' - if remote.endswith(".git"): + """ + if remote.endswith('.git'): return remote - return remote + ".git" + return remote + '.git' def get_base_oebuild(): - ''' + """ return oebuild base dir - ''' + """ return os.path.abspath(os.path.dirname(__file__)) def get_config_yaml_dir(): - ''' + """ return config yaml dir - ''' - return os.path.join(get_base_oebuild(), 'app/conf', - oebuild_const.CONFIG_YAML) + """ + return os.path.join( + get_base_oebuild(), 'app/conf', oebuild_const.CONFIG_YAML + ) def get_plugins_yaml_path(): - ''' + """ return plugin yaml path - ''' - return os.path.join(get_base_oebuild(), 'app/conf', - oebuild_const.PLUGINS_YAML) + """ + return os.path.join( + get_base_oebuild(), 'app/conf', oebuild_const.PLUGINS_YAML + ) def get_compile_yaml_dir(): - ''' + """ return compile.yaml.sample yaml dir - ''' - return os.path.join(get_base_oebuild(), 'app/conf', - oebuild_const.COMPILE_YAML) + """ + return os.path.join( + get_base_oebuild(), 'app/conf', oebuild_const.COMPILE_YAML + ) def get_upgrade_yaml_dir(): - ''' + """ return upgrade yaml dir - ''' - return os.path.join(get_base_oebuild(), 'app/conf', - oebuild_const.UPGRADE_YAML) + """ + return os.path.join( + get_base_oebuild(), 'app/conf', oebuild_const.UPGRADE_YAML + ) def generate_random_str(randomlength=16): - ''' + """ generate a random string by length - ''' + """ random_str = '' base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789' length = len(base_str) - 1 @@ -182,72 +189,76 @@ def generate_random_str(randomlength=16): def get_time_stamp(): - ''' + """ get current timestamp - ''' + """ return time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) def get_oebuild_version(): - ''' + """ return oebuild version - ''' + """ return __version__ def check_docker(): - ''' + """ check docker had be installed or not - ''' + """ try: DockerProxy() except DockerException as exc: - raise ValueError(f''' + raise ValueError(f""" please install docker first, and run follow commands in root: 1, groupadd docker 2, usermod -a -G docker {getpass.getuser()} 3, systemctl daemon-reload && systemctl restart docker 4, chmod o+rw /var/run/docker.sock -''') from exc +""") from exc def get_instance(factory): - ''' + """ Instantiate a class - ''' + """ return factory()() def restore_bashrc_content(old_content): - ''' + """ restore .bashrc - ''' + """ new_content = '' for line in old_content.split('\n'): line: str = line - if line.endswith(oebuild_const.BASH_END_FLAG) or line.replace( - " ", '') == '': + if ( + line.endswith(oebuild_const.BASH_END_FLAG) + or line.replace(' ', '') == '' + ): continue new_content = new_content + line + '\n' return new_content def init_bashrc_content(old_content, init_command: list): - ''' + """ init bashrc - ''' + """ new_content = restore_bashrc_content(old_content=old_content) for command in init_command: - new_content = new_content + command + oebuild_const.BASH_END_FLAG + '\n' + new_content = ( + new_content + command + oebuild_const.BASH_END_FLAG + '\n' + ) return new_content def add_bashrc(content: str, line: str): - ''' + """ add command line to bashrc - ''' + """ if not content.endswith('\n'): content = content + '\n' content = content + line + oebuild_const.BASH_END_FLAG + '\n' @@ -256,34 +267,36 @@ def add_bashrc(content: str, line: str): def get_host_proxy(proxy_name): - ''' + """ get proxy information from host - ''' + """ host_proxy = {} if proxy_name is None: return host_proxy for name in proxy_name: command = "env | grep %s | awk -F'=' '{print$NF}'" % name - res = subprocess.run(command, - shell=True, - capture_output=True, - encoding="utf-8", - check=False) + res = subprocess.run( + command, + shell=True, + capture_output=True, + encoding='utf-8', + check=False, + ) if res.returncode != 0: - logger.error("get proxy variable failed") + logger.error('get proxy variable failed') sys.exit(res.returncode) value = res.stdout.strip() - if value != "": + if value != '': host_proxy[name] = value return host_proxy def download_repo_from_manifest(repo_list, src_dir, manifest_path): - ''' + """ Download the repos set in compile.yaml based on the given base path - ''' + """ if repo_list is None or len(repo_list) == 0: return manifest = None @@ -293,87 +306,99 @@ def download_repo_from_manifest(repo_list, src_dir, manifest_path): repo_dir = os.path.join(src_dir, repo_name) if repo_name in manifest: print(repo_name) - repo_obj: RepoParam = ParseRepoParam.parse_to_obj(manifest[repo_name]) - repo_git = OGit(repo_dir=repo_dir, remote_url=repo_obj.remote_url, branch=None) + repo_obj: RepoParam = ParseRepoParam.parse_to_obj( + manifest[repo_name] + ) + repo_git = OGit( + repo_dir=repo_dir, remote_url=repo_obj.remote_url, branch=None + ) repo_git.check_out_version(version=repo_obj.version) def sync_repo_from_cache(repo_list, src_dir, cache_src_dir): - ''' + """ sync repos from src cache if need - ''' + """ for repo_name in repo_list: - if os.path.exists(f"{src_dir}/{repo_name}"): + if os.path.exists(f'{src_dir}/{repo_name}'): continue - subprocess.run(f"mkdir -p {src_dir}/{repo_name}", - shell=True, - check=True) - subprocess.run(f"rsync -a {cache_src_dir}/{repo_name}/ {src_dir}/{repo_name}/", - shell=True, - check=True) + subprocess.run( + f'mkdir -p {src_dir}/{repo_name}', shell=True, check=True + ) + subprocess.run( + f'rsync -a {cache_src_dir}/{repo_name}/ {src_dir}/{repo_name}/', + shell=True, + check=True, + ) def get_docker_image_from_yocto(yocto_dir): - ''' + """ determine yocto-meta-openeuler's supported docker image in env.yaml - ''' + """ if not os.path.exists(yocto_dir): - raise ValueError("the yocto direction is not exists") + raise ValueError('the yocto direction is not exists') - env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") + env_path = os.path.join(yocto_dir, '.oebuild/env.yaml') if not os.path.exists(env_path): return None env_parse = read_yaml(yaml_path=env_path) - if "docker_image" in env_parse: + if 'docker_image' in env_parse: return str(env_parse['docker_image']) return None def get_sdk_docker_image_from_yocto(yocto_dir): - ''' + """ determine yocto-meta-openeuler's supported sdk docker image in env.yaml - ''' + """ if not os.path.exists(yocto_dir): - raise ValueError("the yocto direction is not exists") + raise ValueError('the yocto direction is not exists') - env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") + env_path = os.path.join(yocto_dir, '.oebuild/env.yaml') if not os.path.exists(env_path): return None env_parse = read_yaml(yaml_path=env_path) - if "sdk_docker_image" in env_parse: + if 'sdk_docker_image' in env_parse: return str(env_parse['sdk_docker_image']) return None def deal_env_container(env: ParseEnv, docker_param: DockerParam): - ''' + """ This operation realizes the processing of the container, controls how the container is processed by parsing the env variable, if the container does not exist, or the original environment and the current environment that needs to be set are inconsistent, you need to create a new container, otherwise directly enable the sleeping container - ''' + """ + def check_container_img_eq(container_id, docker_image): c_mid = docker_proxy.get_container_img_id(container_id=container_id) d_mid = docker_proxy.get_image(docker_image).id return c_mid == d_mid docker_proxy = DockerProxy() - if env.container is None \ - or env.container.short_id is None \ - or not docker_proxy.is_container_exists(env.container.short_id) \ - or not check_container_img_eq(env.container.short_id, docker_param.image): + if ( + env.container is None + or env.container.short_id is None + or not docker_proxy.is_container_exists(env.container.short_id) + or not check_container_img_eq( + env.container.short_id, docker_param.image + ) + ): # judge which container container: Container = docker_proxy.create_container( image=docker_param.image, parameters=docker_param.parameters, volumes=docker_param.volumns, - command=docker_param.command) + command=docker_param.command, + ) env_container = EnvContainer(container.short_id) env.set_env_container(env_container) @@ -387,9 +412,9 @@ def deal_env_container(env: ParseEnv, docker_param: DockerParam): def trans_dict_key_to_list(obj): - ''' + """ transfer dict key to list - ''' + """ res = [] for key in obj: res.append(key) @@ -398,12 +423,12 @@ def trans_dict_key_to_list(obj): @contextmanager def suppress_print(): - ''' + """ make standand stdout to null, so the output can be none - ''' + """ original_stdout = sys.stdout try: - with open('/dev/null', 'w', encoding="utf-8") as f: + with open('/dev/null', 'w', encoding='utf-8') as f: sys.stdout = f yield finally: diff --git a/src/oebuild/version.py b/src/oebuild/version.py index 0d68656..11f5a73 100644 --- a/src/oebuild/version.py +++ b/src/oebuild/version.py @@ -1,4 +1,4 @@ -''' +""" Copyright (c) 2023 openEuler Embedded oebuild is licensed under Mulan PSL v2. You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -8,7 +8,7 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. -''' +""" # release log: # 1, Mugentest is an open-source Linux-OS testing framework developed by openEuler. -- Gitee From 897c7f33930cd3557c3bed696d09897412f47ec7 Mon Sep 17 00:00:00 2001 From: Egg12138 Date: Mon, 24 Nov 2025 20:56:25 +0800 Subject: [PATCH 2/3] format-fix: fix ambigous codes and bad manner deal with: - ambigous imports - outdated,dangerous string format - dead codes, unsued variables add: - tool.flake8 in pyproject.toml (Flake8-pyproject pypi package) Signed-off-by: egg12138 --- pyproject.toml | 4 + .../app/plugins/deploy/target_dev/deploy.py | 170 +++++++----------- src/oebuild/app/plugins/init/init.py | 2 +- src/oebuild/app/plugins/m_env/m_env.py | 73 ++++---- .../app/plugins/mugentest/mugentest.py | 12 +- src/oebuild/auto_completion.py | 7 +- src/oebuild/bb/utils.py | 151 ++++++++-------- src/oebuild/const.py | 5 +- src/oebuild/oebuild_parser.py | 6 +- src/oebuild/util.py | 2 +- 10 files changed, 210 insertions(+), 222 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9352d09..61b05fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,3 +26,7 @@ select = [ ignore = [ "E402", # module level import not at top of file ] + +# need pip install Flake8-pyproject +[tool.flake8] +max-line-length = 79 diff --git a/src/oebuild/app/plugins/deploy/target_dev/deploy.py b/src/oebuild/app/plugins/deploy/target_dev/deploy.py index ccc407d..1f58c9a 100644 --- a/src/oebuild/app/plugins/deploy/target_dev/deploy.py +++ b/src/oebuild/app/plugins/deploy/target_dev/deploy.py @@ -7,28 +7,30 @@ """Devtool plugin containing the deploy subcommands""" import logging +import math import os import shutil import subprocess import tempfile -import argparse_oe -import oe.types -from devtool import DevtoolError, exec_fakeroot, setup_tinfoil +import argparse_oe # pylint: disable=import-error +import oe.types # pylint: disable=import-error +import oe_package # pylint: disable=import-error +from devtool import DevtoolError, exec_fakeroot, setup_tinfoil # pylint: disable=import-error logger = logging.getLogger('devtool') DEPLOYLIST_PATH = '/.devtool' -def _prepare_remote_script( - deploy, +def _prepare_remote_script( # pylint: disable=too-many-positional-arguments, too-many-arguments + should_deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False, -): +): # pylint: disable=too-many-branches, too-many-statements """ Prepare a shell script for running on the target to deploy/undeploy files. We have to be careful what we put in this @@ -41,14 +43,14 @@ def _prepare_remote_script( lines.append('set -e') if undeployall: # Yes, I know this is crude - but it does work - lines.append('for entry in %s/*.list; do' % DEPLOYLIST_PATH) + lines.append(f'for entry in {DEPLOYLIST_PATH}/*.list; do') lines.append('[ ! -f $entry ] && exit') lines.append('set `basename $entry | sed "s/.list//"`') if dryrun: - if not deploy: + if not should_deploy: lines.append('echo "Previously deployed files for $1:"') - lines.append('manifest="%s/$1.list"' % DEPLOYLIST_PATH) - lines.append('preservedir="%s/$1.preserve"' % DEPLOYLIST_PATH) + lines.append(f'manifest="{DEPLOYLIST_PATH}/$1.list"') + lines.append(f'preservedir="{DEPLOYLIST_PATH}/$1.preserve"') lines.append('if [ -f $manifest ] ; then') # Read manifest in reverse and delete files / remove empty dirs lines.append(" sed '1!G;h;$!d' $manifest | while read file") @@ -59,7 +61,6 @@ def _prepare_remote_script( lines.append(' fi') else: lines.append(' if [ -d $file ] ; then') - # Avoid deleting a preserved directory in case it has special perms lines.append(' if [ ! -d $preservedir/$file ] ; then') lines.append(' rmdir $file > /dev/null 2>&1 || true') lines.append(' fi') @@ -69,15 +70,15 @@ def _prepare_remote_script( lines.append(' done') if not dryrun: lines.append(' rm $manifest') - if not deploy and not dryrun: + if not should_deploy and not dryrun: # May as well remove all traces lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true') lines.append('fi') - if deploy: + if should_deploy: if not nocheckspace: # Check for available space - # FIXME This doesn't take into account files spread across multiple + # NOTE: This doesn't take into account files spread across multiple # partitions, but doing that is non-trivial # Find the part of the destination path that exists lines.append('checkpath="$2"') @@ -147,13 +148,10 @@ def _prepare_remote_script( return '\n'.join(lines) -def deploy(args, config, basepath, workspace): +def deploy( + args, config, basepath, workspace +): # pylint: disable=unused-argument, too-many-statements, too-many-branches """Entry point for the devtool 'deploy' subcommand""" - import math - - import oe.package - import oe.recipeutils - # check_workspace_recipe(workspace, args.recipename, checksrc=False) try: @@ -171,14 +169,14 @@ def deploy(args, config, basepath, workspace): rd = tinfoil.parse_recipe(args.recipename) except Exception as e: raise DevtoolError( - 'Exception parsing recipe %s: %s' % (args.recipename, e) - ) + f'Exception parsing recipe {args.recipename}: {e}' + ) from e recipe_outdir = rd.getVar('D') if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): raise DevtoolError( - 'No files to deploy - have you built the %s ' + f'No files to deploy - have you built the {args.recipename} ' 'recipe? If so, the install step has not installed ' - 'any files.' % args.recipename + 'any files.' ) if args.strip and not args.dry_run: @@ -188,16 +186,16 @@ def deploy(args, config, basepath, workspace): rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped' ) if os.path.isdir(recipe_outdir): - exec_fakeroot(rd, 'rm -rf %s' % recipe_outdir, shell=True) + exec_fakeroot(rd, f'rm -rf {recipe_outdir}', shell=True) exec_fakeroot( rd, - 'cp -af %s %s' % (os.path.join(srcdir, '.'), recipe_outdir), + f'cp -af {os.path.join(srcdir, ".")} {recipe_outdir}', shell=True, ) os.environ['PATH'] = ':'.join( [os.environ['PATH'], rd.getVar('PATH') or ''] ) - oe.package.strip_execs( + oe_package.strip_execs( args.recipename, recipe_outdir, rd.getVar('STRIP'), @@ -229,34 +227,31 @@ def deploy(args, config, basepath, workspace): if args.dry_run: print( - 'Files to be deployed for %s on target %s:' - % (args.recipename, args.target) + f'Files to be deployed for {args.recipename} on target {args.target}:' ) for item, _ in filelist: - print(' %s' % item) + print(f' {item}') return 0 - extraoptions = '' + ssh_opts = {'extraoptions': '', 'scp_sshexec': '', 'ssh_sshexec': 'ssh', + 'scp_port': '', 'ssh_port': ''} if args.no_host_check: - extraoptions += ( + ssh_opts['extraoptions'] += ( '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' ) if not args.show_status: - extraoptions += ' -q' + ssh_opts['extraoptions'] += ' -q' - scp_sshexec = '' - ssh_sshexec = 'ssh' if args.ssh_exec: - scp_sshexec = '-S %s' % args.ssh_exec - ssh_sshexec = args.ssh_exec - scp_port = '' - ssh_port = '' + ssh_opts['scp_sshexec'] = f'-S {args.ssh_exec}' + ssh_opts['ssh_sshexec'] = args.ssh_exec + if args.port: - scp_port = '-P %s' % args.port - ssh_port = '-p %s' % args.port + ssh_opts['scp_port'] = f'-P {args.port}' + ssh_opts['ssh_port'] = f'-p {args.port}' if args.key: - extraoptions += ' -i %s' % args.key + ssh_opts['extraoptions'] += f' -i {args.key}' # In order to delete previously deployed files and have the manifest file on # the target, we write out a shell script and then copy it to the target @@ -269,40 +264,34 @@ def deploy(args, config, basepath, workspace): os.path.dirname(tmpscript), 'devtool_deploy.list' ) shellscript = _prepare_remote_script( - deploy=True, + should_deploy=True, verbose=args.show_status, nopreserve=args.no_preserve, nocheckspace=args.no_check_space, ) # Write out the script to a file with open( - os.path.join(tmpdir, os.path.basename(tmpscript)), 'w' + os.path.join(tmpdir, os.path.basename(tmpscript)), 'w', encoding='utf-8' ) as f: f.write(shellscript) # Write out the file list with open( - os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w' + os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w', encoding='utf-8' ) as f: - f.write('%d\n' % ftotalsize) + f.write(f'{ftotalsize}\n') for fpath, fsize in filelist: - f.write('%s %d\n' % (fpath, fsize)) + f.write(f'{fpath} {fsize}\n') # Copy them to the target - ret = subprocess.call( - 'scp %s %s %s %s/* %s:%s' - % ( - scp_sshexec, - scp_port, - extraoptions, - tmpdir, - args.target, - os.path.dirname(tmpscript), - ), - shell=True, + cmd = ( + f'scp {ssh_opts["scp_sshexec"]} {ssh_opts["scp_port"]} ' + f'{ssh_opts["extraoptions"]} {tmpdir}/* ' + f'{args.target}:{os.path.dirname(tmpscript)}' ) + ret = subprocess.call(cmd, shell=True) if ret != 0: raise DevtoolError( - 'Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target + f'Failed to copy script to {args.target} - rerun with -s to ' + 'get a complete error message' ) finally: shutil.rmtree(tmpdir) @@ -310,17 +299,9 @@ def deploy(args, config, basepath, workspace): # Now run the script ret = exec_fakeroot( rd, - "tar cf - . | %s %s %s %s 'sh %s %s %s %s'" - % ( - ssh_sshexec, - ssh_port, - extraoptions, - args.target, - tmpscript, - args.recipename, - destdir, - tmpfilelist, - ), + f"tar cf - . | {ssh_opts['ssh_sshexec']} {ssh_opts['ssh_port']} " + f"{ssh_opts['extraoptions']} {args.target} " + f"'sh {tmpscript} {args.recipename} {destdir} {tmpfilelist}'", cwd=recipe_outdir, shell=True, ) @@ -329,7 +310,7 @@ def deploy(args, config, basepath, workspace): 'Deploy failed - rerun with -s to get a complete error message' ) - logger.info('Successfully deployed %s' % recipe_outdir) + logger.info('Successfully deployed %s', recipe_outdir) files_list = [] for root, _, files in os.walk(recipe_outdir): @@ -344,13 +325,13 @@ def deploy(args, config, basepath, workspace): return 0 -def undeploy(args, config, basepath, workspace): +def undeploy(args, config, basepath, workspace): # pylint: disable=unused-argument """Entry point for the devtool 'undeploy' subcommand""" if args.all and args.recipename: raise argparse_oe.ArgumentUsageError( 'Cannot specify -a/--all with a recipe name', 'undeploy-target' ) - elif not args.recipename and not args.all: + if not args.recipename and not args.all: raise argparse_oe.ArgumentUsageError( "If you don't specify a recipe, you must specify -a/--all", 'undeploy-target', @@ -367,13 +348,13 @@ def undeploy(args, config, basepath, workspace): scp_sshexec = '' ssh_sshexec = 'ssh' if args.ssh_exec: - scp_sshexec = '-S %s' % args.ssh_exec + scp_sshexec = f'-S {args.ssh_exec}' ssh_sshexec = args.ssh_exec scp_port = '' ssh_port = '' if args.port: - scp_port = '-P %s' % args.port - ssh_port = '-p %s' % args.port + scp_port = f'-P {args.port}' + ssh_port = f'-p {args.port}' args.target = args.target.split(':')[0] @@ -381,43 +362,28 @@ def undeploy(args, config, basepath, workspace): try: tmpscript = '/tmp/devtool_undeploy.sh' shellscript = _prepare_remote_script( - deploy=False, dryrun=args.dry_run, undeployall=args.all + should_deploy=False, dryrun=args.dry_run, undeployall=args.all ) # Write out the script to a file - with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: + with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w', encoding='utf-8') as f: f.write(shellscript) # Copy it to the target - ret = subprocess.call( - 'scp %s %s %s %s/* %s:%s' - % ( - scp_sshexec, - scp_port, - extraoptions, - tmpdir, - args.target, - os.path.dirname(tmpscript), - ), - shell=True, + cmd = ( + f'scp {scp_sshexec} {scp_port} {extraoptions} {tmpdir}/* ' + f'{args.target}:{os.path.dirname(tmpscript)}' ) + ret = subprocess.call(cmd, shell=True) if ret != 0: raise DevtoolError( - 'Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target + f'Failed to copy script to {args.target} - rerun with -s to ' + 'get a complete error message' ) finally: shutil.rmtree(tmpdir) # Now run the script ret = subprocess.call( - "%s %s %s %s 'sh %s %s'" - % ( - ssh_sshexec, - ssh_port, - extraoptions, - args.target, - tmpscript, - args.recipename, - ), + f"{ssh_sshexec} {ssh_port} {extraoptions} {args.target} 'sh {tmpscript} {args.recipename}'", shell=True, ) if ret != 0: @@ -426,7 +392,7 @@ def undeploy(args, config, basepath, workspace): ) if not args.all and not args.dry_run: - logger.info('Successfully undeployed %s' % args.recipename) + logger.info('Successfully undeployed %s', args.recipename) return 0 diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py index 1008bc3..4e03528 100644 --- a/src/oebuild/app/plugins/init/init.py +++ b/src/oebuild/app/plugins/init/init.py @@ -214,7 +214,7 @@ please execute the follow commands next try: compil = oebuild_util.get_compile_yaml_dir() shutil.copyfile( - compil, os.path.join(updir, oebuild_const.COMPILE_YAML) + compil, os.path.join(updir, oebuild_const.COMPILE_YAML_SAMPLE) ) except FileNotFoundError: logger.error('mkdir compile.yaml.sample failed') diff --git a/src/oebuild/app/plugins/m_env/m_env.py b/src/oebuild/app/plugins/m_env/m_env.py index 538bf22..1e2b5f0 100644 --- a/src/oebuild/app/plugins/m_env/m_env.py +++ b/src/oebuild/app/plugins/m_env/m_env.py @@ -114,45 +114,56 @@ class Menv(OebuildCommand): sys.exit(0) args = args.parse_args(unknown) + self._handle_command(command, args) + + def _handle_command(self, command, args): + """Handle the environment command.""" if command == 'create': - if not args.env_name: - print(""" -Please enter the correct command: oebuild menv create [-d -f] Create an environment -n env_name - """) - sys.exit(1) - self.create_environment(args=args) + self._handle_create(args) elif command == 'activate': - # Activate Environment - if args.env_name: - self.activate_environment(args.env_name) - sys.exit(0) - print( - 'Please enter the correct command: oebuild menv activate -n env_name' - ) - sys.exit(1) - + self._handle_activate(args) elif command == 'list': - env_dict = oebuild_util.read_yaml(self.oebuild_env_yaml_path) - if env_dict and 'env_config' in env_dict: - self.list_environment(env_dict) - sys.exit(0) - else: - print('No environment has been created yet') - sys.exit(-1) - - # delete environment + self._handle_list() elif command == 'remove': - if args.env_name: - self.delete_environment(args.env_name) - sys.exit(0) - print( - 'Please enter the correct command: oebuild menv remove -n env_name' - ) - sys.exit(1) + self._handle_remove(args) else: # This handles the case where command is None (help was shown) sys.exit(0) + def _handle_create(self, args): + """Handle the create environment command.""" + if not args.env_name: + print(""" +Please enter the correct command: oebuild menv create [-d -f] Create an environment -n env_name + """) + sys.exit(1) + self.create_environment(args=args) + + def _handle_activate(self, args): + """Handle the activate environment command.""" + if args.env_name: + self.activate_environment(args.env_name) + sys.exit(0) + print('Please enter the correct command: oebuild menv activate -n env_name') + sys.exit(1) + + def _handle_list(self): + """Handle the list environment command.""" + env_dict = oebuild_util.read_yaml(self.oebuild_env_yaml_path) + if env_dict and 'env_config' in env_dict: + self.list_environment(env_dict) + sys.exit(0) + print('No environment has been created yet') + sys.exit(-1) + + def _handle_remove(self, args): + """Handle the remove environment command.""" + if args.env_name: + self.delete_environment(args.env_name) + sys.exit(0) + print('Please enter the correct command: oebuild menv remove -n env_name') + sys.exit(1) + def create_environment(self, args): """ create environment file in ~/.local/oebuild_env/ and do something in next step diff --git a/src/oebuild/app/plugins/mugentest/mugentest.py b/src/oebuild/app/plugins/mugentest/mugentest.py index e5ecfad..6524d23 100644 --- a/src/oebuild/app/plugins/mugentest/mugentest.py +++ b/src/oebuild/app/plugins/mugentest/mugentest.py @@ -191,17 +191,21 @@ class MugenTest(OebuildCommand): # if args.env == "qemu": # if suite == "embedded_tiny_image_test": # cmd = ( - # f"bash {mugen_path}/mugen.sh -c --ip {args.ip} --password {args.password} " - # f"--user {args.user} --port {args.port} --put_all --run_remote" + # f"bash {mugen_path}/mugen.sh -c --ip {args.ip} " + # f"--password {args.password} --user {args.user} " + # f"--port {args.port}" + # f"--put_all --run_remote" # ) # else: # if not args.kernal_img_path or not args.initrd_path: # logger.error( - # "For this test, --kernal_img_path and --initrd_path are required." + # "For this test, --kernal_img_path and " + # "--initrd_path are required." # ) # return # qemu_start_cmd = ( - # f"sh qemu_ctl.sh start --put_all --kernal_img_path {args.kernal_img_path} " + # f"sh qemu_ctl.sh start --put_all " + # f"--kernal_img_path {args.kernal_img_path} " # f"--initrd_path {args.initrd_path}" # ) # if suite in { diff --git a/src/oebuild/auto_completion.py b/src/oebuild/auto_completion.py index 97076c2..d19db9e 100644 --- a/src/oebuild/auto_completion.py +++ b/src/oebuild/auto_completion.py @@ -11,10 +11,10 @@ See the Mulan PSL v2 for more details. """ import hashlib +import logging import re import sys import textwrap -import logging from os.path import abspath, expanduser, expandvars, join logger = logging.getLogger() @@ -58,8 +58,9 @@ class AutoCompletion: rc_content = fh.read() except FileNotFoundError: rc_content = '' - except Exception: - raise + except Exception as e: # pylint: disable=broad-exception-caught + logger.warning('Failed to read %s: %s', bashrc_path, e) + rc_content = '' pattern_bashrc = ( r'(?=###!###>>>>>>>>>>>oebuild_complete>>>>>>>>>>>>>>>)[\W\w]+(?<=###!###<<<<<<<<<<<' diff --git a/src/oebuild/bb/utils.py b/src/oebuild/bb/utils.py index f84c3d0..7dd9a02 100644 --- a/src/oebuild/bb/utils.py +++ b/src/oebuild/bb/utils.py @@ -2,7 +2,14 @@ # # SPDX-License-Identifier: GPL-2.0-only # +""" +Bitbake utility functions for oebuild +This module provides utility functions for handling bitbake metadata, +environment variables, and configuration files. +""" + +import fnmatch import os import re @@ -108,18 +115,14 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): else: override_re = '' for var in variables: - if var.endswith('()'): + if var.endswith("()"): var_res[var] = re.compile( - r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' - % (var[:-2].rstrip(), override_re) + fr"^(\{var[:-2].rstrip()}\{override_re})[ \\t]*\\([ \\t]*\\)[ \\t]*{{" ) else: var_res[var] = re.compile( - r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' - % (var, override_re) + fr'^(\{var}\{override_re})[ \\\\\t]*[?+:.]*=[+.]*[ \\t]*(["\\])' ) - - updated = False varset_start = '' varlines = [] newlines = [] @@ -138,10 +141,11 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): if newvalue is None: # Drop the value return True - elif newvalue != full_value or (newop not in [None, op]): + + if newvalue != full_value or (newop not in [None, op]): if newop not in [None, op]: # Callback changed the operator - varset_new = '%s %s' % (in_var, newop) + varset_new = f"{in_var} {newop}" else: varset_new = varset_start @@ -155,55 +159,51 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): if in_var.endswith('()'): # A function definition if isinstance(newvalue, list): + indented_values = f"\n{indentspc}".join(newvalue) newlines.append( - '%s {\n%s%s\n}\n' - % ( - varset_new, - indentspc, - ('\n%s' % indentspc).join(newvalue), - ) + f"{varset_new} {{\n{indentspc}{indented_values}\n}}\n" ) else: if not newvalue.startswith('\n'): newvalue = '\n' + newvalue if not newvalue.endswith('\n'): newvalue = newvalue + '\n' - newlines.append('%s {%s}\n' % (varset_new, newvalue)) + newlines.append(f"{varset_new} {{{newvalue}}}\n") else: # Normal variable if isinstance(newvalue, list): if not newvalue: # Empty list -> empty string - newlines.append('%s ""\n' % varset_new) + newlines.append(f'{varset_new} ""\n') elif minbreak: # First item on first line if len(newvalue) == 1: newlines.append( - '%s "%s"\n' % (varset_new, newvalue[0]) + f'{varset_new} "{newvalue[0]}"\n' ) else: newlines.append( - '%s "%s \\\n' % (varset_new, newvalue[0]) + f'{varset_new} "{newvalue[0]} \\\n' ) for item in newvalue[1:]: newlines.append( - '%s%s \\\n' % (indentspc, item) + f'{indentspc}{item} \\\n' ) - newlines.append('%s"\n' % indentspc) + newlines.append(f'{indentspc}"\n') else: # No item on first line - newlines.append('%s " \\\n' % varset_new) + newlines.append(f'{varset_new} " \\\n') for item in newvalue: - newlines.append('%s%s \\\n' % (indentspc, item)) - newlines.append('%s"\n' % indentspc) + newlines.append(f'{indentspc}{item} \\\n') + newlines.append(f'{indentspc}"\n') else: - newlines.append('%s "%s"\n' % (varset_new, newvalue)) + newlines.append(f'{varset_new} "{newvalue}"\n') return True - else: - # Put the old lines back where they were - newlines.extend(varlines) - # If newlines was touched by the function, we'll need to return True - return changed + + # Put the old lines back where they were + newlines.extend(varlines) + # If newlines was touched by the function, we'll need to return True + return changed checkspc = False @@ -263,6 +263,30 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): return (updated, newlines) +def _remove_trailing_sep(pth): + """Remove trailing separator from path.""" + if pth and pth[-1] == os.sep: + pth = pth[:-1] + return pth + + +def _canonicalise_path(pth, approved_vars): + """Canonicalise path, expanding user directory if needed.""" + pth = _remove_trailing_sep(pth) + if 'HOME' in approved_vars and '~' in pth: + pth = os.path.expanduser(pth) + return pth + + +def _layerlist_param(value): + """Process layer list parameter.""" + if not value: + return [] + if isinstance(value, list): + return [_remove_trailing_sep(x) for x in value] + return [_remove_trailing_sep(value)] + + def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): """Edit bblayers.conf, adding and/or removing layers Parameters: @@ -279,32 +303,9 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): notremoved: list of layers that were specified to be removed but weren't (because they weren't in the list) """ - - import fnmatch - - def remove_trailing_sep(pth): - if pth and pth[-1] == os.sep: - pth = pth[:-1] - return pth - approved = approved_variables() - - def canonicalise_path(pth): - pth = remove_trailing_sep(pth) - if 'HOME' in approved and '~' in pth: - pth = os.path.expanduser(pth) - return pth - - def layerlist_param(value): - if not value: - return [] - elif isinstance(value, list): - return [remove_trailing_sep(x) for x in value] - else: - return [remove_trailing_sep(value)] - - addlayers = layerlist_param(add) - removelayers = layerlist_param(remove) + addlayers = _layerlist_param(add) + removelayers = _layerlist_param(remove) # Need to use a list here because we can't set non-local variables from a callback in python 2.x bblayercalls = [] @@ -312,25 +313,27 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): plusequals = False orig_bblayers = [] - def handle_bblayers_firstpass(varname, origvalue, op, newlines): + def handle_bblayers_firstpass(_varname, origvalue, op, _newlines): + """First pass handler to collect initial BBLAYERS values.""" bblayercalls.append(op) if op == '=': del orig_bblayers[:] - orig_bblayers.extend([canonicalise_path(x) for x in origvalue.split()]) + orig_bblayers.extend([_canonicalise_path(x, approved) for x in origvalue.split()]) return (origvalue, None, 2, False) - def handle_bblayers(varname, origvalue, op, newlines): + def handle_bblayers(_varname, origvalue, op, _newlines): + """Second pass handler to modify BBLAYERS values.""" updated = False - bblayers = [remove_trailing_sep(x) for x in origvalue.split()] + bblayers = [_remove_trailing_sep(x) for x in origvalue.split()] if removelayers: for removelayer in removelayers: - for layer in bblayers: + for i, layer in enumerate(bblayers): if fnmatch.fnmatch( - canonicalise_path(layer), - canonicalise_path(removelayer), + _canonicalise_path(layer, approved), + _canonicalise_path(removelayer, approved), ): updated = True - bblayers.remove(layer) + del bblayers[i] removed.append(removelayer) break if addlayers and not plusequals: @@ -343,7 +346,7 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): if edit_cb: newlist = [] for layer in bblayers: - res = edit_cb(layer, canonicalise_path(layer)) + res = edit_cb(layer, _canonicalise_path(layer, approved)) if res != layer: newlist.append(res) updated = True @@ -355,36 +358,36 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): if op == '+=' and not bblayers: bblayers = None return (bblayers, None, 2, False) - else: - return (origvalue, None, 2, False) - with open(bblayers_conf, 'r') as f: + return (origvalue, None, 2, False) + + with open(bblayers_conf, 'r', encoding='utf-8') as f: (_, newlines) = edit_metadata( f, ['BBLAYERS'], handle_bblayers_firstpass ) if not bblayercalls: - raise Exception('Unable to find BBLAYERS in %s' % bblayers_conf) + raise ValueError(f'Unable to find BBLAYERS in {bblayers_conf}') # Try to do the "smart" thing depending on how the user has laid out # their bblayers.conf file if bblayercalls.count('+=') > 1: plusequals = True - removelayers_canon = [canonicalise_path(layer) for layer in removelayers] + removelayers_canon = [_canonicalise_path(layer, approved) for layer in removelayers] notadded = [] for layer in addlayers: - layer_canon = canonicalise_path(layer) + layer_canon = _canonicalise_path(layer, approved) if ( layer_canon in orig_bblayers and layer_canon not in removelayers_canon ): notadded.append(layer) - notadded_canon = [canonicalise_path(layer) for layer in notadded] + notadded_canon = [_canonicalise_path(layer, approved) for layer in notadded] addlayers[:] = [ layer for layer in addlayers - if canonicalise_path(layer) not in notadded_canon + if _canonicalise_path(layer, approved) not in notadded_canon ] (updated, newlines) = edit_metadata( @@ -393,11 +396,11 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): if addlayers: # Still need to add these for addlayer in addlayers: - newlines.append('BBLAYERS += "%s"\n' % addlayer) + newlines.append(f'BBLAYERS += "{addlayer}"\n') updated = True if updated: - with open(bblayers_conf, 'w') as f: + with open(bblayers_conf, 'w', encoding='utf-8') as f: f.writelines(newlines) notremoved = list(set(removelayers) - set(removed)) diff --git a/src/oebuild/const.py b/src/oebuild/const.py index c36e333..49a4f9f 100644 --- a/src/oebuild/const.py +++ b/src/oebuild/const.py @@ -14,7 +14,7 @@ See the Mulan PSL v2 for more details. CONFIG_YAML = 'config.yaml' PLUGINS_YAML = 'plugins.yaml' UPGRADE_YAML = 'upgrade.yaml' -COMPILE_YAML = 'compile.yaml.sample' +COMPILE_YAML_SAMPLE = 'compile.yaml.sample' BASH_END_FLAG = ' ###!!!###' CONTAINER_USER = 'openeuler' CONTAINER_BUILD = '/home/openeuler/build' @@ -24,7 +24,6 @@ DEFAULT_SDK_DOCKER = ( 'swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-sdk:latest' ) CONTAINER_SRC = '/usr1/openeuler/src' -CONTAINER_USER = 'openeuler' NATIVESDK_DIR = '/opt/buildtools/nativesdk' PROXY_LIST = ['http_proxy', 'https_proxy'] @@ -69,7 +68,7 @@ TOOLCHAIN_BASH_BANNER = """ YOCTO_META_OPENEULER = 'yocto_meta_openeuler' YOCTO_POKY = 'yocto-poky' CONFIG = 'config' -COMPILE_YAML = 'compile.yaml.sample' +COMPILE_YAML_SAMPLE = 'compile.yaml.sample' # used for parse_templete.py PLATFORM = 'platform' diff --git a/src/oebuild/oebuild_parser.py b/src/oebuild/oebuild_parser.py index e6ec0cc..87f661c 100644 --- a/src/oebuild/oebuild_parser.py +++ b/src/oebuild/oebuild_parser.py @@ -11,10 +11,10 @@ See the Mulan PSL v2 for more details. """ import argparse -import sys import shutil -from io import StringIO +import sys import textwrap +from io import StringIO class OebuildHelpAction(argparse.Action): @@ -47,7 +47,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): # come first as our override of that method relies on it. self.oebuild_optionals = [] self.oebuild_app = kwargs.pop('oebuild_app', None) - super(OebuildArgumentParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def print_help(self, file=None): print(self.format_help(), end='', file=file or sys.stdout) diff --git a/src/oebuild/util.py b/src/oebuild/util.py index ba217fb..e8ee49a 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -163,7 +163,7 @@ def get_compile_yaml_dir(): return compile.yaml.sample yaml dir """ return os.path.join( - get_base_oebuild(), 'app/conf', oebuild_const.COMPILE_YAML + get_base_oebuild(), 'app/conf', oebuild_const.COMPILE_YAML_SAMPLE ) -- Gitee From aad4c4d26be58d278491fbf55e2f011c2290b4ac Mon Sep 17 00:00:00 2001 From: Egg12138 Date: Tue, 25 Nov 2025 13:16:35 +0800 Subject: [PATCH 3/3] refactor: refactor code to fix pylint issues Rename neo-generate directory to neo_generate for naming compliance Refactor deploy.py by splitting large functions into smaller ones Refactor utils.py complex functions using helper functions and state dict Add pylint disable comments for unavoidable complex code blocks Improve code readability and maintainability, meet code quality standards Signed-off-by: egg12138 --- src/oebuild/app/conf/plugins.yaml | 2 +- .../app/plugins/deploy/target_dev/deploy.py | 347 ++++++++----- src/oebuild/app/plugins/init/init.py | 14 +- .../__init__.py | 0 src/oebuild/bb/utils.py | 473 +++++++++++------- src/oebuild/const.py | 3 +- src/oebuild/util.py | 20 +- 7 files changed, 520 insertions(+), 339 deletions(-) rename src/oebuild/app/plugins/{neo-generate => neo_generate}/__init__.py (100%) diff --git a/src/oebuild/app/conf/plugins.yaml b/src/oebuild/app/conf/plugins.yaml index e1d7899..c0d2faf 100644 --- a/src/oebuild/app/conf/plugins.yaml +++ b/src/oebuild/app/conf/plugins.yaml @@ -10,7 +10,7 @@ plugins: path: plugins/generate/generate.py - name: neo-generate class: NeoGenerate - path: plugins/neo-generate/neo_generate.py + path: plugins/neo_generate/neo_generate.py - name: bitbake class: Bitbake path: plugins/bitbake/bitbake.py diff --git a/src/oebuild/app/plugins/deploy/target_dev/deploy.py b/src/oebuild/app/plugins/deploy/target_dev/deploy.py index 1f58c9a..4d2415d 100644 --- a/src/oebuild/app/plugins/deploy/target_dev/deploy.py +++ b/src/oebuild/app/plugins/deploy/target_dev/deploy.py @@ -16,13 +16,38 @@ import tempfile import argparse_oe # pylint: disable=import-error import oe.types # pylint: disable=import-error import oe_package # pylint: disable=import-error -from devtool import DevtoolError, exec_fakeroot, setup_tinfoil # pylint: disable=import-error +from devtool import ( # pylint: disable=import-error + DevtoolError, + exec_fakeroot, + setup_tinfoil, +) logger = logging.getLogger('devtool') DEPLOYLIST_PATH = '/.devtool' +class _DeployTempFiles: # pylint: disable=too-few-public-methods + """Temporary paths used during a deployment.""" + + def __init__(self): + self.tmpdir = '' + self.tmpscript = '' + self.tmpfilelist = '' + + +class _DeployContext: # pylint: disable=too-few-public-methods + """Shared deployment state passed between helpers.""" + # pylint: disable=too-many-arguments,too-many-positional-arguments + def __init__(self, args, rd, recipe_outdir, destdir, ssh_opts): + self.args = args + self.rd = rd + self.recipe_outdir = recipe_outdir + self.destdir = destdir + self.ssh_opts = ssh_opts + self.temp = _DeployTempFiles() + + def _prepare_remote_script( # pylint: disable=too-many-positional-arguments, too-many-arguments should_deploy, verbose=False, @@ -148,9 +173,196 @@ def _prepare_remote_script( # pylint: disable=too-many-positional-arguments, to return '\n'.join(lines) +def _prepare_recipe_output_dir(rd, recipe_outdir, args): + """Prepare recipe output directory, applying strip if requested.""" + if not args.strip or args.dry_run: + return recipe_outdir + + # Fakeroot copy to new destination + srcdir = recipe_outdir + new_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped') + + if os.path.isdir(new_outdir): + exec_fakeroot(rd, f'rm -rf {new_outdir}', shell=True) + + exec_fakeroot( + rd, + f'cp -af {os.path.join(srcdir, ".")} {new_outdir}', + shell=True, + ) + + os.environ['PATH'] = ':'.join( + [os.environ['PATH'], rd.getVar('PATH') or ''] + ) + + oe_package.strip_execs( + args.recipename, + new_outdir, + rd.getVar('STRIP'), + rd.getVar('libdir'), + rd.getVar('base_libdir'), + rd, + ) + + return new_outdir + + +def _build_file_list(recipe_outdir, destdir): + """Build list of files to deploy with their sizes.""" + filelist = [] + inodes = set() + ftotalsize = 0 + + for root, _, files in os.walk(recipe_outdir): + for fn in files: + fstat = os.lstat(os.path.join(root, fn)) + + # Get the size in kiB (since we'll be comparing it to the output of du -k) + # MUST use lstat() here not stat() or getfilesize() since we don't want to + # dereference symlinks + if fstat.st_ino in inodes: + fsize = 0 + else: + fsize = int(math.ceil(float(fstat.st_size) / 1024)) + + inodes.add(fstat.st_ino) + ftotalsize += fsize + + # The path as it would appear on the target + fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) + filelist.append((fpath, fsize)) + + return filelist, ftotalsize + + +def _prepare_ssh_options(args): + """Prepare SSH options based on command arguments.""" + opts = { + 'extraoptions': '', + 'scp_sshexec': '', + 'ssh_sshexec': 'ssh', + 'scp_port': '', + 'ssh_port': '' + } + + if args.no_host_check: + opts['extraoptions'] += ( + '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + ) + + if not args.show_status: + opts['extraoptions'] += ' -q' + + if args.ssh_exec: + opts['scp_sshexec'] = f'-S {args.ssh_exec}' + opts['ssh_sshexec'] = args.ssh_exec + + if args.port: + opts['scp_port'] = f'-P {args.port}' + opts['ssh_port'] = f'-p {args.port}' + + if args.key: + opts['extraoptions'] += f' -i {args.key}' + + return opts + + +def _write_deployment_files(context, shellscript, filelist, ftotalsize): + """Write deployment script and file list to temporary directory.""" + tmpscript = context.temp.tmpscript + tmpfilelist = context.temp.tmpfilelist + tmpdir = context.temp.tmpdir + + script_path = os.path.join(tmpdir, os.path.basename(tmpscript)) + with open(script_path, 'w', encoding='utf-8') as f: + f.write(shellscript) + + list_path = os.path.join(tmpdir, os.path.basename(tmpfilelist)) + with open(list_path, 'w', encoding='utf-8') as f: + f.write(f'{ftotalsize}\n') + for fpath, fsize in filelist: + f.write(f'{fpath} {fsize}\n') + + +def _copy_files_to_target(context): + """Copy deployment files to target system.""" + cmd = ( + 'scp ' + f'{context.ssh_opts["scp_sshexec"]} ' + f'{context.ssh_opts["scp_port"]} ' + f'{context.ssh_opts["extraoptions"]} ' + f'{context.temp.tmpdir}/* ' + f'{context.args.target}:{os.path.dirname(context.temp.tmpscript)}' + ) + + ret = subprocess.call(cmd, shell=True) + if ret != 0: + raise DevtoolError( + f'Failed to copy script to {context.args.target} - rerun with -s to ' + 'get a complete error message' + ) + + +def _execute_remote_deployment(context): + """Execute the remote deployment script.""" + remote_cmd = ( + 'tar cf - . | ' + f"{context.ssh_opts['ssh_sshexec']} " + f"{context.ssh_opts['ssh_port']} " + f"{context.ssh_opts['extraoptions']} " + f"{context.args.target} " + f"'sh {context.temp.tmpscript} {context.args.recipename} " + f"{context.destdir} {context.temp.tmpfilelist}'" + ) + ret = exec_fakeroot( + context.rd, + remote_cmd, + cwd=context.recipe_outdir, + shell=True, + ) + + if ret != 0: + raise DevtoolError( + 'Deploy failed - rerun with -s to get a complete error message' + ) + + +def _deploy_files_to_target(context, filelist, ftotalsize): + """Deploy files to target system.""" + # In order to delete previously deployed files and have the manifest file on + # the target, we write out a shell script and then copy it to the target + # so we can then run it (piping tar output to it). + # (We cannot use scp here, because it doesn't preserve symlinks.) + context.temp.tmpdir = tempfile.mkdtemp(prefix='devtool') + + try: + context.temp.tmpscript = '/tmp/devtool_deploy.sh' + context.temp.tmpfilelist = os.path.join( + os.path.dirname(context.temp.tmpscript), + 'devtool_deploy.list', + ) + + shellscript = _prepare_remote_script( + should_deploy=True, + verbose=context.args.show_status, + nopreserve=context.args.no_preserve, + nocheckspace=context.args.no_check_space, + ) + + _write_deployment_files(context, shellscript, filelist, ftotalsize) + _copy_files_to_target(context) + + finally: + if context.temp.tmpdir: + shutil.rmtree(context.temp.tmpdir) + + _execute_remote_deployment(context) + logger.info('Successfully deployed %s', context.recipe_outdir) + + def deploy( args, config, basepath, workspace -): # pylint: disable=unused-argument, too-many-statements, too-many-branches +): # pylint: disable=unused-argument, too-many-statements, too-many-branches, too-many-locals """Entry point for the devtool 'deploy' subcommand""" # check_workspace_recipe(workspace, args.recipename, checksrc=False) @@ -160,10 +372,12 @@ def deploy( destdir = '/' else: args.target = host + if not destdir.endswith('/'): destdir += '/' tinfoil = setup_tinfoil(basepath=basepath) + try: try: rd = tinfoil.parse_recipe(args.recipename) @@ -171,6 +385,7 @@ def deploy( raise DevtoolError( f'Exception parsing recipe {args.recipename}: {e}' ) from e + recipe_outdir = rd.getVar('D') if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): raise DevtoolError( @@ -179,51 +394,8 @@ def deploy( 'any files.' ) - if args.strip and not args.dry_run: - # Fakeroot copy to new destination - srcdir = recipe_outdir - recipe_outdir = os.path.join( - rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped' - ) - if os.path.isdir(recipe_outdir): - exec_fakeroot(rd, f'rm -rf {recipe_outdir}', shell=True) - exec_fakeroot( - rd, - f'cp -af {os.path.join(srcdir, ".")} {recipe_outdir}', - shell=True, - ) - os.environ['PATH'] = ':'.join( - [os.environ['PATH'], rd.getVar('PATH') or ''] - ) - oe_package.strip_execs( - args.recipename, - recipe_outdir, - rd.getVar('STRIP'), - rd.getVar('libdir'), - rd.getVar('base_libdir'), - rd, - ) - - filelist = [] - inodes = set({}) - ftotalsize = 0 - for root, _, files in os.walk(recipe_outdir): - for fn in files: - fstat = os.lstat(os.path.join(root, fn)) - # Get the size in kiB (since we'll be comparing it to the output of du -k) - # MUST use lstat() here not stat() or getfilesize() since we don't want to - # dereference symlinks - if fstat.st_ino in inodes: - fsize = 0 - else: - fsize = int(math.ceil(float(fstat.st_size) / 1024)) - inodes.add(fstat.st_ino) - ftotalsize += fsize - # The path as it would appear on the target - fpath = os.path.join( - destdir, os.path.relpath(root, recipe_outdir), fn - ) - filelist.append((fpath, fsize)) + recipe_outdir = _prepare_recipe_output_dir(rd, recipe_outdir, args) + filelist, ftotalsize = _build_file_list(recipe_outdir, destdir) if args.dry_run: print( @@ -233,84 +405,9 @@ def deploy( print(f' {item}') return 0 - ssh_opts = {'extraoptions': '', 'scp_sshexec': '', 'ssh_sshexec': 'ssh', - 'scp_port': '', 'ssh_port': ''} - if args.no_host_check: - ssh_opts['extraoptions'] += ( - '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' - ) - if not args.show_status: - ssh_opts['extraoptions'] += ' -q' - - if args.ssh_exec: - ssh_opts['scp_sshexec'] = f'-S {args.ssh_exec}' - ssh_opts['ssh_sshexec'] = args.ssh_exec - - if args.port: - ssh_opts['scp_port'] = f'-P {args.port}' - ssh_opts['ssh_port'] = f'-p {args.port}' - - if args.key: - ssh_opts['extraoptions'] += f' -i {args.key}' - - # In order to delete previously deployed files and have the manifest file on - # the target, we write out a shell script and then copy it to the target - # so we can then run it (piping tar output to it). - # (We cannot use scp here, because it doesn't preserve symlinks.) - tmpdir = tempfile.mkdtemp(prefix='devtool') - try: - tmpscript = '/tmp/devtool_deploy.sh' - tmpfilelist = os.path.join( - os.path.dirname(tmpscript), 'devtool_deploy.list' - ) - shellscript = _prepare_remote_script( - should_deploy=True, - verbose=args.show_status, - nopreserve=args.no_preserve, - nocheckspace=args.no_check_space, - ) - # Write out the script to a file - with open( - os.path.join(tmpdir, os.path.basename(tmpscript)), 'w', encoding='utf-8' - ) as f: - f.write(shellscript) - # Write out the file list - with open( - os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w', encoding='utf-8' - ) as f: - f.write(f'{ftotalsize}\n') - for fpath, fsize in filelist: - f.write(f'{fpath} {fsize}\n') - # Copy them to the target - cmd = ( - f'scp {ssh_opts["scp_sshexec"]} {ssh_opts["scp_port"]} ' - f'{ssh_opts["extraoptions"]} {tmpdir}/* ' - f'{args.target}:{os.path.dirname(tmpscript)}' - ) - ret = subprocess.call(cmd, shell=True) - if ret != 0: - raise DevtoolError( - f'Failed to copy script to {args.target} - rerun with -s to ' - 'get a complete error message' - ) - finally: - shutil.rmtree(tmpdir) - - # Now run the script - ret = exec_fakeroot( - rd, - f"tar cf - . | {ssh_opts['ssh_sshexec']} {ssh_opts['ssh_port']} " - f"{ssh_opts['extraoptions']} {args.target} " - f"'sh {tmpscript} {args.recipename} {destdir} {tmpfilelist}'", - cwd=recipe_outdir, - shell=True, - ) - if ret != 0: - raise DevtoolError( - 'Deploy failed - rerun with -s to get a complete error message' - ) - - logger.info('Successfully deployed %s', recipe_outdir) + ssh_opts = _prepare_ssh_options(args) + context = _DeployContext(args, rd, recipe_outdir, destdir, ssh_opts) + _deploy_files_to_target(context, filelist, ftotalsize) files_list = [] for root, _, files in os.walk(recipe_outdir): diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py index 4e03528..8cf7838 100644 --- a/src/oebuild/app/plugins/init/init.py +++ b/src/oebuild/app/plugins/init/init.py @@ -11,17 +11,17 @@ See the Mulan PSL v2 for more details. """ import argparse -import shutil import os -import textwrap -import sys import pathlib +import shutil +import sys +import textwrap -from oebuild.command import OebuildCommand +import oebuild.const as oebuild_const import oebuild.util as oebuild_util -from oebuild.configure import Configure, ConfigBasicRepo, Config +from oebuild.command import OebuildCommand +from oebuild.configure import Config, ConfigBasicRepo, Configure from oebuild.m_log import logger -import oebuild.const as oebuild_const class Init(OebuildCommand): @@ -214,7 +214,7 @@ please execute the follow commands next try: compil = oebuild_util.get_compile_yaml_dir() shutil.copyfile( - compil, os.path.join(updir, oebuild_const.COMPILE_YAML_SAMPLE) + compil, os.path.join(updir, oebuild_const.compile_yaml_sample) ) except FileNotFoundError: logger.error('mkdir compile.yaml.sample failed') diff --git a/src/oebuild/app/plugins/neo-generate/__init__.py b/src/oebuild/app/plugins/neo_generate/__init__.py similarity index 100% rename from src/oebuild/app/plugins/neo-generate/__init__.py rename to src/oebuild/app/plugins/neo_generate/__init__.py diff --git a/src/oebuild/bb/utils.py b/src/oebuild/bb/utils.py index 7dd9a02..feef341 100644 --- a/src/oebuild/bb/utils.py +++ b/src/oebuild/bb/utils.py @@ -61,7 +61,125 @@ def approved_variables(): return approved -def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): +def _prepare_variable_regexes(variables, match_overrides): + """Prepare regex patterns for variable matching.""" + var_res = {} + override_re = r'(_[a-zA-Z0-9-_$(){}]+)?' if match_overrides else '' + + for var in variables: + if var.endswith("()"): + var_res[var] = re.compile( + fr"^(\{var[:-2].rstrip()}\{override_re})[ \t]*\([ \t]*\)[ \t]*{{" + ) + else: + var_res[var] = re.compile( + fr'^(\{var}\{override_re})[ \\\\\t]*[?+:.]*=[+.]*[ \\t]*(["\\])' + ) + + return var_res + + +def _handle_function_formatting(varset_new, newvalue, newop, indent): + """Handle formatting for function definitions.""" + # A function definition + if isinstance(indent, int): + if indent == -1: + indentspc = ' ' * (len(f"{varset_new} {newop}") + 2) + else: + indentspc = ' ' * indent + else: + indentspc = indent + + if isinstance(newvalue, list): + indented_values = f"\n{indentspc}".join(newvalue) + return f"{varset_new} {{\n{indentspc}{indented_values}\n}}\n" + + # Single value + if not newvalue.startswith('\n'): + newvalue = '\n' + newvalue + if not newvalue.endswith('\n'): + newvalue = newvalue + '\n' + return f"{varset_new} {{{newvalue}}}\n" + + +def _handle_variable_formatting(varset_new, newvalue, minbreak, indent): + """Handle formatting for normal variables.""" + # Normal variable + if isinstance(indent, int): + if indent == -1: + indentspc = ' ' * (len(varset_new) + 2) + else: + indentspc = ' ' * indent + else: + indentspc = indent + + if not isinstance(newvalue, list): + return f'{varset_new} "{newvalue}"\n' + + # List value + if not newvalue: + # Empty list -> empty string + return f'{varset_new} ""\n' + + if minbreak: + # First item on first line + if len(newvalue) == 1: + return f'{varset_new} "{newvalue[0]}"\n' + + result = [f'{varset_new} "{newvalue[0]} \\\n'] + for item in newvalue[1:]: + result.append(f'{indentspc}{item} \\\n') + result.append(f'{indentspc}"\n') + return ''.join(result) + + # No item on first line + result = [f'{varset_new} " \\\n'] + for item in newvalue: + result.append(f'{indentspc}{item} \\\n') + result.append(f'{indentspc}"\n') + return ''.join(result) + + +def _handle_var_end(state, varfunc): + """Handle the end of a variable/function block.""" + in_var = state['in_var'] + full_value = state['full_value'] + varset_start = state['varset_start'] + varlines = state['varlines'] + newlines = state['newlines'] + + prerun_newlines = newlines[:] + op = varset_start[len(in_var):].strip() + newvalue, newop, indent, minbreak = varfunc(in_var, full_value, op, newlines) + changed = prerun_newlines != newlines + + if newvalue is None: + # Drop the value + return True, changed + + if newvalue == full_value and newop in [None, op]: + # Put the old lines back where they were + newlines.extend(varlines) + return False, changed + + # Variable value or operator changed + varset_new = f"{in_var} {newop}" if newop not in [None, op] else varset_start + + if in_var.endswith('()'): + newlines.append(_handle_function_formatting( + varset_new, newvalue, newop, indent + )) + else: + newlines.append(_handle_variable_formatting( + varset_new, newvalue, minbreak, indent + )) + + return True, changed + + +def edit_metadata( + meta_lines, variables, varfunc, match_overrides=False +): # pylint: disable=too-many-statements """Edit lines from a recipe or config file and modify one or more specified variable values set in the file using a specified callback function. Lines are expected to have trailing newlines. @@ -108,159 +226,92 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): newlines: Lines after processing """ - - var_res = {} - if match_overrides: - override_re = r'(_[a-zA-Z0-9-_$(){}]+)?' - else: - override_re = '' - for var in variables: - if var.endswith("()"): - var_res[var] = re.compile( - fr"^(\{var[:-2].rstrip()}\{override_re})[ \\t]*\\([ \\t]*\\)[ \\t]*{{" - ) + var_res = _prepare_variable_regexes(variables, match_overrides) + state = { + 'varset_start': '', + 'varlines': [], + 'newlines': [], + 'in_var': None, + 'full_value': '', + 'var_end': '', + 'checkspc': False, + 'updated': False + } + + def _process_line(line): + if not state['in_var']: + _process_non_variable_line(line) else: - var_res[var] = re.compile( - fr'^(\{var}\{override_re})[ \\\\\t]*[?+:.]*=[+.]*[ \\t]*(["\\])' - ) - varset_start = '' - varlines = [] - newlines = [] - in_var = None - full_value = '' - var_end = '' - - def handle_var_end(): - prerun_newlines = newlines[:] - op = varset_start[len(in_var):].strip() - (newvalue, newop, indent, minbreak) = varfunc( - in_var, full_value, op, newlines - ) - changed = prerun_newlines != newlines - - if newvalue is None: - # Drop the value - return True - - if newvalue != full_value or (newop not in [None, op]): - if newop not in [None, op]: - # Callback changed the operator - varset_new = f"{in_var} {newop}" - else: - varset_new = varset_start - - if isinstance(indent, int): - if indent == -1: - indentspc = ' ' * (len(varset_new) + 2) - else: - indentspc = ' ' * indent - else: - indentspc = indent - if in_var.endswith('()'): - # A function definition - if isinstance(newvalue, list): - indented_values = f"\n{indentspc}".join(newvalue) - newlines.append( - f"{varset_new} {{\n{indentspc}{indented_values}\n}}\n" - ) - else: - if not newvalue.startswith('\n'): - newvalue = '\n' + newvalue - if not newvalue.endswith('\n'): - newvalue = newvalue + '\n' - newlines.append(f"{varset_new} {{{newvalue}}}\n") - else: - # Normal variable - if isinstance(newvalue, list): - if not newvalue: - # Empty list -> empty string - newlines.append(f'{varset_new} ""\n') - elif minbreak: - # First item on first line - if len(newvalue) == 1: - newlines.append( - f'{varset_new} "{newvalue[0]}"\n' - ) - else: - newlines.append( - f'{varset_new} "{newvalue[0]} \\\n' - ) - for item in newvalue[1:]: - newlines.append( - f'{indentspc}{item} \\\n' - ) - newlines.append(f'{indentspc}"\n') - else: - # No item on first line - newlines.append(f'{varset_new} " \\\n') - for item in newvalue: - newlines.append(f'{indentspc}{item} \\\n') - newlines.append(f'{indentspc}"\n') - else: - newlines.append(f'{varset_new} "{newvalue}"\n') - return True + _process_variable_continuation(line) + + def _process_non_variable_line(line): + skip = False + for varname, var_re in var_res.items(): + res = var_re.match(line) + if res: + _start_variable_processing(line, varname, res) + skip = True + break + if not skip: + _append_regular_line(line) + + def _start_variable_processing(line, varname, res): + isfunc = varname.endswith('()') + if isfunc: + splitvalue = line.split('{', 1) + state['var_end'] = '}' + else: + state['var_end'] = res.groups()[-1] + splitvalue = line.split(state['var_end'], 1) + state['varset_start'] = splitvalue[0].rstrip() + value = splitvalue[1].rstrip() + if not isfunc and value.endswith('\\'): + value = value[:-1] + state['full_value'] = value + state['varlines'] = [line] + state['in_var'] = res.group(1) + if isfunc: + state['in_var'] += '()' + if value.endswith(state['var_end']): + state['full_value'] = state['full_value'][:-1] + if _handle_var_end(state, varfunc)[0]: + state['updated'] = True + state['checkspc'] = True + state['in_var'] = None + + def _append_regular_line(line): + if state['checkspc']: + state['checkspc'] = False + if state['newlines'] and state['newlines'][-1] == '\n' and line == '\n': + # Squash blank line if there are two consecutive blanks after a removal + return + state['newlines'].append(line) + + def _process_variable_continuation(line): + value = line.rstrip() + state['varlines'].append(line) + if state['in_var'].endswith('()'): + state['full_value'] += '\n' + value + else: + state['full_value'] += value[:-1] + if not value.endswith(state['var_end']): + return - # Put the old lines back where they were - newlines.extend(varlines) - # If newlines was touched by the function, we'll need to return True - return changed + # End of variable value + if state['in_var'].endswith('()'): + if state['full_value'].count('{') - state['full_value'].count('}') >= 0: + return + state['full_value'] = state['full_value'][:-1] - checkspc = False + if _handle_var_end(state, varfunc)[0]: + state['updated'] = True + state['checkspc'] = True + state['in_var'] = None for line in meta_lines: - if in_var: - value = line.rstrip() - varlines.append(line) - if in_var.endswith('()'): - full_value += '\n' + value - else: - full_value += value[:-1] - if value.endswith(var_end): - if in_var.endswith('()'): - if full_value.count('{') - full_value.count('}') >= 0: - continue - full_value = full_value[:-1] - if handle_var_end(): - updated = True - checkspc = True - in_var = None - else: - skip = False - for varname, var_re in var_res.items(): - res = var_re.match(line) - if res: - isfunc = varname.endswith('()') - if isfunc: - splitvalue = line.split('{', 1) - var_end = '}' - else: - var_end = res.groups()[-1] - splitvalue = line.split(var_end, 1) - varset_start = splitvalue[0].rstrip() - value = splitvalue[1].rstrip() - if not isfunc and value.endswith('\\'): - value = value[:-1] - full_value = value - varlines = [line] - in_var = res.group(1) - if isfunc: - in_var += '()' - if value.endswith(var_end): - full_value = full_value[:-1] - if handle_var_end(): - updated = True - checkspc = True - in_var = None - skip = True - break - if not skip: - if checkspc: - checkspc = False - if newlines and newlines[-1] == '\n' and line == '\n': - # Squash blank line if there are two consecutive blanks after a removal - continue - newlines.append(line) - return (updated, newlines) + _process_line(line) + + return (state['updated'], state['newlines']) def _remove_trailing_sep(pth): @@ -287,7 +338,76 @@ def _layerlist_param(value): return [_remove_trailing_sep(value)] -def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): +def _process_layer_removals(bblayers, removelayers, approved, removed): + """Process layer removals from BBLAYERS list.""" + updated = False + if not removelayers: + return updated + + for removelayer in removelayers: + for i, layer in enumerate(bblayers): + if fnmatch.fnmatch( + _canonicalise_path(layer, approved), + _canonicalise_path(removelayer, approved), + ): + updated = True + del bblayers[i] + removed.append(removelayer) + break + return updated + + +def _process_layer_additions(bblayers, addlayers, plusequals): + """Process layer additions to BBLAYERS list.""" + updated = False + if not addlayers or plusequals: + return updated + + for addlayer in addlayers: + if addlayer not in bblayers: + updated = True + bblayers.append(addlayer) + addlayers.clear() + return updated + + +def _process_layer_edits(bblayers, edit_cb, approved): + """Process layer edits using callback function.""" + updated = False + if not edit_cb: + return updated, bblayers + + newlist = [] + for layer in bblayers: + res = edit_cb(layer, _canonicalise_path(layer, approved)) + if res != layer: + newlist.append(res) + updated = True + else: + newlist.append(layer) + return updated, newlist + + +def _calculate_not_added_layers(addlayers, orig_bblayers, removelayers, approved): + """Calculate layers that were not added.""" + removelayers_canon = [_canonicalise_path(layer, approved) for layer in removelayers] + notadded = [] + + for layer in addlayers: + layer_canon = _canonicalise_path(layer, approved) + if layer_canon in orig_bblayers and layer_canon not in removelayers_canon: + notadded.append(layer) + + notadded_canon = [_canonicalise_path(layer, approved) for layer in notadded] + addlayers[:] = [ + layer for layer in addlayers + if _canonicalise_path(layer, approved) not in notadded_canon + ] + + return notadded + + +def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): # pylint: disable=too-many-locals """Edit bblayers.conf, adding and/or removing layers Parameters: bblayers_conf: path to bblayers.conf file to edit @@ -323,36 +443,14 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): def handle_bblayers(_varname, origvalue, op, _newlines): """Second pass handler to modify BBLAYERS values.""" - updated = False bblayers = [_remove_trailing_sep(x) for x in origvalue.split()] - if removelayers: - for removelayer in removelayers: - for i, layer in enumerate(bblayers): - if fnmatch.fnmatch( - _canonicalise_path(layer, approved), - _canonicalise_path(removelayer, approved), - ): - updated = True - del bblayers[i] - removed.append(removelayer) - break - if addlayers and not plusequals: - for addlayer in addlayers: - if addlayer not in bblayers: - updated = True - bblayers.append(addlayer) - del addlayers[:] - - if edit_cb: - newlist = [] - for layer in bblayers: - res = edit_cb(layer, _canonicalise_path(layer, approved)) - if res != layer: - newlist.append(res) - updated = True - else: - newlist.append(layer) - bblayers = newlist + updated = False + + updated = _process_layer_removals(bblayers, removelayers, approved, removed) or updated + updated = _process_layer_additions(bblayers, addlayers, plusequals) or updated + + edit_updated, bblayers = _process_layer_edits(bblayers, edit_cb, approved) + updated = updated or edit_updated if updated: if op == '+=' and not bblayers: @@ -374,25 +472,12 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): if bblayercalls.count('+=') > 1: plusequals = True - removelayers_canon = [_canonicalise_path(layer, approved) for layer in removelayers] - notadded = [] - for layer in addlayers: - layer_canon = _canonicalise_path(layer, approved) - if ( - layer_canon in orig_bblayers - and layer_canon not in removelayers_canon - ): - notadded.append(layer) - notadded_canon = [_canonicalise_path(layer, approved) for layer in notadded] - addlayers[:] = [ - layer - for layer in addlayers - if _canonicalise_path(layer, approved) not in notadded_canon - ] + notadded = _calculate_not_added_layers(addlayers, orig_bblayers, removelayers, approved) (updated, newlines) = edit_metadata( newlines, ['BBLAYERS'], handle_bblayers ) + if addlayers: # Still need to add these for addlayer in addlayers: diff --git a/src/oebuild/const.py b/src/oebuild/const.py index 49a4f9f..110433d 100644 --- a/src/oebuild/const.py +++ b/src/oebuild/const.py @@ -14,7 +14,7 @@ See the Mulan PSL v2 for more details. CONFIG_YAML = 'config.yaml' PLUGINS_YAML = 'plugins.yaml' UPGRADE_YAML = 'upgrade.yaml' -COMPILE_YAML_SAMPLE = 'compile.yaml.sample' +compile_yaml_sample = 'compile.yaml.sample' # pylint: disable=invalid-name BASH_END_FLAG = ' ###!!!###' CONTAINER_USER = 'openeuler' CONTAINER_BUILD = '/home/openeuler/build' @@ -68,7 +68,6 @@ TOOLCHAIN_BASH_BANNER = """ YOCTO_META_OPENEULER = 'yocto_meta_openeuler' YOCTO_POKY = 'yocto-poky' CONFIG = 'config' -COMPILE_YAML_SAMPLE = 'compile.yaml.sample' # used for parse_templete.py PLATFORM = 'platform' diff --git a/src/oebuild/util.py b/src/oebuild/util.py index e8ee49a..2c74f56 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -10,28 +10,28 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. """ -import pathlib +import getpass import os -import time +import pathlib import random -import getpass -import sys import re import subprocess +import sys +import time from contextlib import contextmanager -from ruamel.yaml import YAML from docker.errors import DockerException from docker.models.containers import Container +from ruamel.yaml import YAML +import oebuild.const as oebuild_const from oebuild.docker_proxy import DockerProxy from oebuild.m_log import logger -from oebuild.version import __version__ -import oebuild.const as oebuild_const from oebuild.ogit import OGit -from oebuild.struct import RepoParam, DockerParam -from oebuild.parse_param import ParseRepoParam from oebuild.parse_env import EnvContainer, ParseEnv +from oebuild.parse_param import ParseRepoParam +from oebuild.struct import DockerParam, RepoParam +from oebuild.version import __version__ def get_nativesdk_environment( @@ -163,7 +163,7 @@ def get_compile_yaml_dir(): return compile.yaml.sample yaml dir """ return os.path.join( - get_base_oebuild(), 'app/conf', oebuild_const.COMPILE_YAML_SAMPLE + get_base_oebuild(), 'app/conf', oebuild_const.compile_yaml_sample ) -- Gitee