From bbbd372b9e66ecfc33180c0e3aebae142e59b08c Mon Sep 17 00:00:00 2001 From: alichinese Date: Mon, 24 Jun 2024 16:59:08 +0800 Subject: [PATCH 1/6] plugins: add dcommit command * the dcommit command is used to generate docker image quickly, it need .env file to get container id, so it can commit a docker image quickly. Signed-off-by: alichinese --- src/oebuild/app/conf/plugins.yaml | 3 + src/oebuild/app/plugins/dcommit/dcommit.py | 89 ++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 src/oebuild/app/plugins/dcommit/dcommit.py diff --git a/src/oebuild/app/conf/plugins.yaml b/src/oebuild/app/conf/plugins.yaml index aa2541f..04a1bd2 100644 --- a/src/oebuild/app/conf/plugins.yaml +++ b/src/oebuild/app/conf/plugins.yaml @@ -35,3 +35,6 @@ plugins: - name: toolchain class: Toolchain path: plugins/toolchain/toolchain.py +- name: dcommit + class: Dcommit + path: plugins/dcommit/dcommit.py diff --git a/src/oebuild/app/plugins/dcommit/dcommit.py b/src/oebuild/app/plugins/dcommit/dcommit.py new file mode 100644 index 0000000..a39ff0c --- /dev/null +++ b/src/oebuild/app/plugins/dcommit/dcommit.py @@ -0,0 +1,89 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import argparse +import sys +import textwrap +import os + +from docker.errors import APIError + +from oebuild.command import OebuildCommand +from oebuild.configure import Configure +from oebuild.m_log import logger +from oebuild.parse_env import ParseEnv +from oebuild.docker_proxy import DockerProxy + + +class Dcommit(OebuildCommand): + ''' + This class is designed to rapidly generate a customized container image, aiming + to address scenarios where the compilation environment has been specially tailored + but reuse of the container environment is required. + ''' + + help_msg = 'help to commit a docker image' + description = textwrap.dedent(''' + This is designed to rapidly generate a customized container image, aiming + to address scenarios where the compilation environment has been specially tailored + but reuse of the container environment is required. + ''') + + def __init__(self): + self.configure = Configure() + self.client = DockerProxy() + super().__init__('dcommit', self.help_msg, self.description) + + def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: + parser = self._parser(parser_adder, + usage=''' + %(prog)s [docker-image] +''') + + # Secondary command + return parser + + def do_run(self, args: argparse.Namespace, unknown=None): + # perpare parse help command + if '-h' in unknown: + unknown[0] = '-h' + self.pre_parse_help(args, unknown) + sys.exit(1) + docker_image = unknown[0] + docker_image_split = docker_image.split(":") + if len(docker_image_split) != 2: + logger.error("the docker image format is repository:tag," + "should be set like openeuler:latest") + sys.exit(-1) + + if not self.configure.is_oebuild_dir(): + logger.error('Your current directory had not finished init') + sys.exit(-1) + + if ".env" not in os.listdir(): + # the command must run based on .env file + logger.error("dcommand need .env to get container id" + "so you must run it in compile directory") + sys.exit(-1) + env_obj = ParseEnv(env_dir=".env") + if not self.client.is_container_exists(env_obj.container.short_id): + logger.error("the container id: %s is not exist in .env") + sys.exit(-1) + + container = self.client.get_container(env_obj.container.short_id) + logger.info("the docker image %s is generatting ...", docker_image) + try: + container.commit(docker_image_split[0], docker_image_split[1]) + except APIError: + logger.error("the commit %s failed") + sys.exit(-1) + logger.info("the new docker image %s is generated", docker_image) -- Gitee From 497d2d83248e0a4112e1f0b429b6e285b866e806 Mon Sep 17 00:00:00 2001 From: alichinese Date: Tue, 25 Jun 2024 09:52:10 +0800 Subject: [PATCH 2/6] samples: add samples command * The 'samples' command is used to efficiently manage the existing compilation templates in yocto-meta-openeuler. These templates are located in the yocto-meta-openeuler/.oebuild/samples directory, and this command allows for rapid implementation of the build process. Signed-off-by: alichinese --- src/oebuild/app/conf/plugins.yaml | 3 + src/oebuild/app/plugins/samples/samples.py | 93 ++++++++++++++++++++++ src/oebuild/configure.py | 7 ++ src/oebuild/version.py | 2 +- 4 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 src/oebuild/app/plugins/samples/samples.py diff --git a/src/oebuild/app/conf/plugins.yaml b/src/oebuild/app/conf/plugins.yaml index 04a1bd2..d21734d 100644 --- a/src/oebuild/app/conf/plugins.yaml +++ b/src/oebuild/app/conf/plugins.yaml @@ -38,3 +38,6 @@ plugins: - name: dcommit class: Dcommit path: plugins/dcommit/dcommit.py +- name: samples + class: Samples + path: plugins/samples/samples.py diff --git a/src/oebuild/app/plugins/samples/samples.py b/src/oebuild/app/plugins/samples/samples.py new file mode 100644 index 0000000..e897ff8 --- /dev/null +++ b/src/oebuild/app/plugins/samples/samples.py @@ -0,0 +1,93 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import argparse +import sys +import textwrap +import os + +from oebuild.command import OebuildCommand +from oebuild.configure import Configure +from oebuild.m_log import logger + + +class Samples(OebuildCommand): + ''' + The 'samples' command is used to efficiently manage the existing compilation templates in + yocto-meta-openeuler. These templates are located in the yocto-meta-openeuler/.oebuild/samples + directory, and this command allows for rapid implementation of the build process. + ''' + + help_msg = "manage the yocto-meta-openeuler's samples compile files" + description = textwrap.dedent(''' + The 'samples' command is used to efficiently manage the existing compilation templates in + yocto-meta-openeuler. These templates are located in the yocto-meta-openeuler/.oebuild/samples + directory, and this command allows for rapid implementation of the build process. + ''') + + def __init__(self): + self.configure = Configure() + super().__init__('samples', self.help_msg, self.description) + + def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: + parser = self._parser(parser_adder, + usage=''' + %(prog)s [list] +''') + + # Secondary command + return parser + + def do_run(self, args: argparse.Namespace, unknown=None): + # perpare parse help command + if '-h' in unknown: + unknown = ['-h'] + self.pre_parse_help(args, unknown) + sys.exit(0) + if not self.configure.is_oebuild_dir(): + logger.error('Your current directory had not finished init') + sys.exit(-1) + samples = self._get_samples() + if len(unknown) > 0 and 'list' == unknown[0]: + for key, value in samples.items(): + print(f"{key}: {value}") + return + + self.do_exec(samples=samples) + + def _get_samples(self): + list_samples = os.listdir(self.configure.yocto_samples_dir()) + res = {} + for index, sample in enumerate(list_samples): + res[str(index + 1)] = sample + return res + + def do_exec(self, samples: dict): + """ + we let the user select the sample num for next build task + """ + for key, value in samples.items(): + print(f"{key}: {value}") + select_num = "" + while True: + res = input("please select what you want build, enter the num, q for exit: ") + if res not in samples and res != "q": + logger.info("please enter the valid num or q") + continue + if res == "q": + sys.exit(0) + select_num = res + break + sample = samples[select_num] + sample_path = os.path.join(self.configure.yocto_samples_dir(), sample) + + os.system(f"oebuild {sample_path}") diff --git a/src/oebuild/configure.py b/src/oebuild/configure.py index ab916f1..109f62b 100644 --- a/src/oebuild/configure.py +++ b/src/oebuild/configure.py @@ -139,6 +139,13 @@ class Configure: ''' return os.path.join(Configure.source_yocto_dir(), ".oebuild/manifest.yaml") + @staticmethod + def yocto_samples_dir(): + ''' + return .oebuild/samples + ''' + return os.path.join(Configure.source_yocto_dir(), ".oebuild/samples") + @staticmethod def source_poky_dir(): ''' diff --git a/src/oebuild/version.py b/src/oebuild/version.py index 69f5bb8..181b436 100644 --- a/src/oebuild/version.py +++ b/src/oebuild/version.py @@ -10,4 +10,4 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -__version__ = '0.0.45' +__version__ = '0.0.45.20' -- Gitee From 8f3b19eaab346db016c0143f75e85cfb3954608d Mon Sep 17 00:00:00 2001 From: alichinese Date: Tue, 25 Jun 2024 15:15:20 +0800 Subject: [PATCH 3/6] bitbake: optimize the bitbake command * add --with-dm param, for docker image, use like: oebuild bitbake --with-dm=xxx:yyy the alter will be set in compile.yaml's docker_param->image Signed-off-by: alichinese --- src/oebuild/app/plugins/bitbake/bitbake.py | 35 +++++++++++++++++++++- src/oebuild/docker_proxy.py | 12 ++++++++ src/oebuild/util.py | 8 ++++- src/oebuild/version.py | 2 +- 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 6d55155..1f00ea1 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -54,7 +54,10 @@ class Bitbake(OebuildCommand): parser = self._parser(parser_adder, usage=''' - %(prog)s [command] + %(prog)s [command] [--with-dm] + --with-dm dm for docker image, that is pointed which docker image will be + started, and the docker_param->image will be modified from compile.yaml + the command example like: oebuild bitbake --with-dm=demo:latest ''') parser_adder.add_argument( @@ -79,6 +82,7 @@ class Bitbake(OebuildCommand): sys.exit(0) set_log_to_file() + oe_params, unknown = self._get_oebuild_param(unknown) command = self._get_command(unknow=unknown) if not self.check_support_bitbake(): @@ -92,6 +96,7 @@ class Bitbake(OebuildCommand): compile_param_dict = oebuild_util.read_yaml(self.compile_conf_dir) compile_param: CompileParam = ParseCompileParam.parse_to_obj( compile_param_dict) + compile_param = self._deal_oe_params(oe_params, compile_param) # if has manifest.yaml, init layer repo with it yocto_dir = os.path.join(self.configure.source_dir(), @@ -132,3 +137,31 @@ class Bitbake(OebuildCommand): return None return 'bitbake ' + ' '.join(unknow) + + def _get_oebuild_param(self, unknow: list): + if len(unknow) == 0: + return [],[] + oe_params = [] + new_unknow = [] + for item in unknow: + if item.startswith("--with-dm"): + oe_params.append(item) + else: + new_unknow.append(item) + return oe_params, new_unknow + + def _deal_oe_params(self, oe_params, compile_param: CompileParam): + is_modify = False + for item in oe_params: + if item.startswith("--with-dm"): + item_split = item.split("=") + if len(item_split) < 2: + logger.error("the format is --with-dm=xxx") + sys.exit(-1) + if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: + compile_param.docker_param.image = item_split[1] + is_modify = True + if is_modify: + oebuild_util.write_yaml(self.compile_conf_dir, + ParseCompileParam().parse_to_dict(compile_param)) + return compile_param diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py index f1a33fe..1005fae 100644 --- a/src/oebuild/docker_proxy.py +++ b/src/oebuild/docker_proxy.py @@ -45,6 +45,18 @@ class DockerProxy: except ImageNotFound: return False + def get_container_img_id(self, container_id): + ''' + determize if container exists + args: + container_id (str): docker container short_id or id + ''' + try: + container = self._docker.containers.get(container_id=container_id) + return container.image.id + except NotFound: + return None + def is_container_exists(self, container_id): ''' determize if container exists diff --git a/src/oebuild/util.py b/src/oebuild/util.py index a979aac..593f683 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -343,10 +343,16 @@ def deal_env_container(env: ParseEnv, docker_param: DockerParam): are inconsistent, you need to create a new container, otherwise directly enable the sleeping container ''' + def check_container_img_eq(container_id, docker_image): + c_mid = docker_proxy.get_container_img_id(container_id=container_id) + d_mid = docker_proxy.get_image(docker_image).id + return c_mid == d_mid + docker_proxy = DockerProxy() if env.container is None \ or env.container.short_id is None \ - or not docker_proxy.is_container_exists(env.container.short_id): + or not docker_proxy.is_container_exists(env.container.short_id) \ + or not check_container_img_eq(env.container.short_id, docker_param.image): # judge which container container: Container = docker_proxy.create_container( image=docker_param.image, diff --git a/src/oebuild/version.py b/src/oebuild/version.py index 181b436..efe6fe2 100644 --- a/src/oebuild/version.py +++ b/src/oebuild/version.py @@ -10,4 +10,4 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -__version__ = '0.0.45.20' +__version__ = '0.0.45.21' -- Gitee From 9912890f0e8496924a7c7d77c6244c2b6532a20a Mon Sep 17 00:00:00 2001 From: alichinese Date: Tue, 25 Jun 2024 15:39:49 +0800 Subject: [PATCH 4/6] one-click: optimize the one-click * add some notice msg when build finished Signed-off-by: alichinese --- src/oebuild/app/main.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py index 6975bfc..510d5ba 100644 --- a/src/oebuild/app/main.py +++ b/src/oebuild/app/main.py @@ -98,7 +98,7 @@ class OebuildApp: description='''The openEuler Embedded meta-tool. you can directly run oebuild in oebuild workspace to perform the build, for example: -oebuild qemu-aarch64-ros.yaml +oebuild ''', epilog='''Run "oebuild -h" for help on each .''', add_help=False, oebuild_app=self @@ -301,10 +301,11 @@ class QuickBuild(): ''' xxx ''' - os.chdir(self.build_dir) + os.chdir(os.path.abspath(self.build_dir)) if self.compile_param.bitbake_cmds is None: print("================================================\n\n" - f"please enter {self.build_dir} for next steps!!!\n\n" + "please enter follow directory for next steps!!!\n\n" + f"{os.path.abspath(self.build_dir)}\n\n" "================================================\n") return for bitbake_cmd in self.compile_param.bitbake_cmds: @@ -314,6 +315,14 @@ class QuickBuild(): bitbake_cmd.lstrip("bitbake") ] self.app.run(argv or sys.argv[1:]) + logger.info(""" +====================================================================== +the build directory is show in line, please run: + + cd %s + +to enter the build directory +""", os.path.abspath(self.build_dir)) def main(argv=None): -- Gitee From e74a7655185342573eae551b66add04f8a1f5bf7 Mon Sep 17 00:00:00 2001 From: alichinese Date: Tue, 25 Jun 2024 15:50:51 +0800 Subject: [PATCH 5/6] main: drop auto complete function * the auto compiletion will be disabled, when get more good idea and enable it Signed-off-by: alichinese --- src/oebuild/app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py index 510d5ba..0155927 100644 --- a/src/oebuild/app/main.py +++ b/src/oebuild/app/main.py @@ -21,7 +21,6 @@ import oebuild.const as oebuild_const from oebuild.m_log import logger from oebuild.parse_param import ParseCompileParam from oebuild.configure import Configure -from oebuild.auto_completion import AutoCompletion from oebuild.version import __version__ from oebuild.spec import get_spec, _ExtCommand from oebuild.command import OebuildCommand @@ -332,7 +331,8 @@ def main(argv=None): if not check_user(): return - AutoCompletion().run() + # the auto compiletion will be disabled, when get more good idea and enable it + # AutoCompletion().run() if (len(sys.argv) > 1) and 'yaml' in sys.argv[1]: build = QuickBuild(build_yaml_path=sys.argv[1]) build.run() -- Gitee From aba96ab5ddd2a259d422fa9c76c448a669c70c80 Mon Sep 17 00:00:00 2001 From: alichinese Date: Wed, 26 Jun 2024 09:56:38 +0800 Subject: [PATCH 6/6] code: optimize user experience * add docke-save for generate docker image * add --with-docker-image for pointed the start docker image Signed-off-by: alichinese --- src/oebuild/app/conf/plugins.yaml | 6 +++--- src/oebuild/app/main.py | 8 +++----- src/oebuild/app/plugins/bitbake/bitbake.py | 18 ++++++++++-------- .../dcommit.py => docker-save/docker_save.py} | 8 ++++---- 4 files changed, 20 insertions(+), 20 deletions(-) rename src/oebuild/app/plugins/{dcommit/dcommit.py => docker-save/docker_save.py} (94%) diff --git a/src/oebuild/app/conf/plugins.yaml b/src/oebuild/app/conf/plugins.yaml index d21734d..d81d6df 100644 --- a/src/oebuild/app/conf/plugins.yaml +++ b/src/oebuild/app/conf/plugins.yaml @@ -35,9 +35,9 @@ plugins: - name: toolchain class: Toolchain path: plugins/toolchain/toolchain.py -- name: dcommit - class: Dcommit - path: plugins/dcommit/dcommit.py +- name: docker-save + class: DockerSave + path: plugins/docker-save/docker_save.py - name: samples class: Samples path: plugins/samples/samples.py diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py index 0155927..1bd9f82 100644 --- a/src/oebuild/app/main.py +++ b/src/oebuild/app/main.py @@ -316,12 +316,10 @@ class QuickBuild(): self.app.run(argv or sys.argv[1:]) logger.info(""" ====================================================================== -the build directory is show in line, please run: - +Please enter the building directory according to the command prompt below: + cd %s - -to enter the build directory -""", os.path.abspath(self.build_dir)) +""", os.path.dirname(os.path.abspath(self.build_dir))) def main(argv=None): diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 1f00ea1..f24a8ed 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -54,10 +54,12 @@ class Bitbake(OebuildCommand): parser = self._parser(parser_adder, usage=''' - %(prog)s [command] [--with-dm] - --with-dm dm for docker image, that is pointed which docker image will be - started, and the docker_param->image will be modified from compile.yaml - the command example like: oebuild bitbake --with-dm=demo:latest + %(prog)s [command] [--with-docker-image] + --with-docker-image + that is pointed which docker image will be started, and the + docker_param->image will be modified from compile.yaml + the command example like: + oebuild bitbake --with-docker-image=demo:latest ''') parser_adder.add_argument( @@ -140,11 +142,11 @@ class Bitbake(OebuildCommand): def _get_oebuild_param(self, unknow: list): if len(unknow) == 0: - return [],[] + return [], [] oe_params = [] new_unknow = [] for item in unknow: - if item.startswith("--with-dm"): + if item.startswith("--with-docker-image"): oe_params.append(item) else: new_unknow.append(item) @@ -153,10 +155,10 @@ class Bitbake(OebuildCommand): def _deal_oe_params(self, oe_params, compile_param: CompileParam): is_modify = False for item in oe_params: - if item.startswith("--with-dm"): + if item.startswith("--with-docker-image"): item_split = item.split("=") if len(item_split) < 2: - logger.error("the format is --with-dm=xxx") + logger.error("the format is --with-docker-image=xxx:yyy") sys.exit(-1) if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: compile_param.docker_param.image = item_split[1] diff --git a/src/oebuild/app/plugins/dcommit/dcommit.py b/src/oebuild/app/plugins/docker-save/docker_save.py similarity index 94% rename from src/oebuild/app/plugins/dcommit/dcommit.py rename to src/oebuild/app/plugins/docker-save/docker_save.py index a39ff0c..0337dec 100644 --- a/src/oebuild/app/plugins/dcommit/dcommit.py +++ b/src/oebuild/app/plugins/docker-save/docker_save.py @@ -24,14 +24,14 @@ from oebuild.parse_env import ParseEnv from oebuild.docker_proxy import DockerProxy -class Dcommit(OebuildCommand): +class DockerSave(OebuildCommand): ''' This class is designed to rapidly generate a customized container image, aiming to address scenarios where the compilation environment has been specially tailored but reuse of the container environment is required. ''' - help_msg = 'help to commit a docker image' + help_msg = 'help to save a docker image' description = textwrap.dedent(''' This is designed to rapidly generate a customized container image, aiming to address scenarios where the compilation environment has been specially tailored @@ -41,7 +41,7 @@ class Dcommit(OebuildCommand): def __init__(self): self.configure = Configure() self.client = DockerProxy() - super().__init__('dcommit', self.help_msg, self.description) + super().__init__('docker-save', self.help_msg, self.description) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser(parser_adder, @@ -84,6 +84,6 @@ class Dcommit(OebuildCommand): try: container.commit(docker_image_split[0], docker_image_split[1]) except APIError: - logger.error("the commit %s failed") + logger.error("save %s failed") sys.exit(-1) logger.info("the new docker image %s is generated", docker_image) -- Gitee