From c58cdfce38bfc178ee1deff1242f09ac1775cec5 Mon Sep 17 00:00:00 2001 From: alichinese Date: Thu, 8 Jun 2023 14:28:02 +0800 Subject: [PATCH] oebuild: add container versioning * with the evolution of the new branch of openEuler Embedded, the adapted container is no longer applicable, so the container version management function is added to cope with such business scenarios * upgrade version to 0.0.23 Signed-off-by: lixinyu --- src/oebuild/app/conf/config.yaml | 5 +- src/oebuild/app/plugins/bitbake/bitbake.py | 6 +- .../app/plugins/bitbake/in_container.py | 17 ++-- src/oebuild/app/plugins/generate/generate.py | 37 ++++++++- src/oebuild/app/plugins/update/update.py | 55 ++----------- src/oebuild/check_docker_tag.py | 82 +++++++++++++++++++ src/oebuild/parse_compile.py | 14 +++- src/oebuild/parse_template.py | 5 +- src/oebuild/version.py | 2 +- 9 files changed, 164 insertions(+), 59 deletions(-) create mode 100644 src/oebuild/check_docker_tag.py diff --git a/src/oebuild/app/conf/config.yaml b/src/oebuild/app/conf/config.yaml index 0784271..ce4cc08 100644 --- a/src/oebuild/app/conf/config.yaml +++ b/src/oebuild/app/conf/config.yaml @@ -1,8 +1,9 @@ docker: repo_url: swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container tag_map: - openEuler-23.03: "23.03" - master: "latest" + openEuler-23.03: latest + master: latest + openEuler-22.03-LTS-SP2: 22.03-lts-sp2 basic_repo: yocto_meta_openeuler: path: yocto-meta-openeuler diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 20861a6..a92ad40 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -52,9 +52,13 @@ class Bitbake(OebuildCommand): %(prog)s [command] ''') + parser_adder.add_argument( + 'command', nargs='?', default=None, + help='''The name of the directory that will be initialized''') + return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.ArgumentParser, unknown = None): ''' The BitBake execution logic is: the first step is to prepare the code that initializes diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index 2ff94b7..d497d7c 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -48,6 +48,13 @@ class InContainer(BaseBuild): execute bitbake command ''' logger.info("bitbake starting ...") + # check docker image if exists + docker_proxy = DockerProxy() + if not docker_proxy.is_image_exists(parse_compile.docker_image): + logger.error(f'''the docker image does not exists, please run fellow command: + `oebuild update docker`''') + return + yocto_dir = os.path.join( self.configure.source_dir(), "yocto-meta-openeuler") remote, branch = OGit.get_repo_info(yocto_dir) @@ -56,7 +63,8 @@ class InContainer(BaseBuild): remote=remote, branch=branch, toolchain_dir=parse_compile.toolchain_dir, - sstate_cache=parse_compile.sstate_cache) + sstate_cache=parse_compile.sstate_cache, + docker_image=parse_compile.docker_image) self.exec_compile(parse_compile=parse_compile, command=command) @@ -65,7 +73,8 @@ class InContainer(BaseBuild): remote: str, branch: str, toolchain_dir=None, - sstate_cache = None): + sstate_cache = None, + docker_image = ""): ''' This operation realizes the processing of the container, controls how the container is processed by parsing the env @@ -100,10 +109,8 @@ class InContainer(BaseBuild): # judge which container config = self.configure.parse_oebuild_config() container_config = config.docker - # here use default mater branch bind container and fix next - image_name = f"{container_config.repo_url}:{container_config.tag_map.get('master')}" container:Container = self.client.container_run_simple( - image=image_name, + image=docker_image, volumes=volumns) # type: ignore env_container.short_id = container.short_id diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index d27bfc7..c0d25c7 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -21,6 +21,7 @@ import oebuild.util as oebuild_util from oebuild.configure import Configure from oebuild.parse_template import BaseParseTemplate, ParseTemplate, BUILD_IN_DOCKER, BUILD_IN_HOST from oebuild.m_log import logger, INFO_COLOR +from oebuild.check_docker_tag import CheckDockerTag class Generate(OebuildCommand): ''' @@ -101,6 +102,12 @@ class Generate(OebuildCommand): this param is for external nativesdk dir, the param will be useful when you want to build in host ''' ) + + parser.add_argument('-tag', dest='docker_tag', default = '', + help=''' + when build in docker, the param can be point docker image + ''' + ) parser.add_argument('-dt', '--datetime', dest = "is_datetime", action = "store_true", help=''' @@ -193,6 +200,33 @@ class Generate(OebuildCommand): if os.path.exists(os.path.join(build_dir,'compile.yaml')): os.remove(os.path.join(build_dir,'compile.yaml')) + check_docker_tag = CheckDockerTag(args.docker_tag, self.configure) + oebuild_config = self.configure.parse_oebuild_config() + if check_docker_tag.get_tag() is not None: + docker_tag = check_docker_tag.get_tag() + else: + # select docker image + while True: + print(''' +If the system does not recognize which container image to use, select the +following container, enter it numerically, and enter q to exit:''') + image_list = check_docker_tag.get_tags() + + for key,value in enumerate(image_list): + print(f"{key}, {oebuild_config.docker.repo_url}:{value}") + k = input("please entry number:") + if k == "q": + return + try: + index = int(k) + docker_tag = image_list[index] + break + except: + print("please entry true number") + docker_tag = docker_tag.strip() + docker_tag = docker_tag.strip('\n') + docker_image = f"{oebuild_config.docker.repo_url}:{docker_tag}" + out_dir = pathlib.Path(os.path.join(build_dir,'compile.yaml')) oebuild_util.write_yaml(out_dir, parser_template.generate_template( nativesdk_dir= self.nativesdk_dir, @@ -201,7 +235,8 @@ class Generate(OebuildCommand): sstate_cache= self.sstate_cache, tmp_dir = self.tmp_dir, is_datetime = args.is_datetime, - is_disable_fetch = args.is_disable_fetch + is_disable_fetch = args.is_disable_fetch, + docker_image=docker_image )) format_dir = f''' diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index d1759d2..e90010a 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -25,6 +25,7 @@ from oebuild.parse_compile import ParseCompile from oebuild.configure import Configure, ConfigBasicRepo, YOCTO_META_OPENEULER from oebuild.docker_proxy import DockerProxy from oebuild.ogit import OGit +from oebuild.check_docker_tag import CheckDockerTag from oebuild.m_log import logger @@ -51,7 +52,7 @@ class Update(OebuildCommand): usage=''' %(prog)s [yocto docker layer] [-tag] ''') - parser.add_argument('-tag', dest='docker_tag', default="latest", + parser.add_argument('-tag', dest='docker_tag', help=''' with platform will list support archs, with feature will list support features ''' @@ -98,6 +99,7 @@ class Update(OebuildCommand): if update_docker: try: oebuild_util.check_docker() + # check yocto/oebuild/env.yaml，get container_tag and update docker image self.docker_image_update(args.docker_tag) except DockerException as d_e: logger.error(str(d_e)) @@ -106,21 +108,6 @@ class Update(OebuildCommand): if update_layer: self.get_layer_repo() - def list_image_tag(self,): - ''' - print compile docker image tag list - ''' - oebuild_config = self.configure.parse_oebuild_config() - docker_config = oebuild_config.docker - log = f''' - the openeuler embedded docker image repo url: - {docker_config.repo_url} - the openeuler embedded docker tag list: - ''' - for tag in docker_config.tag_map.values(): - log += f" {tag}\n" - print(log) - def get_layer_repo(self,): ''' download or update layers that will be needed @@ -185,38 +172,12 @@ class Update(OebuildCommand): oebuild_config = self.configure.parse_oebuild_config() docker_config = oebuild_config.docker - if docker_tag is not None and docker_tag not in docker_config.tag_map.values(): - warn_msg = "please select valid docker_tag follow list" - print(warn_msg) - for tag in docker_config.tag_map.keys(): - print(docker_config.tag_map.get(tag)) + check_docker_tag = CheckDockerTag(docker_tag=docker_tag,configure=self.configure) + if check_docker_tag.get_tag() is None or check_docker_tag.get_tag() == "": + check_docker_tag.list_image_tag() return - if docker_tag is None: - basic_config = oebuild_config.basic_repo - yocto_config: ConfigBasicRepo = basic_config[YOCTO_META_OPENEULER] - if yocto_config.branch in docker_config.tag_map: - docker_tag = docker_config.tag_map[yocto_config.branch] - else: - input_msg = "please select follow docker_tag:\n" - key_list = [] - for index, key in enumerate(docker_config.tag_map): - input_msg += f"{index+1}, {docker_config.repo_url}:\ - {docker_config.tag_map[key]}\n" - key_list.append(key) - input_msg += "please enter index number(enter q will exit):" - while True: - i = input(input_msg) - if i == 'q': - sys.exit() - - i = int(i) - if i <= 0 or i > len(key_list): - logger.warning("enter wrong") - continue - docker_tag = docker_config.tag_map[key_list[i-1]] - break - - docker_image = docker_config.repo_url + ":" + docker_tag + + docker_image = docker_config.repo_url + ":" + check_docker_tag.get_tag() client = DockerProxy() logger.info("pull %s ...", docker_image) client.pull_image_with_progress(docker_image) diff --git a/src/oebuild/check_docker_tag.py b/src/oebuild/check_docker_tag.py new file mode 100644 index 0000000..35c43c1 --- /dev/null +++ b/src/oebuild/check_docker_tag.py @@ -0,0 +1,82 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os +import pathlib + +from git import Repo + +from oebuild.configure import Configure +import oebuild.util as oebuild_util + +class CheckDockerTag: + ''' + This class is used to synthesize the build environment + parameters to obtain the docker image version that should + be updated when building with OEBUILD, and these environment + parameters are the docker_tag entered by the user, the env + configuration file, the yocto branch name, etc. Judge down + in turn, and finally return a suitable docker image version + ''' + def __init__(self, docker_tag: str, configure: Configure): + self.docker_tag = docker_tag + self.configure = configure + self.tag_list = [] + self._parse_docker_tag() + + def _parse_docker_tag(self,): + oebuild_config = self.configure.parse_oebuild_config() + docker_config = oebuild_config.docker + tags = {} + for tag in docker_config.tag_map.values(): + tags[tag] = True + for key in tags: + self.tag_list.append(key) + + def get_tags(self,): + return self.tag_list + + def list_image_tag(self,): + ''' + print compile docker image tag list + ''' + oebuild_config = self.configure.parse_oebuild_config() + docker_config = oebuild_config.docker + log = f'''the openeuler embedded docker image repo url: +{docker_config.repo_url} +the openeuler embedded docker tag can be selected list: +''' + for tag in self.tag_list: + log += f"{tag}\n" + print(log) + + def get_tag(self,) -> str: + + if self.docker_tag is not None and self.docker_tag != "": + if self.docker_tag not in self.tag_list: + return None + else: + return self.docker_tag + + yocto_dir = self.configure.source_yocto_dir() + env_path = os.path.join(yocto_dir,".oebuild/env.yaml") + if os.path.exists(env_path): + env_parse = oebuild_util.read_yaml(pathlib.Path(env_path)) + return env_parse['docker_tag'] + + yocto_repo = Repo.init(yocto_dir) + oebuild_config = self.configure.parse_oebuild_config() + docker_config = oebuild_config.docker + if yocto_repo.active_branch.name in docker_config.tag_map: + return docker_config.tag_map[yocto_repo.active_branch.name] + + return None diff --git a/src/oebuild/parse_compile.py b/src/oebuild/parse_compile.py index 16d46f6..e141d00 100644 --- a/src/oebuild/parse_compile.py +++ b/src/oebuild/parse_compile.py @@ -18,6 +18,7 @@ from typing import Optional import oebuild.util as oebuild_util from oebuild.ogit import OGit from oebuild.parse_template import PlatformTemplate, ParseTemplate, BUILD_IN_DOCKER +from oebuild.check_docker_tag import CheckDockerTag @dataclass class Compile(PlatformTemplate): @@ -38,6 +39,8 @@ class Compile(PlatformTemplate): tmp_dir: Optional[str] + docker_image: Optional[str] + class BaseParseCompileError(ValueError): ''' parse compile basic error @@ -69,6 +72,7 @@ class ParseCompile: except Exception as e_p: raise e_p + default_image = "swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container:latest" self.compile = Compile( build_in=BUILD_IN_DOCKER if 'build_in' not in data else data['build_in'], platform=data['platform'], @@ -82,7 +86,8 @@ class ParseCompile: not_use_repos=False if 'not_use_repos' not in data else data['not_use_repos'], repos=None if "repos" not in data else ParseTemplate.parse_oebuild_repo(data['repos']), local_conf=None if "local_conf" not in data else data['local_conf'], - layers=None if "layers" not in data else data['layers'] + layers=None if "layers" not in data else data['layers'], + docker_image=default_image if "docker_image" not in data else data['docker_image'] ) @property @@ -175,6 +180,13 @@ class ParseCompile: return attr of repos ''' return self.compile.repos + + @property + def docker_image(self): + ''' + return attr of docker_tag + ''' + return self.compile.docker_image def check_with_version(self, base_dir, manifest_path): ''' diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 921f984..e235457 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -170,7 +170,8 @@ class ParseTemplate: sstate_cache = None, tmp_dir = None, is_datetime = False, - is_disable_fetch = False): + is_disable_fetch = False, + docker_image = ""): ''' first param common yaml ''' @@ -183,6 +184,7 @@ class ParseTemplate: common_yaml_dir = pathlib.Path(common_yaml_dir) data = oebuild_util.read_yaml(common_yaml_dir) + data['docker_image'] = docker_image repos = {} if 'repos' in data : @@ -239,6 +241,7 @@ class ParseTemplate: compile_conf = { 'build_in': build_in, + 'docker_image': docker_image, 'platform': self.platform_template.platform, 'machine': self.platform_template.machine, 'toolchain_type': self.platform_template.toolchain_type} diff --git a/src/oebuild/version.py b/src/oebuild/version.py index 05f2c9a..70e29ee 100644 --- a/src/oebuild/version.py +++ b/src/oebuild/version.py @@ -10,4 +10,4 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -__version__ = '0.0.21' +__version__ = '0.0.23' -- Gitee