diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py index 89142fd9a02c4f8b3770b7fc1dd253bb8e1ee32f..616af9b75ff8d068813a99ff59af3f761eaa482f 100644 --- a/src/oebuild/app/main.py +++ b/src/oebuild/app/main.py @@ -24,7 +24,7 @@ import colorama import oebuild.util as oebuild_util from oebuild.version import __version__ from oebuild.command import ExtCommand, extension_commands -from oebuild.my_log import MyLog as log +from oebuild.m_log import logger APP = "app" diff --git a/src/oebuild/app/plugins/bitbake/base_build.py b/src/oebuild/app/plugins/bitbake/base_build.py index 545ecc2deff88310de22e3ab85bdc2be622679d3..0331cdd00b91920c1ce4cb3d896d34a08620196a 100644 --- a/src/oebuild/app/plugins/bitbake/base_build.py +++ b/src/oebuild/app/plugins/bitbake/base_build.py @@ -38,7 +38,7 @@ class BaseBuild: local_conf = LocalConf(local_conf_dir=local_dir) local_conf.update(parse_compile=parse_compile, src_dir=src_dir) - def add_bblayers(self, bblayers_dir: str, pre_dir: str, base_dir: str, layers: list): + def add_bblayers(self, bblayers_dir: str, pre_dir: str, base_dir: str, layers): ''' add_layers has two main functions, one is to initialize the compilation directory, and the other is to add the diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index f4cf0cdd352aea8af034ccd0d148d77ffde768d0..20861a6cdf8286151e061c4cb45e4b7e71b1a67d 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -19,10 +19,10 @@ from oebuild.configure import Configure from oebuild.parse_compile import ParseCompile, CheckCompileError from oebuild.parse_env import ParseEnv import oebuild.util as oebuild_util -from oebuild.my_log import MyLog as log from oebuild.app.plugins.bitbake.in_container import InContainer from oebuild.app.plugins.bitbake.in_host import InHost from oebuild.parse_template import BUILD_IN_HOST +from oebuild.m_log import logger from docker.errors import DockerException class Bitbake(OebuildCommand): @@ -70,7 +70,7 @@ class Bitbake(OebuildCommand): command = self._get_command(unknow=unknown) if not self.check_support_bitbake(): - log.warning("please do it in compile workspace which contain compile.yaml") + logger.error("please do it in compile workspace which contain compile.yaml") return if not os.path.exists('.env'): @@ -79,13 +79,13 @@ class Bitbake(OebuildCommand): try: parse_compile = ParseCompile(self.compile_conf_dir) except CheckCompileError as c_e: - log.err(str(c_e)) + logger.error(str(c_e)) return # if has manifest.yaml, init layer repo with it yocto_dir = os.path.join(self.configure.source_dir(), "yocto-meta-openeuler") manifest_path = os.path.join(yocto_dir, ".oebuild/manifest.yaml") - parse_compile.pull_repos(self.configure.source_dir(), manifest_path=manifest_path) + parse_compile.check_with_version(self.configure.source_dir(), manifest_path=manifest_path) parse_env = ParseEnv(env_dir='.env') if parse_compile.build_in == BUILD_IN_HOST: @@ -95,7 +95,7 @@ class Bitbake(OebuildCommand): try: oebuild_util.check_docker() except DockerException as d_e: - log.err(str(d_e)) + logger.error(str(d_e)) return in_container = InContainer(self.configure) in_container.exec(parse_env=parse_env, diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index cce3213994610b1c5282f97d7275cd0d969a6f84..2ff94b7cb5d51f5825b9e20b4b07c5ae7ab47ad8 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -20,7 +20,7 @@ from oebuild.docker_proxy import DockerProxy from oebuild.configure import Configure from oebuild.ogit import OGit from oebuild.parse_compile import ParseCompile -from oebuild.my_log import MyLog as log +from oebuild.m_log import logger import oebuild.app.plugins.bitbake.const as bitbake_const from oebuild.app.plugins.bitbake.base_build import BaseBuild @@ -47,6 +47,7 @@ class InContainer(BaseBuild): ''' execute bitbake command ''' + logger.info("bitbake starting ...") yocto_dir = os.path.join( self.configure.source_dir(), "yocto-meta-openeuler") remote, branch = OGit.get_repo_info(yocto_dir) @@ -87,7 +88,7 @@ class InContainer(BaseBuild): remote=remote, branch=branch, volumns=volumns, - short_id=None + short_id="" ) check_container = env.is_same_container(data=env_container) except Exception as e_p: @@ -100,26 +101,25 @@ class InContainer(BaseBuild): config = self.configure.parse_oebuild_config() container_config = config.docker # here use default mater branch bind container and fix next - image_name = container_config.repo_url + ":" + \ - container_config.tag_map.get('master') - container = self.client.container_run_simple( + image_name = f"{container_config.repo_url}:{container_config.tag_map.get('master')}" + container:Container = self.client.container_run_simple( image=image_name, - volumes=volumns) + volumes=volumns) # type: ignore env_container.short_id = container.short_id env.set_env_container(env_container) env.export_env() self.container_id = env.container.short_id - container = self.client.get_container(self.container_id) + container:Container = self.client.get_container(self.container_id) # type: ignore if not self.client.is_container_running(container): self.client.start_container(container) - def exec_compile(self, parse_compile: ParseCompile = None, command: str = "" or None): + def exec_compile(self, parse_compile: ParseCompile, command: str = ""): ''' execute compile task ''' - container = self.client.get_container(self.container_id) + container:Container = self.client.get_container(self.container_id) # type: ignore self.init_bash(container=container, build_dir=os.path.basename(os.getcwd())) @@ -127,7 +127,7 @@ class InContainer(BaseBuild): try: self.init_bitbake(container=container) except ValueError as v_e: - log.err(str(v_e)) + logger.error(str(v_e)) return # add bblayers, this action must before replace local_conf @@ -155,7 +155,7 @@ class InContainer(BaseBuild): work_space=f"/home/{bitbake_const.CONTAINER_USER}") for line in res.output: - log.info(line.decode().strip('\n')) + logger.info(line.decode().strip('\n')) else: content = self._get_bashrc_content(container=container) for b_s in bitbake_const.BASH_BANNER.split('\n'): @@ -248,7 +248,7 @@ class InContainer(BaseBuild): stream=False ) if resp.exit_code != 0: - log.info( + logger.info( "=========================install sudo===============================") self._install_software(container=container, software="sudo") @@ -270,7 +270,7 @@ class InContainer(BaseBuild): stream=True ) for line in resp.output: - log.info(line.decode().strip('\n')) + logger.info(line.decode().strip('\n')) def init_bash(self, container: Container, build_dir): ''' diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py index ad0bf1a3be43f631951df10b9ea382e411e9fb63..3041a51b843cc02490d1ac08a83d0c17e8eaf5d7 100644 --- a/src/oebuild/app/plugins/bitbake/in_host.py +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -17,7 +17,7 @@ import shutil from oebuild.local_conf import NativesdkNotExist, NativesdkNotValid from oebuild.configure import Configure from oebuild.parse_compile import ParseCompile -from oebuild.my_log import MyLog as log +from oebuild.m_log import logger import oebuild.app.plugins.bitbake.const as bitbake_const from oebuild.app.plugins.bitbake.base_build import BaseBuild @@ -53,12 +53,12 @@ class InHost(BaseBuild): local_dir=local_dir, src_dir=self.configure.source_dir()) except NativesdkNotExist as n_e: - log.err(str(n_e)) - log.err("please set valid nativesdk directory") + logger.error(str(n_e)) + logger.error("please set valid nativesdk directory") return except NativesdkNotValid as n_e: - log.err(str(n_e)) - log.err(''' + logger.error(str(n_e)) + logger.error(''' The nativesdk path must be valid, it is recommended that you download the nativesdk script and then perform initialization operations''') @@ -73,12 +73,14 @@ initialization operations''') encoding="utf-8") as s_p: if s_p.returncode is not None and s_p.returncode != 0: err_msg = '' - for line in s_p.stderr: - err_msg.join(line) - raise ValueError(err_msg) - - for line in s_p.stdout: - log.info(line.strip('\n')) + if s_p.stderr is not None: + for line in s_p.stderr: + err_msg.join(line) + raise ValueError(err_msg) + + if s_p.stdout is not None: + for line in s_p.stdout: + logger.info(line.strip('\n')) else: # run in Interactive mode banner_list = [] diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index 42997ad9fc070ffc0a1550c217762eabfb7fdc7d..d27bfc70959ccaddc4dc53b9d1a2f27cb68ceeab 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -20,7 +20,7 @@ from oebuild.command import OebuildCommand import oebuild.util as oebuild_util from oebuild.configure import Configure from oebuild.parse_template import BaseParseTemplate, ParseTemplate, BUILD_IN_DOCKER, BUILD_IN_HOST -from oebuild.my_log import MyLog as log +from oebuild.m_log import logger, INFO_COLOR class Generate(OebuildCommand): ''' @@ -107,7 +107,10 @@ class Generate(OebuildCommand): this param is add DATETIME to local.conf, the value is getting current time ''') - parser.add_argument('-df', '--disable_fetch', dest = "is_disable_fetch", action = "store_true", + parser.add_argument('-df', + '--disable_fetch', + dest = "is_disable_fetch", + action = "store_true", help=''' this param is set openeuler_fetch in local.conf, the default value is enable, if set -df, the OPENEULER_FETCH will set to 'disable' ''') @@ -124,7 +127,7 @@ class Generate(OebuildCommand): args = args.parse_args(unknown) if not self.configure.is_oebuild_dir(): - log.err('your current directory had not finishd init') + logger.error('your current directory had not finishd init') sys.exit(-1) build_in = BUILD_IN_DOCKER @@ -132,7 +135,7 @@ class Generate(OebuildCommand): try: self._check_param_in_host(args=args) except ValueError as v_e: - log.err(str(v_e)) + logger.error(str(v_e)) return self.nativesdk_dir = args.nativesdk_dir build_in = BUILD_IN_HOST @@ -148,7 +151,7 @@ class Generate(OebuildCommand): yocto_dir = self.configure.source_yocto_dir() if not self.check_support_oebuild(yocto_dir): - log.err('Currently, yocto-meta-openeuler does not support oebuild, \ + logger.error('Currently, yocto-meta-openeuler does not support oebuild, \ please modify .oebuild/config and re-execute `oebuild update`') return @@ -169,10 +172,10 @@ class Generate(OebuildCommand): yocto_oebuild_dir=yocto_oebuild_dir, parser_template=parser_template) except BaseParseTemplate as b_t: - log.err(str(b_t)) + logger.error(str(b_t)) return except ValueError as v_e: - log.err(str(v_e)) + logger.error(str(v_e)) return try: @@ -180,11 +183,11 @@ class Generate(OebuildCommand): yocto_oebuild_dir=yocto_oebuild_dir, parser_template=parser_template) except BaseParseTemplate as b_t: - log.err(str(b_t)) + logger.error(str(b_t)) self._list_feature() return except ValueError as v_e: - log.err(str(v_e)) + logger.error(str(v_e)) return if os.path.exists(os.path.join(build_dir,'compile.yaml')): @@ -201,16 +204,18 @@ class Generate(OebuildCommand): is_disable_fetch = args.is_disable_fetch )) - log.info("=============================================") - log.successful("generate compile.yaml successful") - log.info("please run `oebuild bitbake` in directory next") format_dir = f''' +generate compile.yaml successful + +please run follow command: +============================================= cd {build_dir} +oebuild bitbake +============================================= ''' - log.info(format_dir) - log.info("=============================================") + logger.info(format_dir) def _check_param_in_host(self, args): if args.toolchain_dir == '': @@ -256,7 +261,7 @@ cd {build_dir} build_dir = os.path.join(self.configure.build_dir(), args.directory) if not os.path.abspath(build_dir).startswith(self.configure.build_dir()): - log.err("build path must in oebuild workspace") + logger.error("build path must in oebuild workspace") return None if not os.path.exists(build_dir): @@ -275,35 +280,35 @@ cd {build_dir} return def _list_platform(self): - log.info("=============================================") + logger.info("=============================================") yocto_dir = self.configure.source_yocto_dir() yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") list_platform = os.listdir(os.path.join(yocto_oebuild_dir, 'platform')) - log.info("the platform list is:") + logger.info("the platform list is:") for platform in list_platform: if platform.endswith('.yml'): - log.info(platform.replace('.yml', '')) + print(platform.replace('.yml', ''), INFO_COLOR) if platform.endswith('.yaml'): - log.info(platform.replace('.yaml', '')) + print(platform.replace('.yaml', ''), INFO_COLOR) def _list_feature(self,): - log.info("=============================================") + logger.info("=============================================") yocto_dir = self.configure.source_yocto_dir() yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") list_feature = os.listdir(os.path.join(yocto_oebuild_dir, 'features')) - log.info("the feature list is:") + logger.info("the feature list is:") for feature in list_feature: if feature.endswith('.yml'): - log.info(feature.replace('.yml','')) + print(feature.replace('.yml',''), INFO_COLOR) if feature.endswith('.yaml'): - log.info(feature.replace('.yaml','')) + print(feature.replace('.yaml',''), INFO_COLOR) feat = oebuild_util.read_yaml(pathlib.Path(os.path.join(yocto_oebuild_dir, 'features', feature))) if "support" in feat: - log.info(f" support arch: {feat.get('support')}") + logger.info(" support arch: %s", feat.get('support')) else: - log.info(" support arch: all") + logger.info(" support arch: all") def check_support_oebuild(self, yocto_dir): ''' diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py index 42750062acb2183f1158e5336e1b5f87ff4334e5..87ede08a9ff952d85f65eda8494c702f753048ea 100644 --- a/src/oebuild/app/plugins/init/init.py +++ b/src/oebuild/app/plugins/init/init.py @@ -18,8 +18,8 @@ import sys from oebuild.command import OebuildCommand import oebuild.util as oebuild_util -from oebuild.configure import Configure, YOCTO_META_OPENEULER, ConfigBasicRepo, CONFIG -from oebuild.my_log import MyLog as log +from oebuild.configure import Configure, YOCTO_META_OPENEULER, ConfigBasicRepo, CONFIG, Config +from oebuild.m_log import logger class Init(OebuildCommand): ''' @@ -70,24 +70,25 @@ class Init(OebuildCommand): args = args.parse_args(unknown) if self.configure.is_oebuild_dir(): - log.err(f'The "{os.path.dirname(self.configure.oebuild_dir())}" \ - has already been initialized, please change other directory') + log = f'The "{os.path.dirname(self.configure.oebuild_dir())}" \ + has already been initialized, please change other directory' + logger.error(log) sys.exit(-1) if args.directory is None: - log.err("'oebuild init' need param directory") - log.info("\noebuild init help:") + logger.error("'oebuild init' need param directory") + logger.info("\noebuild init help:") self.print_help(iargs) return if not self.init_workspace(args.directory): - log.err(f"mkdir {args.directory} faild") + logger.error("mkdir %s faild", args.directory) return os.chdir(args.directory) - oebuild_config = self.configure.parse_oebuild_config() + oebuild_config:Config = self.configure.parse_oebuild_config() - yocto_config:ConfigBasicRepo = oebuild_config.basic_repo.get(YOCTO_META_OPENEULER) + yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[YOCTO_META_OPENEULER] if args.yocto_remote_url is not None: yocto_config.remote_url = args.yocto_remote_url if args.branch is not None: @@ -96,14 +97,14 @@ class Init(OebuildCommand): self.configure.update_oebuild_config(oebuild_config) - log.successful(f"init {args.directory} successful") + logger.info("init %s successful",args.directory) format_msg = f''' please execute the follow commands next cd {os.path.abspath(os.getcwd())} oebuild update ''' - log.info(format_msg) + print(format_msg) def init_workspace(self, directory): ''' @@ -129,8 +130,8 @@ please execute the follow commands next os.mkdir(oebuild_dir) return oebuild_dir except FileExistsError: - log.err("mkdir .oebuild faild") - return None + logger.error("mkdir .oebuild faild") + return "" @staticmethod def create_src_directory(updir : str): @@ -142,7 +143,7 @@ please execute the follow commands next os.makedirs(src_dir) return src_dir except FileExistsError: - log.err("mkdir src faild") + logger.error("mkdir src faild") return None @staticmethod @@ -154,4 +155,4 @@ please execute the follow commands next config = oebuild_util.get_config_yaml_dir() shutil.copyfile(config, os.path.join(updir, CONFIG)) except FileNotFoundError: - log.err("mkdir config faild") + logger.error("mkdir config faild") diff --git a/src/oebuild/app/plugins/manifest/manifest.py b/src/oebuild/app/plugins/manifest/manifest.py index 6ad9f95c56b01d55dd5fa6e0fb154d394361e9e1..e5502329e6e3941d61a3101fb23ec37efa0cf3d1 100644 --- a/src/oebuild/app/plugins/manifest/manifest.py +++ b/src/oebuild/app/plugins/manifest/manifest.py @@ -21,13 +21,13 @@ from threading import Thread from concurrent.futures import ThreadPoolExecutor,wait,ALL_COMPLETED import git +from git.repo import Repo from git.exc import GitCommandError from oebuild.command import OebuildCommand from oebuild.configure import Configure import oebuild.util as oebuild_util - -from oebuild.my_log import MyLog as log +from oebuild.m_log import logger class Manifest(OebuildCommand): ''' @@ -86,7 +86,7 @@ class Manifest(OebuildCommand): args = args.parse_args(unknown) if not self.configure.is_oebuild_dir(): - log.err('your current directory had not finishd init') + logger.error('your current directory had not finishd init') sys.exit(-1) if args.is_create: @@ -100,7 +100,7 @@ class Manifest(OebuildCommand): for index, repo_dir in enumerate(src_list): local_dir = os.path.join(self.configure.source_dir(), repo_dir) try: - repo = git.Repo(local_dir) + repo = Repo(local_dir) remote_url = repo.remote().url version = repo.head.commit.hexsha except git.GitError: @@ -171,7 +171,7 @@ class Manifest(OebuildCommand): def _download_repo(self, out_q: Queue ,src_dir, key, value): repo_dir = os.path.join(src_dir, key) - repo = git.Repo.init(repo_dir) + repo = Repo.init(repo_dir) remote = None for item in repo.remotes: if value['remote_url'] == item.url: diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index 0dee3b02253a853dc8b5728c7f80a7ca48122078..d1759d2baa938715a8ed35e74bde91606f6b2218 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -14,17 +14,19 @@ import argparse import os import textwrap import sys -import shutil +import pathlib from docker.errors import DockerException import oebuild.util as oebuild_util from oebuild.command import OebuildCommand +from oebuild.parse_template import OebuildRepo +from oebuild.parse_compile import ParseCompile from oebuild.configure import Configure, ConfigBasicRepo, YOCTO_META_OPENEULER from oebuild.docker_proxy import DockerProxy from oebuild.ogit import OGit -from oebuild.my_log import MyLog as log, INFO_COLOR +from oebuild.m_log import logger class Update(OebuildCommand): ''' @@ -47,27 +49,17 @@ class Update(OebuildCommand): parser = self._parser( parser_adder, usage=''' - %(prog)s [-t docker_tag] [-l list] [-i ignore] [-e enable] + %(prog)s [yocto docker layer] [-tag] ''') - parser.add_argument('-t', dest = 'docker_tag', - help='''specifying the -t parameter will update the corresponding docker image''') - - parser.add_argument('-l',dest = 'list',choices=['docker'], - help='''specifying the -l parameter lists the specified modules''') - - parser.add_argument('-i', dest='ignore', choices=['docker', 'meta'], action='append', + parser.add_argument('-tag', dest='docker_tag', default="latest", help=''' - specify the -i parameter to ignore the corresponding setting when updating, - when the -e parameter is used at the same time, the -i parameter no longer takes effect + with platform will list support archs, with feature will list support features ''' ) - parser.add_argument('-e', dest='enable', choices=['docker', 'meta'], action='append', - help=''' - specify the -e parameter to enable the corresponding setting when updating, - when the -e parameter is used at the same time, the -i parameter no longer takes effect - ''' - ) + parser.add_argument( + 'item', nargs='?', default=None, + help='''The name of the directory that will be initialized''') return parser @@ -78,27 +70,29 @@ class Update(OebuildCommand): args = args.parse_args(unknown) if not self.configure.is_oebuild_dir(): - log.err('your current directory had not finishd init') + logger.error('your current directory had not finishd init') + sys.exit(-1) + + # if args.list is not None: + # if args.list == "docker": + # self.list_image_tag() + # return + + update_yocto, update_docker, update_layer = False, False, False + if args.item is None: + update_yocto, update_docker, update_layer = True, True, True + elif args.item == "yocto": + update_yocto = True + elif args.item == "docker": + update_docker = True + elif args.item == "layer": + update_layer = True + else: + logger.error('please run oebuild update [yocto docker layer]') sys.exit(-1) - if args.list is not None: - if args.list == "docker": - self.list_image_tag() - return - update_docker, update_meta = True, True - if args.enable is not None: - if "docker" not in args.enable: - update_docker = False - if "meta" not in args.enable: - update_meta = False - elif args.ignore is not None: - if "docker" in args.ignore: - update_docker = False - if "meta" in args.ignore: - update_meta = False - - if update_meta: + if update_yocto: self.get_basic_repo() if update_docker: @@ -106,20 +100,60 @@ class Update(OebuildCommand): oebuild_util.check_docker() self.docker_image_update(args.docker_tag) except DockerException as d_e: - log.err(str(d_e)) + logger.error(str(d_e)) return + if update_layer: + self.get_layer_repo() + def list_image_tag(self,): ''' print compile docker image tag list ''' oebuild_config = self.configure.parse_oebuild_config() docker_config = oebuild_config.docker - log.info("the openeuler embedded docker image repo url:") - log.info(" " + docker_config.repo_url) - log.info("the openeuler embedded docker tag list:") + log = f''' + the openeuler embedded docker image repo url: + {docker_config.repo_url} + the openeuler embedded docker tag list: + ''' for tag in docker_config.tag_map.values(): - log.info(" "+tag) + log += f" {tag}\n" + print(log) + + def get_layer_repo(self,): + ''' + download or update layers that will be needed + ''' + # check the main layer if exists + yocto_dir = os.path.join(self.configure.source_dir(), "yocto-meta-openeuler") + if not os.path.exists(yocto_dir): + # update main layer + self.get_basic_repo() + # get rely layers from yocto-meta-openeuler/.oebuild/common.yaml when not in build directory + # or /compile.yaml where in build directory + repos = None + if os.path.exists(os.path.join(os.getcwd(), "compile.yaml")): + parse_compile = ParseCompile(compile_conf_dir=os.path.join(os.getcwd(), "compile.yaml")) + repos = parse_compile.repos + else: + common_path = pathlib.Path(os.path.join(yocto_dir, ".oebuild/common.yaml")) + repos = oebuild_util.read_yaml(yaml_dir=common_path)['repos'] + + if repos is None: + return + for _ , item in repos.items(): + if isinstance(item, OebuildRepo): + local_dir = os.path.join(self.configure.source_dir(), item.path) + key_repo = OGit(repo_dir = local_dir, + remote_url = item.url, + branch = item.refspec) + else: + local_dir = os.path.join(self.configure.source_dir(), item['path']) + key_repo = OGit(repo_dir = local_dir, + remote_url = item['url'], + branch = item['refspec']) + key_repo.clone_or_pull_repo() def get_basic_repo(self,): ''' @@ -132,26 +166,13 @@ class Update(OebuildCommand): embedded/src/yocto-meta-openeuler not exists, so just clone from config setting. ''' oebuild_config = self.configure.parse_oebuild_config() - yocto_config:ConfigBasicRepo = oebuild_config.basic_repo.get(YOCTO_META_OPENEULER) + yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[YOCTO_META_OPENEULER] local_dir = os.path.join(self.configure.source_dir(), yocto_config.path) - if os.path.exists(local_dir): - remote_url, _ = OGit.get_repo_info(local_dir) - if remote_url != yocto_config.remote_url: - if not os.path.exists(self.configure.yocto_bak_dir()): - os.makedirs(self.configure.yocto_bak_dir()) - bak_dir = os.path.join(self.configure.yocto_bak_dir(), - yocto_config.path + "_" + oebuild_util.get_time_stamp()) - log.warning(f"yocto-meta-openeuler remote is changed, \ - bak yocto-meta-openeuler to {bak_dir}") - shutil.move(local_dir, bak_dir) - - log.info(f"clone or pull {yocto_config.remote_url}:{yocto_config.branch} ...") yocto_repo = OGit(repo_dir=local_dir, remote_url=yocto_config.remote_url, branch=yocto_config.branch) yocto_repo.clone_or_pull_repo() - log.info(f"clone or pull {yocto_config.remote_url}:{yocto_config.branch} finish") def docker_image_update(self, docker_tag = None): ''' @@ -166,13 +187,13 @@ class Update(OebuildCommand): if docker_tag is not None and docker_tag not in docker_config.tag_map.values(): warn_msg = "please select valid docker_tag follow list" - log.warning(warn_msg) + print(warn_msg) for tag in docker_config.tag_map.keys(): - log.warning(docker_config.tag_map.get(tag)) + print(docker_config.tag_map.get(tag)) return if docker_tag is None: basic_config = oebuild_config.basic_repo - yocto_config: ConfigBasicRepo = basic_config.get(YOCTO_META_OPENEULER) + yocto_config: ConfigBasicRepo = basic_config[YOCTO_META_OPENEULER] if yocto_config.branch in docker_config.tag_map: docker_tag = docker_config.tag_map[yocto_config.branch] else: @@ -184,19 +205,19 @@ class Update(OebuildCommand): key_list.append(key) input_msg += "please enter index number(enter q will exit):" while True: - i = input(INFO_COLOR + input_msg) + i = input(input_msg) if i == 'q': sys.exit() i = int(i) if i <= 0 or i > len(key_list): - log.warning("enter wrong") + logger.warning("enter wrong") continue docker_tag = docker_config.tag_map[key_list[i-1]] break docker_image = docker_config.repo_url + ":" + docker_tag client = DockerProxy() - log.info(f"pull {docker_image} ...") + logger.info("pull %s ...", docker_image) client.pull_image_with_progress(docker_image) - log.info(f"finishd pull {docker_image} ...") + logger.info("finishd pull %s ...", docker_image) diff --git a/src/oebuild/bblayers.py b/src/oebuild/bblayers.py index ab7f0987fa448cc4c7596a36c9ca11781c9c1e99..d9e5541a299f532dd011ad9c87118849c2ffe1c5 100644 --- a/src/oebuild/bblayers.py +++ b/src/oebuild/bblayers.py @@ -53,15 +53,14 @@ class BBLayers: except Exception as e_p: raise e_p + bblayers = [] if isinstance(layers, str): - layers = [os.path.join(pre_dir, layers)] - else: - tmp_layers = [] + bblayers = [os.path.join(pre_dir, layers)] + if isinstance(layers, list): for layer in layers: - tmp_layers.append(os.path.join(pre_dir, layer)) - layers = tmp_layers + bblayers.append(os.path.join(pre_dir, layer)) - bb_utils.edit_bblayers_conf(self.bblayers_dir, add=layers, remove=None) + bb_utils.edit_bblayers_conf(self.bblayers_dir, add=bblayers, remove=None) def check_layer_exist(self, layers:str or list): ''' @@ -70,10 +69,13 @@ class BBLayers: args: layers (str or list): needed to add to bblayers.conf ''' + bblayers = [] if isinstance(layers, str): - layers = [layers] + bblayers.append(layers) + else: + bblayers.extend(bblayers) - for layer in layers: + for layer in bblayers: layer_dir = os.path.join(self.base_dir, layer) if not os.path.exists(layer_dir): raise ValueError("layer does not exists") diff --git a/src/oebuild/command.py b/src/oebuild/command.py index ed0076e2ae1e3aa4f9e245663465500a53268b53..a87abd3623dfc8ad94dd5bb7a457a14ab5ce0b65 100644 --- a/src/oebuild/command.py +++ b/src/oebuild/command.py @@ -211,9 +211,8 @@ def _commands_module_from_file(file, mod_name): these modules in an (otherwise unpopulated) oebuild.commands.ext package. ''' - - spec = importlib.util.spec_from_file_location(mod_name, file) - mod = importlib.util.module_from_spec(spec) + spec = importlib.util.spec_from_file_location(mod_name, file) # type: ignore + mod = importlib.util.module_from_spec(spec) # type: ignore spec.loader.exec_module(mod) return mod diff --git a/src/oebuild/configure.py b/src/oebuild/configure.py index 5abbc4ae9dd24a2e46f3fa7029d141546c3f2893..f68575446ecac0bdc08ff4a3e840090f8ea6ab4f 100644 --- a/src/oebuild/configure.py +++ b/src/oebuild/configure.py @@ -58,7 +58,7 @@ class Config: ''' docker: ConfigContainer - basic_repo: Dict + basic_repo: dict class Configure: ''' @@ -126,7 +126,7 @@ class Configure: ''' config = Configure.parse_oebuild_config() basic_config = config.basic_repo - yocto_config:ConfigBasicRepo = basic_config.get(YOCTO_META_OPENEULER) + yocto_config:ConfigBasicRepo = basic_config[YOCTO_META_OPENEULER] yocto_dir = yocto_config.path return os.path.join(Configure.source_dir(), yocto_dir) diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py index 48b2c70e22ee985422fc05749c559ce54124c2fa..9e7629002fe26f7dae87a11c11f26fcc5c3db978 100644 --- a/src/oebuild/docker_proxy.py +++ b/src/oebuild/docker_proxy.py @@ -246,14 +246,15 @@ class DockerProxy: detach=True, tty=True ) - - res = self.container_exec_command( + if isinstance(container, Container): + res = self.container_exec_command( container=container, command=command, user=user, work_space=work_space) - - return container, res.output + return container, res.output + else: + raise ValueError("docker start faild") def container_run_simple(self, image:str, volumes: list, network="host"): ''' diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index c1763c6129ec792e1d1ab5be74243e853bf4cb85..83828b9a31bf8929c8298e9340989de1ba55d4fb 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -120,6 +120,8 @@ class LocalConf: # replace nativesdk OPENEULER_SP_DIR if parse_compile.build_in == BUILD_IN_HOST: self.check_nativesdk_valid(parse_compile.nativesdk_dir) + if parse_compile.nativesdk_dir is None: + raise ValueError("please set nativesdk dir") nativesdk_sys_dir = os.path.join(parse_compile.nativesdk_dir, NATIVESDK_SYSROOT) content = match_and_replace( pre=NATIVESDK_DIR_NAME, @@ -129,7 +131,7 @@ class LocalConf: content = match_and_replace( pre=OPENEULER_SP_DIR, - new_str=OPENEULER_SP_DIR + ' = "' + src_dir + '"', + new_str= f"{OPENEULER_SP_DIR} = '{src_dir}'", content=content ) @@ -168,6 +170,8 @@ class LocalConf: content = self.replace_param(parse_compile=parse_compile, content=content) + if content is None: + return with open(local_dir, 'w', encoding="utf-8") as r_f: r_f.write(content) @@ -191,6 +195,8 @@ class LocalConf: ''' match and replace param by ParseCompile.local_conf ''' + if parse_compile.local_conf is None: + return for line in parse_compile.local_conf.split('\n'): ret = re.match(r'^([A-Z0-9_]+)(append)(\s)', line) if ret is not None: diff --git a/src/oebuild/my_log.py b/src/oebuild/m_log.py similarity index 59% rename from src/oebuild/my_log.py rename to src/oebuild/m_log.py index 54ce026075a1c5cca0b88eaf65ef428d5bc92ddb..2b6137142af9815482dc4c2a2c8b4e219573c387 100644 --- a/src/oebuild/my_log.py +++ b/src/oebuild/m_log.py @@ -10,6 +10,7 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' +import logging import colorama #: Color used (when applicable) for printing with successful() @@ -24,35 +25,25 @@ WRN_COLOR = colorama.Fore.LIGHTYELLOW_EX #: Color used (when applicable) for printing with err() and die() ERR_COLOR = colorama.Fore.LIGHTRED_EX -class MyLog: - ''' - Simple log output is implemented, including info, successful, warning, err four output types - ''' - - @staticmethod - def info(msg): - ''' - normal message print - ''' - print(INFO_COLOR + msg) - - @staticmethod - def successful(msg): - ''' - successful message print - ''' - print(SUCCESS_COLOR + msg) - - @staticmethod - def warning(msg): - ''' - warning messaage print - ''' - print(WRN_COLOR + msg) - - @staticmethod - def err(msg): - ''' - err message print - ''' - print(ERR_COLOR + msg) +# 创建logger对象 +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +# 创建文件处理器 +fh = logging.FileHandler('oebuild.log') +fh.setLevel(logging.INFO) + +# 创建控制台处理器 +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) + +# 创建格式化器 +formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + +# 将格式化器添加到文件处理器和控制台处理器 +fh.setFormatter(formatter) +ch.setFormatter(formatter) + +# 将处理器添加到logger对象 +logger.addHandler(fh) +logger.addHandler(ch) diff --git a/src/oebuild/ogit.py b/src/oebuild/ogit.py index 608fe6bb9efd7a090def1bcc74eebad2a8ca55fc..92f1466f5b2da0beb2208220fa8394e53ba4c696 100644 --- a/src/oebuild/ogit.py +++ b/src/oebuild/ogit.py @@ -13,21 +13,10 @@ See the Mulan PSL v2 for more details. import os import git -from git import GitCommandError +from git.repo import Repo +from git import GitCommandError, RemoteProgress -from oebuild.my_log import MyLog as log - -( - BEGIN, - END, - COUNTING, - COMPRESSING, - WRITING, - RECEIVING, - RESOLVING, - FINDING_SOURCES, - CHECKING_OUT, -) = [1 << x for x in range(9)] +from oebuild.m_log import logger class OGit: ''' @@ -41,7 +30,7 @@ class OGit: try: _, self._screen_width = os.popen('stty size', 'r').read().split() except ValueError as v_e: - log.warning(str(v_e)) + logger.warning(str(v_e)) @property def repo_dir(self): @@ -64,69 +53,36 @@ class OGit: ''' return self.branch - def clone_or_pull_repo(self, ): + def check_out_version(self, version): ''' - clone or pull git repo + check out version ''' - if os.path.exists(self._repo_dir): - try: - repo = git.Repo(self._repo_dir) - remote = repo.remote() - if repo.head.is_detached: - repo.git.checkout(self._branch) - if repo.active_branch.name != self._branch: - log.info(f"Fetching into '{self._repo_dir}'...") - remote.fetch(progress=self.clone_process) - repo.git.checkout(self._branch) - log.info(f"Pulling into '{self._repo_dir}'...") - remote.pull(progress=self.clone_process) - except Exception as e_p: - raise e_p - else: - try: - log.info(f"Cloning into '{self._repo_dir}'...") - git.Repo.clone_from( - url=self._remote_url, - to_path=self._repo_dir, - branch=self._branch, - progress=self.clone_process) - except Exception as e_p: - raise e_p + repo = Repo.init(self._repo_dir) + try: + repo.git.checkout(version) + except GitCommandError: + logger.error("checkout faild") - def clone_process(self, op_code, cur_count, max_count, message): + def clone_or_pull_repo(self,): ''' - print clone or pull progress + clone or pull git repo ''' - if op_code % 2 == BEGIN: - print("") - return - op_title = '' - pmsg = '' - if op_code == COUNTING: - op_title = "remote: Counting objects" - pmsg = f"{op_title}: {int(cur_count/max_count*100)}% \ - ({cur_count}/{max_count}), {message}" - - elif op_code == COMPRESSING: - op_title = "remote: Compressing objects" - pmsg = f"{op_title}: {int(cur_count/max_count*100)}% \ - ({cur_count}/{max_count}), {message}" - - elif op_code == RECEIVING: - op_title = "Receiving objects" - pmsg = f"{op_title}: {int(cur_count/max_count*100)}% \ - ({cur_count}/{max_count}), {message}" - - elif op_code == RESOLVING: - op_title = "Resolving deltas" - pmsg = f"{op_title}: {int(cur_count/max_count*100)}% ({cur_count}/{max_count})" - else: - return - - pmsg = "\r" + pmsg - if hasattr(self, '_screen_width'): - pmsg = pmsg.ljust(int(self._screen_width), ' ') - print(pmsg, end='', flush=True) + repo = Repo.init(self._repo_dir) + remote = None + for item in repo.remotes: + if self._remote_url == item.url: + remote = item + else: + continue + if remote is None: + remote_name = "upstream" + remote = git.Remote.add(repo=repo, name=remote_name, url=self._remote_url) + logger.info("Fetching into %s ...", self._repo_dir) + remote.fetch(progress=CustomRemote()) + try: + repo.git.checkout(self._branch) + except GitCommandError: + logger.error("update faild") @staticmethod def get_repo_info(repo_dir: str): @@ -134,34 +90,47 @@ class OGit: return git repo info: remote_url, branch ''' try: - repo = git.Repo(repo_dir) + repo = Repo(repo_dir) remote_url = repo.remote().url branch = repo.active_branch.name return remote_url, branch except TypeError: - return remote_url, '' + return "", '' except git.GitError: return "","" except ValueError: return "","" - def clone_or_pull_with_version(self, version, depth): +class CustomRemote(git.RemoteProgress): + ''' + Rewrote RemoteProgress to show the process of code updates + ''' + def update(self, op_code, cur_count, max_count=None, message=''): ''' - clone or pull with version and depth + rewrote update function ''' - repo = git.Repo.init(self._repo_dir) - remote = None - for item in repo.remotes: - if self._remote_url == item.url: - remote = item - else: - continue - if remote is None: - remote_name = "manifest" - remote = git.Remote.add(repo = repo, name = remote_name, url = self._remote_url) - log.info(f"Pulling into '{self._repo_dir}'...") - try: - repo.git.checkout(version) - except GitCommandError: - remote.fetch(version, depth = depth, progress=self.clone_process) - repo.git.checkout(version) + def print_progress(op_title, cur_count, max_count, message): + percent_done = int(cur_count / max_count * 100) + if percent_done == 100: + message = "done" + pmsg = f"{op_title}: {percent_done}% ({cur_count}/{max_count}), {message}" + print(pmsg, end="\r") + + if op_code % 2 == RemoteProgress.BEGIN: + print("") + elif op_code == RemoteProgress.COUNTING: + op_title = "remote: Counting objects" + print_progress(op_title, cur_count, max_count, message) + elif op_code == RemoteProgress.COMPRESSING: + op_title = "remote: Compressing objects" + print_progress(op_title, cur_count, max_count, message) + elif op_code == RemoteProgress.RECEIVING: + op_title = "Receiving objects" + print_progress(op_title, cur_count, max_count, message) + elif op_code == RemoteProgress.RESOLVING: + op_title = "Resolving deltas" + print_progress(op_title, cur_count, max_count, message) + elif op_code and RemoteProgress.END == 2: + print("") + else: + return diff --git a/src/oebuild/parse_compile.py b/src/oebuild/parse_compile.py index 5e4f3889f2b31932f92c6511969336f4a5a47f28..385d9524bd712ba03e8a287c1e4ac1b6f8a01ba1 100644 --- a/src/oebuild/parse_compile.py +++ b/src/oebuild/parse_compile.py @@ -13,29 +13,30 @@ See the Mulan PSL v2 for more details. import os from dataclasses import dataclass import pathlib +from typing import Optional import oebuild.util as oebuild_util from oebuild.ogit import OGit -from oebuild.parse_template import PlatformTemplate,ParseTemplate,BUILD_IN_DOCKER +from oebuild.parse_template import PlatformTemplate, ParseTemplate, BUILD_IN_DOCKER @dataclass class Compile(PlatformTemplate): ''' Compile is the parsed object of compile.yaml and is used to manipulate the build file ''' - toolchain_dir: str + toolchain_dir: Optional[str] - nativesdk_dir: str + nativesdk_dir: Optional[str] not_use_repos: bool build_in: str - sstate_cache: str + sstate_cache: Optional[str] - sstate_dir: str + sstate_dir: Optional[str] - tmp_dir: str + tmp_dir: Optional[str] class BaseParseCompileError(ValueError): ''' @@ -53,10 +54,6 @@ class ParseCompile: download the relevant code repository ''' def __init__(self, compile_conf_dir): - self.compile = None - self.init_parse(compile_conf_dir) - - def init_parse(self, compile_conf_dir): ''' The initialization operation is used to parse the compile.yaml file and perform a series of checks before parsing @@ -86,7 +83,7 @@ class ParseCompile: repos=None if "repos" not in data else ParseTemplate.parse_oebuild_repo(data['repos']), local_conf=None if "local_conf" not in data else data['local_conf'], layers=None if "layers" not in data else data['layers'] - ) + ) @property def build_in(self): @@ -172,7 +169,14 @@ class ParseCompile: ''' return self.compile.tmp_dir - def pull_repos(self, base_dir, manifest_path): + @property + def repos(self): + ''' + return attr of repos + ''' + return self.compile.repos + + def check_with_version(self, base_dir, manifest_path): ''' Download the repos set in compile.yaml based on the given base path ''' @@ -181,17 +185,19 @@ class ParseCompile: manifest = oebuild_util.read_yaml(pathlib.Path(manifest_path))['manifest_list'] if not self.compile.not_use_repos: repos = self.compile.repos + if repos is None: + return for repo_local, repo in repos.items(): repo_dir = os.path.join(base_dir, repo.path) - try: + if not os.path.exists(repo_dir): + repo_git = OGit(repo_dir=repo_dir, remote_url=repo.url, branch=repo.refspec) + repo_git.clone_or_pull_repo() + if manifest is None: + continue + if repo_local in manifest: repo_git = OGit(repo_dir=repo_dir, remote_url=repo.url, branch=repo.refspec) - if manifest is not None and repo_local in manifest: - repo_item = manifest[repo_local] - repo_git.clone_or_pull_with_version(version=repo_item['version'], depth=1) - else: - repo_git.clone_or_pull_repo() - except Exception as e_p: - raise e_p + repo_item = manifest[repo_local] + repo_git.check_out_version(version=repo_item['version']) @staticmethod def check_compile_conf(data): diff --git a/src/oebuild/parse_env.py b/src/oebuild/parse_env.py index 4bf638c8faa00dd68da855f371d8fd5c1fce5b39..4cb482c8aebf809263819a9565173a8bab9c00d7 100644 --- a/src/oebuild/parse_env.py +++ b/src/oebuild/parse_env.py @@ -24,7 +24,7 @@ class EnvContainer: branch: str - short_id: str + short_id: str or None volumns: list @@ -33,7 +33,7 @@ class Env: ''' the env object ''' - container: EnvContainer + container: EnvContainer or None class ParseEnv: ''' @@ -42,8 +42,8 @@ class ParseEnv: ''' def __init__(self, env_dir): self.env_dir = pathlib.Path(env_dir) if isinstance(env_dir, str) else env_dir - self.env = Env(container=None) - self.parse_env() + self.env = Env + self._parse_env() @property def container(self): @@ -52,7 +52,7 @@ class ParseEnv: ''' return self.env.container - def parse_env(self): + def _parse_env(self): ''' parse env.yaml to env object ''' @@ -126,7 +126,7 @@ class ParseEnv: oebuild_util.write_yaml(pathlib.Path(self.env_dir), data=data) @staticmethod - def check_env_container(env_container: EnvContainer): + def check_env_container(env_container): ''' Check that the env.yaml content is compliant ''' diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 00a6eae67ef23ad18540738960c12345913f359e..4a5464f32083eefe49edb339f375c85aaaacb7d4 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -11,7 +11,7 @@ See the Mulan PSL v2 for more details. ''' from dataclasses import dataclass -from typing import Dict +from typing import Dict, Optional import pathlib import os import time @@ -48,11 +48,11 @@ class Template: ''' basic template for paltform and feature ''' - repos: Dict[str, 'OebuildRepo'] + repos: Optional[Dict[str, 'OebuildRepo']] - layers: list + layers: Optional[list] - local_conf: LiteralScalarString + local_conf: Optional[LiteralScalarString] @dataclass class PlatformTemplate(Template): @@ -133,7 +133,7 @@ class ParseTemplate: config_name = os.path.basename(config_dir) if config_type == PLATFORM: self.platform_template = PlatformTemplate( - platform=os.path.splitext(config_name)[0], + platform=LiteralScalarString(os.path.splitext(config_name)[0]), machine=data['machine'], toolchain_type=data['toolchain_type'], repos=repo_dict, @@ -153,10 +153,10 @@ class ParseTemplate: application support archs') self.feature_template.append(FeatureTemplate( - feature_name=os.path.splitext(config_name)[0], + feature_name=LiteralScalarString(os.path.splitext(config_name)[0]), repos=repo_dict, support=support_arch, - local_conf=local_conf, + local_conf=LiteralScalarString(local_conf), layers=layers )) diff --git a/src/oebuild/version.py b/src/oebuild/version.py index e18c9dbdcb1ed8ce4597379f8664d85d348f48f9..3723df2a8a8d401982bdffde9129823e69eae17e 100644 --- a/src/oebuild/version.py +++ b/src/oebuild/version.py @@ -10,4 +10,4 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -__version__ = '0.0.15' +__version__ = '0.0.16'