From 051883bed5afd057f6c51401a9270713dcd1c146 Mon Sep 17 00:00:00 2001 From: zhaohang Date: Thu, 27 Feb 2025 11:16:30 +0800 Subject: [PATCH] =?UTF-8?q?=E9=80=82=E9=85=8Dhpm=E8=A7=A3=E5=8E=8B?= =?UTF-8?q?=E6=89=93=E5=8D=B0=E4=BC=98=E5=8C=96=EF=BC=8C=E6=96=B0=E5=A2=9E?= =?UTF-8?q?keep-out=E7=AD=89=E5=8F=82=E6=95=B0=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zhaohang --- hb/resolver/indep_build_args_resolver.py | 74 ++++++++++++++- hb/resources/args/default/indepbuildargs.json | 18 +++- hb/services/hpm.py | 63 +++++++++--- hb/util/system_util.py | 95 +++++++++++++++---- indep_configs/build_indep.sh | 8 +- 5 files changed, 216 insertions(+), 42 deletions(-) diff --git a/hb/resolver/indep_build_args_resolver.py b/hb/resolver/indep_build_args_resolver.py index de805ae319..412ff88139 100644 --- a/hb/resolver/indep_build_args_resolver.py +++ b/hb/resolver/indep_build_args_resolver.py @@ -25,6 +25,9 @@ from resolver.interface.args_resolver_interface import ArgsResolverInterface from modules.interface.indep_build_module_interface import IndepBuildModuleInterface from util.component_util import ComponentUtil from exceptions.ohos_exception import OHOSException +from util.log_util import LogUtil +import subprocess +from distutils.spawn import find_executable def get_part_name(): @@ -50,6 +53,13 @@ def _search_bundle_path(part_name: str) -> str: return bundle_path +def rename_file(source_file, target_file): + try: + os.rename(source_file, target_file) + except FileNotFoundError as rename_error: + LogUtil.hb_warning(rename_error) + + class IndepBuildArgsResolver(ArgsResolverInterface): def __init__(self, args_dict: dict): @@ -188,5 +198,65 @@ class IndepBuildArgsResolver(ArgsResolverInterface): indep_build_module.indep_build.regist_flag('build-target', target_arg.arg_value) @staticmethod - def resolve_fast_rebuild(target_arg: Arg, indep_build_module: IndepBuildModuleInterface): - indep_build_module.indep_build.regist_flag('fast-rebuild', target_arg.arg_value) + def resolve_keep_out(target_arg: Arg, indep_build_module: IndepBuildModuleInterface): + indep_build_module.indep_build.regist_flag('keep-out', target_arg.arg_value) + + @staticmethod + def resolve_ccache(target_arg: Arg, indep_build_module: IndepBuildModuleInterface): + """resolve '--ccache' arg + :param target_arg: arg object which is used to get arg value. + :param build_module [maybe unused]: build module object which is used to get other services. + :phase: prebuild. + """ + if target_arg.arg_value: + ccache_path = find_executable('ccache') + if ccache_path is None: + LogUtil.hb_warning('Failed to find ccache, ccache disabled.') + return + else: + indep_build_module.indep_build.regist_arg( + 'ohos_build_enable_ccache', target_arg.arg_value) + + ccache_local_dir = os.environ.get('CCACHE_LOCAL_DIR') + ccache_base = os.environ.get('CCACHE_BASE') + if not ccache_local_dir: + ccache_local_dir = '.ccache' + if not ccache_base: + ccache_base = os.environ.get('HOME') + ccache_base = os.path.join(ccache_base, ccache_local_dir) + if not os.path.exists(ccache_base): + os.makedirs(ccache_base, exist_ok=True) + + ccache_log_suffix = os.environ.get('CCACHE_LOG_SUFFIX') + if ccache_log_suffix: + logfile = os.path.join( + ccache_base, "ccache.{}.log".format(ccache_log_suffix)) + elif os.environ.get('CCACHE_LOGFILE'): + logfile = os.environ.get('CCACHE_LOGFILE') + if not os.path.exists(os.path.dirname(logfile)): + os.makedirs(os.path.dirname(logfile), exist_ok=True) + else: + logfile = os.path.join(ccache_base, "ccache.log") + if os.path.exists(logfile): + oldfile = '{}.old'.format(logfile) + if os.path.exists(oldfile): + os.unlink(oldfile) + rename_file(logfile, oldfile) + ccache_basedir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + os.environ['CCACHE_EXEC'] = ccache_path + os.environ['CCACHE_LOGFILE'] = logfile + os.environ['USE_CCACHE'] = '1' + os.environ['CCACHE_DIR'] = ccache_base + os.environ['CCACHE_UMASK'] = '002' + os.environ['CCACHE_BASEDIR'] = ccache_basedir + ccache_max_size = os.environ.get('CCACHE_MAXSIZE') + if not ccache_max_size: + ccache_max_size = '100G' + + cmd = ['ccache', '-M', ccache_max_size] + try: + subprocess.check_output(cmd, text=True) + except FileNotFoundError: + print("错误：找不到 ccache 命令") + except subprocess.CalledProcessError as e: + print(f"执行 ccache 命令失败: {e}") \ No newline at end of file diff --git a/hb/resources/args/default/indepbuildargs.json b/hb/resources/args/default/indepbuildargs.json index 5942de9158..fbeaf1a825 100644 --- a/hb/resources/args/default/indepbuildargs.json +++ b/hb/resources/args/default/indepbuildargs.json @@ -122,14 +122,24 @@ "resolve_function": "resolve_build_target", "testFunction": "" }, - "fast_rebuild": { - "arg_name": "--fast-rebuild", + "keep_out": { + "arg_name": "--keep-out", "argDefault": false, - "arg_help": "Default:false. Help:You use this option to fast rebuild a target or a component", + "arg_help": "Default:false. Help: keep out dir", "arg_phase": "indepCompilation", "arg_type": "bool", "arg_attribute": {}, - "resolve_function": "resolve_fast_rebuild", + "resolve_function": "resolve_keep_out", + "testFunction": "" + }, + "ccache": { + "arg_name": "--ccache", + "argDefault": false, + "arg_help": "Default:false. Help:You use this option to use build cache", + "arg_phase": "indepCompilation", + "arg_type": "bool", + "arg_attribute": {}, + "resolve_function": "resolve_ccache", "testFunction": "" } } \ No newline at end of file diff --git a/hb/services/hpm.py b/hb/services/hpm.py index 998c15f51e..c71dc09339 100644 --- a/hb/services/hpm.py +++ b/hb/services/hpm.py @@ -21,6 +21,7 @@ import os import threading import subprocess import re +import sys from enum import Enum from containers.status import throw_exception @@ -142,21 +143,57 @@ class Hpm(BuildFileGeneratorInterface): hpm_update_cmd = [self.exec, "update"] + self._convert_flags() self._run_hpm_cmd(hpm_update_cmd) - '''Description: Convert all registed args into a list - @parameter: none - @return: list of all registed args - ''' - def _run_hpm_cmd(self, cmd, log_path): - ret_code = SystemUtil.exec_command(cmd, log_path=log_path, pre_msg="start run hpm command", - after_msg="end hpm command") + ret_code = SystemUtil.exec_command( + cmd, + log_path=log_path, + pre_msg="start run hpm command", + after_msg="end hpm command", + custom_line_handle=self._custom_line_handle, + ) hpm_info = get_hpm_check_info() if hpm_info: print(hpm_info) if ret_code != 0: - raise OHOSException(f'ERROR: hpm command failed, cmd: {cmd}', '0001') + raise OHOSException(f"ERROR: hpm command failed, cmd: {cmd}", "0001") + + + def _custom_line_handle(self, line): + """ + Handle the output line from the hpm command. + Args: + line (str): The output line from the hpm command. + Returns: + tuple: A tuple containing a boolean indicating whether the line should be processed, + and the processed line (or an empty string if the line should be skipped). + """ + if not hasattr(self, "custom_line_handle_preline"): + setattr(self, "custom_line_handle_preline", "") + + if line.strip() == "" and "Extracting" in self.custom_line_handle_preline: + self.custom_line_handle_preline = line + return False, "" + + if "Extracting..." in line: + if "Extracted successfully." in line: + sys.stdout.write("\r") + self.custom_line_handle_preline = line + return True, "Extracted successfully.\n" + else: + if "DISABLE_PROGRESS" not in os.environ: + sys.stdout.write(f"\r[OHOS INFO] {line.strip()}") + self.custom_line_handle_preline = line + return False, "" + else: + self.custom_line_handle_preline = line + return True, line def _convert_args(self) -> list: + ''' + Description: Convert all registed args into a list + @parameter: none + @return: list of all registed args + ''' args_list = [] for key, value in self.args_dict.items(): @@ -174,12 +211,12 @@ class Hpm(BuildFileGeneratorInterface): return args_list - '''Description: Convert all registed flags into a list - @parameter: none - @return: list of all registed flags - ''' - def _convert_flags(self) -> list: + ''' + Description: Convert all registed flags into a list + @parameter: none + @return: list of all registed flags + ''' flags_list = [] for key, value in self.flags_dict.items(): diff --git a/hb/util/system_util.py b/hb/util/system_util.py index 7c478b583d..498fec11d0 100755 --- a/hb/util/system_util.py +++ b/hb/util/system_util.py @@ -25,29 +25,88 @@ from datetime import datetime from util.log_util import LogUtil from hb.helper.no_instance import NoInstance from containers.status import throw_exception +import copy -class SystemUtil(metaclass=NoInstance): +class HandleKwargs(metaclass=NoInstance): + compile_item_pattern = re.compile(r'\[\d+/\d+\].+') + key_word_register_list = ["pre_msg", "log_stage", "after_msg", "log_filter", "custom_line_handle"] + filter_print = False + @staticmethod - def exec_command(cmd: list, log_path='out/build.log', exec_env=None, log_mode='normal', pre_msg="", after_msg="", - **kwargs): - useful_info_pattern = re.compile(r'\[\d+/\d+\].+') - is_log_filter = kwargs.pop('log_filter', False) - if log_mode == 'silent': - is_log_filter = True + def remove_registry_kwargs(kw_dict): + for item in HandleKwargs.key_word_register_list: + kw_dict.pop(item, "") + + @staticmethod + def before_msg(kw_dict): + pre_msg = kw_dict.get('pre_msg', '') + if pre_msg: + LogUtil.hb_info(pre_msg) - log_stage = kwargs.pop('log_stage', '') + @staticmethod + def set_log_stage(kw_dict): + log_stage = kw_dict.get('log_stage', '') if log_stage: LogUtil.set_stage(log_stage) + @staticmethod + def remove_useless_space(cmd): while "" in cmd: cmd.remove("") + return cmd + + @staticmethod + def after_msg(kw_dict): + after_msg = kw_dict.get('after_msg', '') + if after_msg: + LogUtil.hb_info(after_msg) + + @staticmethod + def clear_log_stage(kw_dict): + log_stage = kw_dict.get('log_stage', '') + if log_stage: + LogUtil.clear_stage() + + @staticmethod + def handle_line(line, kw_dict): + filter_function = kw_dict.get('custom_line_handle', False) + if filter_function: + return filter_function(line) + else: + return True, line + + @staticmethod + def set_filter_print(log_mode, kw_dict): + if kw_dict.get('log_filter', False) or log_mode == 'silent': + HandleKwargs.filter_print = True + + @staticmethod + def handle_print(line, log_mode): + if HandleKwargs.filter_print: + info = re.findall(HandleKwargs.compile_item_pattern, line) + if len(info): + LogUtil.hb_info(info[0], mode=log_mode) + else: + LogUtil.hb_info(line) + + +class SystemUtil(metaclass=NoInstance): + @staticmethod + def exec_command(cmd: list, log_path='out/build.log', exec_env=None, log_mode='normal', + **kwargs): + raw_kwargs = copy.deepcopy(kwargs) + HandleKwargs.remove_registry_kwargs(kwargs) + + HandleKwargs.set_log_stage(raw_kwargs) + HandleKwargs.set_filter_print(log_mode, raw_kwargs) + cmd = HandleKwargs.remove_useless_space(cmd) if not os.path.exists(os.path.dirname(log_path)): os.makedirs(os.path.dirname(log_path), exist_ok=True) + HandleKwargs.before_msg(raw_kwargs) with open(log_path, 'at', encoding='utf-8') as log_file: - LogUtil.hb_info(pre_msg) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -55,19 +114,15 @@ class SystemUtil(metaclass=NoInstance): env=exec_env, **kwargs) for line in iter(process.stdout.readline, ''): - log_file.write(line) - info = "" - if is_log_filter: - info = re.findall(useful_info_pattern, line) - else: - LogUtil.hb_info(line) - if len(info): - LogUtil.hb_info(info[0], mode=log_mode) + keep_deal, new_line = HandleKwargs.handle_line(line, raw_kwargs) + if keep_deal: + log_file.write(new_line) + HandleKwargs.handle_print(new_line, log_mode) process.wait() - LogUtil.hb_info(after_msg) - if log_stage: - LogUtil.clear_stage() + HandleKwargs.after_msg(raw_kwargs) + HandleKwargs.clear_log_stage(raw_kwargs) + ret_code = process.returncode if ret_code != 0: LogUtil.get_failed_log(log_path) diff --git a/indep_configs/build_indep.sh b/indep_configs/build_indep.sh index 3a85556aea..09e79f6d73 100755 --- a/indep_configs/build_indep.sh +++ b/indep_configs/build_indep.sh @@ -29,9 +29,11 @@ esac mkdir -p out/preloader mkdir -p out/$VARIANTS/$OUT_DIR/ -# keep the logs of hpm -find out/$VARIANTS -type f -not -name '*.log' -delete -find out/$VARIANTS -type d -empty -delete +if [[ ! "$@" =~ "--keep-out" ]]; then + # keep the logs of hpm + find out/$VARIANTS -type f -not -name '*.log' -delete + find out/$VARIANTS -type d -empty -delete +fi rm -rf out/preloader/$VARIANTS rm -rf .gn -- Gitee