diff --git a/hb/resolver/indep_build_args_resolver.py b/hb/resolver/indep_build_args_resolver.py index de805ae319febc9f4eabc2b9f0fbea39e41cfc76..f8a012e4b67747bd196171ba158e1e872be4215e 100644 --- a/hb/resolver/indep_build_args_resolver.py +++ b/hb/resolver/indep_build_args_resolver.py @@ -25,6 +25,9 @@ from resolver.interface.args_resolver_interface import ArgsResolverInterface from modules.interface.indep_build_module_interface import IndepBuildModuleInterface from util.component_util import ComponentUtil from exceptions.ohos_exception import OHOSException +from util.log_util import LogUtil +import subprocess +from distutils.spawn import find_executable def get_part_name(): @@ -50,6 +53,13 @@ def _search_bundle_path(part_name: str) -> str: return bundle_path +def rename_file(source_file, target_file): + try: + os.rename(source_file, target_file) + except FileNotFoundError as rename_error: + LogUtil.hb_warning(rename_error) + + class IndepBuildArgsResolver(ArgsResolverInterface): def __init__(self, args_dict: dict): @@ -188,5 +198,91 @@ class IndepBuildArgsResolver(ArgsResolverInterface): indep_build_module.indep_build.regist_flag('build-target', target_arg.arg_value) @staticmethod - def resolve_fast_rebuild(target_arg: Arg, indep_build_module: IndepBuildModuleInterface): - indep_build_module.indep_build.regist_flag('fast-rebuild', target_arg.arg_value) + def resolve_keep_out(target_arg: Arg, indep_build_module: IndepBuildModuleInterface): + indep_build_module.indep_build.regist_flag('keep-out', target_arg.arg_value) + + @staticmethod + def resolve_ccache(target_arg: Arg, indep_build_module: IndepBuildModuleInterface): + # 检查是否启用了 ccache + if target_arg.arg_value: + # 查找 ccache 可执行文件的路径 + ccache_path = find_executable('ccache') + # 如果未找到 ccache 可执行文件，打印警告信息并返回 + if ccache_path is None: + LogUtil.hb_warning('Failed to find ccache, ccache disabled.') + return + else: + # 注册 ccache 启用标志 + indep_build_module.indep_build.regist_arg( + 'ohos_build_enable_ccache', target_arg.arg_value) + + # 获取 ccache 本地目录环境变量 + ccache_local_dir = os.environ.get('CCACHE_LOCAL_DIR') + # 获取 ccache 基础目录环境变量 + ccache_base = os.environ.get('CCACHE_BASE') + # 如果未设置 ccache 本地目录，使用默认值 + if not ccache_local_dir: + ccache_local_dir = '.ccache' + # 如果未设置 ccache 基础目录，使用用户主目录 + if not ccache_base: + ccache_base = os.environ.get('HOME') + # 拼接 ccache 基础目录 + ccache_base = os.path.join(ccache_base, ccache_local_dir) + # 如果 ccache 基础目录不存在，则创建它 + if not os.path.exists(ccache_base): + os.makedirs(ccache_base, exist_ok=True) + + # 获取 ccache 日志后缀环境变量 + ccache_log_suffix = os.environ.get('CCACHE_LOG_SUFFIX') + if ccache_log_suffix: + # 如果设置了日志后缀，拼接日志文件路径 + logfile = os.path.join( + ccache_base, "ccache.{}.log".format(ccache_log_suffix)) + elif os.environ.get('CCACHE_LOGFILE'): + # 如果设置了日志文件环境变量，使用该环境变量的值 + logfile = os.environ.get('CCACHE_LOGFILE') + # 如果日志文件所在目录不存在，则创建它 + if not os.path.exists(os.path.dirname(logfile)): + os.makedirs(os.path.dirname(logfile), exist_ok=True) + else: + # 如果未设置日志后缀和日志文件环境变量，使用默认日志文件路径 + logfile = os.path.join(ccache_base, "ccache.log") + # 如果日志文件已存在，将其重命名为旧文件 + if os.path.exists(logfile): + oldfile = '{}.old'.format(logfile) + # 如果旧文件已存在，删除它 + if os.path.exists(oldfile): + os.unlink(oldfile) + # 重命名日志文件 + rename_file(logfile, oldfile) + # 获取项目根目录 + ccache_basedir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + # 设置 ccache 可执行文件路径环境变量 + os.environ['CCACHE_EXEC'] = ccache_path + # 设置 ccache 日志文件路径环境变量 + os.environ['CCACHE_LOGFILE'] = logfile + # 设置启用 ccache 环境变量 + os.environ['USE_CCACHE'] = '1' + # 设置 ccache 缓存目录环境变量 + os.environ['CCACHE_DIR'] = ccache_base + # 设置 ccache 权限掩码环境变量 + os.environ['CCACHE_UMASK'] = '002' + # 设置 ccache 基础目录环境变量 + os.environ['CCACHE_BASEDIR'] = ccache_basedir + # 获取 ccache 最大缓存大小环境变量 + ccache_max_size = os.environ.get('CCACHE_MAXSIZE') + # 如果未设置最大缓存大小，使用默认值 + if not ccache_max_size: + ccache_max_size = '100G' + + # 构建设置 ccache 最大缓存大小的命令 + cmd = ['ccache', '-M', ccache_max_size] + try: + # 执行命令 + subprocess.check_output(cmd, text=True) + except FileNotFoundError: + # 如果找不到 ccache 命令，打印错误信息 + print("错误：找不到 ccache 命令") + except subprocess.CalledProcessError as e: + # 如果命令执行失败，打印错误信息 + print(f"执行 ccache 命令失败: {e}") \ No newline at end of file diff --git a/hb/resources/args/default/indepbuildargs.json b/hb/resources/args/default/indepbuildargs.json index 5942de91584cb478e7893341383ada1bce88ad75..fbeaf1a8252d620acdea15e05c7b3885c5d30a8d 100644 --- a/hb/resources/args/default/indepbuildargs.json +++ b/hb/resources/args/default/indepbuildargs.json @@ -122,14 +122,24 @@ "resolve_function": "resolve_build_target", "testFunction": "" }, - "fast_rebuild": { - "arg_name": "--fast-rebuild", + "keep_out": { + "arg_name": "--keep-out", "argDefault": false, - "arg_help": "Default:false. Help:You use this option to fast rebuild a target or a component", + "arg_help": "Default:false. Help: keep out dir", "arg_phase": "indepCompilation", "arg_type": "bool", "arg_attribute": {}, - "resolve_function": "resolve_fast_rebuild", + "resolve_function": "resolve_keep_out", + "testFunction": "" + }, + "ccache": { + "arg_name": "--ccache", + "argDefault": false, + "arg_help": "Default:false. Help:You use this option to use build cache", + "arg_phase": "indepCompilation", + "arg_type": "bool", + "arg_attribute": {}, + "resolve_function": "resolve_ccache", "testFunction": "" } } \ No newline at end of file diff --git a/hb/services/hpm.py b/hb/services/hpm.py index 998c15f51e93aa4945f162da745524f90b70f896..c71dc09339a6ae7609c3231a3a381e64bd3e55dc 100644 --- a/hb/services/hpm.py +++ b/hb/services/hpm.py @@ -21,6 +21,7 @@ import os import threading import subprocess import re +import sys from enum import Enum from containers.status import throw_exception @@ -142,21 +143,57 @@ class Hpm(BuildFileGeneratorInterface): hpm_update_cmd = [self.exec, "update"] + self._convert_flags() self._run_hpm_cmd(hpm_update_cmd) - '''Description: Convert all registed args into a list - @parameter: none - @return: list of all registed args - ''' - def _run_hpm_cmd(self, cmd, log_path): - ret_code = SystemUtil.exec_command(cmd, log_path=log_path, pre_msg="start run hpm command", - after_msg="end hpm command") + ret_code = SystemUtil.exec_command( + cmd, + log_path=log_path, + pre_msg="start run hpm command", + after_msg="end hpm command", + custom_line_handle=self._custom_line_handle, + ) hpm_info = get_hpm_check_info() if hpm_info: print(hpm_info) if ret_code != 0: - raise OHOSException(f'ERROR: hpm command failed, cmd: {cmd}', '0001') + raise OHOSException(f"ERROR: hpm command failed, cmd: {cmd}", "0001") + + + def _custom_line_handle(self, line): + """ + Handle the output line from the hpm command. + Args: + line (str): The output line from the hpm command. + Returns: + tuple: A tuple containing a boolean indicating whether the line should be processed, + and the processed line (or an empty string if the line should be skipped). + """ + if not hasattr(self, "custom_line_handle_preline"): + setattr(self, "custom_line_handle_preline", "") + + if line.strip() == "" and "Extracting" in self.custom_line_handle_preline: + self.custom_line_handle_preline = line + return False, "" + + if "Extracting..." in line: + if "Extracted successfully." in line: + sys.stdout.write("\r") + self.custom_line_handle_preline = line + return True, "Extracted successfully.\n" + else: + if "DISABLE_PROGRESS" not in os.environ: + sys.stdout.write(f"\r[OHOS INFO] {line.strip()}") + self.custom_line_handle_preline = line + return False, "" + else: + self.custom_line_handle_preline = line + return True, line def _convert_args(self) -> list: + ''' + Description: Convert all registed args into a list + @parameter: none + @return: list of all registed args + ''' args_list = [] for key, value in self.args_dict.items(): @@ -174,12 +211,12 @@ class Hpm(BuildFileGeneratorInterface): return args_list - '''Description: Convert all registed flags into a list - @parameter: none - @return: list of all registed flags - ''' - def _convert_flags(self) -> list: + ''' + Description: Convert all registed flags into a list + @parameter: none + @return: list of all registed flags + ''' flags_list = [] for key, value in self.flags_dict.items(): diff --git a/hb/util/system_util.py b/hb/util/system_util.py index 7c478b583d23d00507507ca447a56df345ad18ab..498fec11d0e43a3f1baf71d50eadf8f6d7c37f0f 100755 --- a/hb/util/system_util.py +++ b/hb/util/system_util.py @@ -25,29 +25,88 @@ from datetime import datetime from util.log_util import LogUtil from hb.helper.no_instance import NoInstance from containers.status import throw_exception +import copy -class SystemUtil(metaclass=NoInstance): +class HandleKwargs(metaclass=NoInstance): + compile_item_pattern = re.compile(r'\[\d+/\d+\].+') + key_word_register_list = ["pre_msg", "log_stage", "after_msg", "log_filter", "custom_line_handle"] + filter_print = False + @staticmethod - def exec_command(cmd: list, log_path='out/build.log', exec_env=None, log_mode='normal', pre_msg="", after_msg="", - **kwargs): - useful_info_pattern = re.compile(r'\[\d+/\d+\].+') - is_log_filter = kwargs.pop('log_filter', False) - if log_mode == 'silent': - is_log_filter = True + def remove_registry_kwargs(kw_dict): + for item in HandleKwargs.key_word_register_list: + kw_dict.pop(item, "") + + @staticmethod + def before_msg(kw_dict): + pre_msg = kw_dict.get('pre_msg', '') + if pre_msg: + LogUtil.hb_info(pre_msg) - log_stage = kwargs.pop('log_stage', '') + @staticmethod + def set_log_stage(kw_dict): + log_stage = kw_dict.get('log_stage', '') if log_stage: LogUtil.set_stage(log_stage) + @staticmethod + def remove_useless_space(cmd): while "" in cmd: cmd.remove("") + return cmd + + @staticmethod + def after_msg(kw_dict): + after_msg = kw_dict.get('after_msg', '') + if after_msg: + LogUtil.hb_info(after_msg) + + @staticmethod + def clear_log_stage(kw_dict): + log_stage = kw_dict.get('log_stage', '') + if log_stage: + LogUtil.clear_stage() + + @staticmethod + def handle_line(line, kw_dict): + filter_function = kw_dict.get('custom_line_handle', False) + if filter_function: + return filter_function(line) + else: + return True, line + + @staticmethod + def set_filter_print(log_mode, kw_dict): + if kw_dict.get('log_filter', False) or log_mode == 'silent': + HandleKwargs.filter_print = True + + @staticmethod + def handle_print(line, log_mode): + if HandleKwargs.filter_print: + info = re.findall(HandleKwargs.compile_item_pattern, line) + if len(info): + LogUtil.hb_info(info[0], mode=log_mode) + else: + LogUtil.hb_info(line) + + +class SystemUtil(metaclass=NoInstance): + @staticmethod + def exec_command(cmd: list, log_path='out/build.log', exec_env=None, log_mode='normal', + **kwargs): + raw_kwargs = copy.deepcopy(kwargs) + HandleKwargs.remove_registry_kwargs(kwargs) + + HandleKwargs.set_log_stage(raw_kwargs) + HandleKwargs.set_filter_print(log_mode, raw_kwargs) + cmd = HandleKwargs.remove_useless_space(cmd) if not os.path.exists(os.path.dirname(log_path)): os.makedirs(os.path.dirname(log_path), exist_ok=True) + HandleKwargs.before_msg(raw_kwargs) with open(log_path, 'at', encoding='utf-8') as log_file: - LogUtil.hb_info(pre_msg) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -55,19 +114,15 @@ class SystemUtil(metaclass=NoInstance): env=exec_env, **kwargs) for line in iter(process.stdout.readline, ''): - log_file.write(line) - info = "" - if is_log_filter: - info = re.findall(useful_info_pattern, line) - else: - LogUtil.hb_info(line) - if len(info): - LogUtil.hb_info(info[0], mode=log_mode) + keep_deal, new_line = HandleKwargs.handle_line(line, raw_kwargs) + if keep_deal: + log_file.write(new_line) + HandleKwargs.handle_print(new_line, log_mode) process.wait() - LogUtil.hb_info(after_msg) - if log_stage: - LogUtil.clear_stage() + HandleKwargs.after_msg(raw_kwargs) + HandleKwargs.clear_log_stage(raw_kwargs) + ret_code = process.returncode if ret_code != 0: LogUtil.get_failed_log(log_path) diff --git a/indep_configs/build_indep.sh b/indep_configs/build_indep.sh index 3a85556aea88f31e1832330a3ff540e0dc725f0e..09e79f6d730188ec8296a563c6681d019936363b 100755 --- a/indep_configs/build_indep.sh +++ b/indep_configs/build_indep.sh @@ -29,9 +29,11 @@ esac mkdir -p out/preloader mkdir -p out/$VARIANTS/$OUT_DIR/ -# keep the logs of hpm -find out/$VARIANTS -type f -not -name '*.log' -delete -find out/$VARIANTS -type d -empty -delete +if [[ ! "$@" =~ "--keep-out" ]]; then + # keep the logs of hpm + find out/$VARIANTS -type f -not -name '*.log' -delete + find out/$VARIANTS -type d -empty -delete +fi rm -rf out/preloader/$VARIANTS rm -rf .gn diff --git a/indep_configs/scripts/gn_ninja_cmd.py b/indep_configs/scripts/gn_ninja_cmd.py index e4eb48ae57b7cce4fcaf1770c33c9eaaed0341ed..b72f85289e83f3b43c2ae67406b61ab1f267dbb8 100644 --- a/indep_configs/scripts/gn_ninja_cmd.py +++ b/indep_configs/scripts/gn_ninja_cmd.py @@ -17,7 +17,7 @@ import sys import argparse import os import json -from utils import get_json, get_ninja_args, get_gn_args +from utils import get_json, get_ninja_args, get_gn_args, is_enable_ccache, print_ccache_stats, clean_ccache_info sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -82,6 +82,8 @@ def _gn_cmd(root_path, variants, out_dir, test_filter): _features_info = _get_all_features_info(root_path, variants) _args_list = [f"ohos_indep_compiler_enable=true", f"product_name=\"{variants}\""] _args_list.extend(_features_info) + if is_enable_ccache(): + _args_list.append(f"ohos_build_enable_ccache=true") # Add 'use_thin_lto=false' to _args_list if test_filter equals 2 if test_filter in (1, 2): @@ -114,7 +116,9 @@ def _exec_cmd(root_path, variants, out_dir, test_filter): gn_cmd = _gn_cmd(root_path, variants, out_dir, test_filter) _run_cmd(gn_cmd) ninja_cmd = _ninja_cmd(root_path, variants, out_dir) + clean_ccache_info() _run_cmd(ninja_cmd) + print_ccache_stats() def main(): diff --git a/indep_configs/scripts/utils.py b/indep_configs/scripts/utils.py index 30b74d51948a53aa60ac18c32360ab599e83d321..8697939c87ed6fa651696edf4c14b5a825245274 100755 --- a/indep_configs/scripts/utils.py +++ b/indep_configs/scripts/utils.py @@ -15,7 +15,7 @@ import json import os -import stat +import subprocess def get_json(file_path): @@ -88,3 +88,55 @@ def get_build_target(): if data["build_target"]["argDefault"]: input_build_target.extend(data["build_target"]["argDefault"]) return input_build_target + + +def is_enable_ccache(): + """ + 检查是否启用了ccache。 + :return: 如果启用了ccache，则返回True；否则返回False。 + """ + # Read the independent build parameters JSON file and get the value of the "ccache" key's "argDefault" + return get_indep_args()["ccache"]["argDefault"] + + + +def print_ccache_stats(): + """ + 打印ccache的统计信息。 + 如果ccache已启用，则执行ccache -s命令并打印输出。 + 如果ccache命令未找到或执行失败，则打印相应的错误消息。 + """ + # Check if ccache is enabled + if is_enable_ccache(): + try: + # Execute the 'ccache -s' command and capture the output + output = subprocess.check_output(["ccache", "-s"], text=True) + # Print the header for ccache hit statistics + print("ccache hit statistics:") + # Print the output of the 'ccache -s' command + print(output) + except FileNotFoundError: + # Print an error message if the 'ccache' command is not found + print("Error: ccache command not found") + except subprocess.CalledProcessError as e: + # Print an error message if the 'ccache -s' command fails + print(f"Failed to execute ccache command: {e}") + + +def clean_ccache_info(): + """ + 清除ccache的统计信息。 + 如果ccache已启用，则执行ccache -z命令以重置ccache的统计信息。 + 如果ccache命令未找到或执行失败，则打印相应的错误消息。 + """ + # Check if ccache is enabled + if is_enable_ccache(): + try: + # Execute the 'ccache -z' command to reset the ccache statistics + subprocess.check_output(["ccache", "-z"], text=True) + except FileNotFoundError: + # Print an error message if the 'ccache' command is not found + print("Error: ccache command not found") + except subprocess.CalledProcessError as e: + # Print an error message if the 'ccache -z' command fails + print(f"Failed to execute ccache command: {e}")