From a268943c0c9083a42e4a9977291e291024aff006 Mon Sep 17 00:00:00 2001 From: liutuantuan Date: Tue, 15 Jul 2025 11:30:50 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E9=A2=84=E4=B8=8B=E8=BD=BD=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: liutuantuan --- hb/services/prebuilts.py | 80 ++++++++++++++++++++++++++++- indep_configs/build_indep.sh | 1 - prebuilts_config.py | 2 +- prebuilts_service/README_zh.md | 53 ++++++++++--------- prebuilts_service/common_utils.py | 34 +++++++++++++ prebuilts_service/config_parser.py | 5 +- prebuilts_service/operater.py | 82 ++++++++++++++++++++++-------- 7 files changed, 206 insertions(+), 51 deletions(-) diff --git a/hb/services/prebuilts.py b/hb/services/prebuilts.py index 20d9c6c48d..a77a4324dd 100644 --- a/hb/services/prebuilts.py +++ b/hb/services/prebuilts.py @@ -22,16 +22,24 @@ from services.interface.build_file_generator_interface import ( BuildFileGeneratorInterface, ) from util.log_util import LogUtil +import os +import json +import time class PreuiltsService(BuildFileGeneratorInterface): def __init__(self): + ohos_dir = self.get_ohos_dir() + self.last_update = os.path.join(ohos_dir, "prebuilts/.local_data/last_update.json") super().__init__() def run(self): - if not "--open-prebuilts" in sys.argv: + if not "--enable-prebuilts" in sys.argv: return + if not self.check_whether_need_update(): + LogUtil.hb_info("you have already execute prebuilts download step and no configs changed, skip this step") + return flags_list = self._convert_flags() if "--skip-prebuilts" in flags_list: print("Skip preuilts download") @@ -49,6 +57,7 @@ class PreuiltsService(BuildFileGeneratorInterface): subprocess.run( cmd, check=True, stdout=None, stderr=None # 直接输出到终端 ) # 直接输出到终端 + self.write_last_update({"last_update": time.time()}) except subprocess.CalledProcessError as e: print(f"{cmd} execute failed: {e.returncode}") raise e @@ -75,3 +84,72 @@ class PreuiltsService(BuildFileGeneratorInterface): flags_list.append(f"--{key}") flags_list.extend(self.flags_dict[key]) return flags_list + + def check_whether_need_update(self) -> bool: + last_update = self.read_last_update().get("last_update", 0) + if not last_update: + LogUtil.hb_info("No last update record found, will update prebuilts") + return True + else: + if self.check_file_changes(): + LogUtil.hb_info("Prebuilts config file has changed, will update prebuilts") + return True + else: + return False + + def read_last_update(self): + if not os.path.exists(self.last_update): + return {} + try: + with open(self.last_update, 'r') as f: + return json.load(f) + except Exception as e: + LogUtil.hb_error(f"Failed to read last update file: {e}") + return {} + + def write_last_update(self, data): + os.makedirs(os.path.dirname(self.last_update), exist_ok=True) + try: + with open(self.last_update, 'w') as f: + json.dump(data, f, indent=4) + except Exception as e: + LogUtil.hb_error(f"Failed to write last update file: {e}") + + def get_ohos_dir(self): + cur_dir = os.getcwd() + while cur_dir != "/": + global_var = os.path.join( + cur_dir, 'build', 'hb', 'resources', 'global_var.py') + if os.path.exists(global_var): + return cur_dir + cur_dir = os.path.dirname(cur_dir) + raise Exception("you must run this script in ohos dir") + + def get_preguilt_download_related_files_mtimes(self) -> dict: + dir_path = os.path.join(self.get_ohos_dir(), "build/prebuilts_service") + mtimes = {} + for root, _, files in os.walk(dir_path): + for file in files: + file_path = os.path.join(root, file) + mtimes[file_path] = os.path.getmtime(file_path) + prebuilts_config_json_path = os.path.join(self.get_ohos_dir(), "build/prebuilts_config.json") + prebuilts_config_py_path = os.path.join(self.get_ohos_dir(), "build/prebuilts_config.py") + prebuilts_config_shell_path = os.path.join(self.get_ohos_dir(), "build/prebuilts_config.sh") + mtimes.update({prebuilts_config_json_path: os.path.getmtime(prebuilts_config_json_path)}) + mtimes.update({prebuilts_config_py_path: os.path.getmtime(prebuilts_config_py_path)}) + mtimes.update({prebuilts_config_shell_path: os.path.getmtime(prebuilts_config_shell_path)}) + return mtimes + + def check_file_changes(self) -> bool: + """ + check if the directory has changed by comparing file modification times. + :param dir_path: directory + :param prev_mtimes: last known modification times of files in the directory + :return: if the directory has changed, and the current modification times of files in the directory + """ + last_update = self.read_last_update().get("last_update", 0) + current_mtimes = self.get_preguilt_download_related_files_mtimes() + for _, mtime in current_mtimes.items(): + if mtime > last_update: + return True + return False \ No newline at end of file diff --git a/indep_configs/build_indep.sh b/indep_configs/build_indep.sh index 27f29303f0..62850edde4 100755 --- a/indep_configs/build_indep.sh +++ b/indep_configs/build_indep.sh @@ -103,5 +103,4 @@ rm -rf .gn ln -s build/core/gn/dotfile.gn .gn echo -e "\033[0;33myou can use --skip-download to skip download binary dependencies while using hb build command\033[0m" - exit 0 diff --git a/prebuilts_config.py b/prebuilts_config.py index 2df03e2a7f..c3ad3886ef 100644 --- a/prebuilts_config.py +++ b/prebuilts_config.py @@ -56,7 +56,7 @@ def main(): global_args._create_default_https_context = ssl._create_unverified_context global_args.code_dir = get_code_dir() - config_file = os.path.join(global_args.code_dir,"build", "prebuilts_config.json") + config_file = os.path.join(global_args.code_dir, "build", "prebuilts_config.json") if global_args.config_file: config_file = global_args.config_file diff --git a/prebuilts_service/README_zh.md b/prebuilts_service/README_zh.md index 64615a0223..544e6ec608 100644 --- a/prebuilts_service/README_zh.md +++ b/prebuilts_service/README_zh.md @@ -3,8 +3,8 @@ 1. [核心配置说明](#section-download-core-01) 2. [基础配置示例](#section-download-basic-demo) 3. [高级配置示例](#section-download-advanced-demo) -- [后续处理配置](#advanced-process) -- [变量查找规则](#value-search) +- [处理配置](#advanced-process) +- [变量处理](#value-search) ## 工具下载配置 下载配置用于配置下载和解压参数 @@ -12,8 +12,8 @@ |参数|描述| |--|--| -remote_url|远程包下载地址（HTTP/HTTPS）| -unzip_dir|解压目标路径（绝对或相对路径）| +remote_url|远程包下载地址| +unzip_dir|解压目标路径| unzip_filename|解压后的顶层目录名（用于版本管理和旧文件清理）| ### 基础配置示例 @@ -55,8 +55,8 @@ unzip_filename|解压后的顶层目录名（用于版本管理和旧文件清 ``` -#### 场景3：跨平台配置 -若工具包同时兼容多操作系统和CPU架构，配置进一步简化： +#### 场景3：平台无关配置 +若工具包和平台无关，配置进一步简化： ```json { "name": "ark_js_prebuilts", @@ -81,17 +81,17 @@ unzip_filename|解压后的顶层目录名（用于版本管理和旧文件清 "x86_64": [ { "remote_url": "/openharmony/compiler/clang/15.0.4-3cec00/ohos_arm64/clang_ohos-arm64-3cec00-20250320.tar.gz", - "unzip_dir": "${code_dir}/prebuilts/clang/ohos/ohos-arm64"， + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/ohos-arm64", "unzip_filename": "llvm", }, { "remote_url": "/openharmony/compiler/clang/15.0.4-3cec00/windows/clang_windows-x86_64-3cec00-20250320.tar.gz", - "unzip_dir": "${code_dir}/prebuilts/clang/ohos/windows-x86_64"， + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/windows-x86_64", "unzip_filename": "llvm", }, { "remote_url": "/openharmony/compiler/clang/15.0.4-3cec00/linux/clang_linux-x86_64-3cec00-20250320.tar.gz", - "unzip_dir": "${code_dir}/prebuilts/clang/ohos/linux-x86_64"， + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/linux-x86_64", "unzip_filename": "llvm", } ] @@ -102,8 +102,8 @@ unzip_filename|解压后的顶层目录名（用于版本管理和旧文件清 -#### 使用公共变量 -当配置中存在值相同的配置项时，可提取公共变量避免冗余：
+#### 使用公共配置 +当配置中存在值相同的配置项时，可提取公共配置避免冗余：
**原始冗余配置** ```json { @@ -156,21 +156,24 @@ unzip_filename|解压后的顶层目录名（用于版本管理和旧文件清 #### 配置继承规则 - 工具配置会继承全局配置 - 平台配置会继承工具配置 -- 内部配置优于继承配置 - -## 后续处理配置 -工具下载解压完成后可能需要进行后续处理，该部分在handle中配置，handle是一个列表，其中的每一项都代表一个操作 +- 存在相同配置项时，内部配置会覆盖继承的配置 +#### 说明 +- 全局配置在工具配置的外层定义 +- 平台配置在config里面定义 +- 除config和handle，都属于工具配置 + +## 处理配置 +部分工具在下载解压完成后需要进行额外的处理，这些处理操作可以在handle中定义，handle会在下载解压完成后执行，若没有下载解压操作，handle则会直接执行。handle是一个列表，其中的每一项都代表一个操作 ### handle配置特点 -- 顺序执行：操作按配置顺序依次执行。 -- 变量继承：操作中可引用config和外部的配置参数 -- 灵活控制：可通过handle_index指定执行的操作序号。 -- 容错机制：若操作中的变量解析失败，跳过当前操作。 +- 顺序执行：操作项按配置顺序依次执行 +- 使用变量：操作中可使用外部变量 +- 灵活控制：平台配置中可通过指定handle_index，定制操作序列 +- 容错机制：若操作中的变量解析失败，跳过当前操作 ### 公共操作列表 |操作类型|参数|用途| |-|-|-| -|download| remote_url: 远程下载地
unzip_dir: 本地解压目
unzip_filename: 用于哈希校验和清理
**注：该操作通常而言无需显示声明，脚本会根据平台配置的remote_url自动生成对应的下载作 **| 下载和解压 | |symlink| src: 链接源
dest: 目的链接地址| 生成符号链接 |copy | src: 源
dest: 目的| 复制文件或文件夹 | |remove | path:要删除的路径, 可以是字符串，也可以是一个列表 | 删除文件或文件夹 | @@ -207,9 +210,9 @@ unzip_filename|解压后的顶层目录名（用于版本管理和旧文件清 ``` -## 变量查找规则 +## 变量处理 - 变量只能使用${var_name}的方式指定 -- 工具配置可以使用自身以及全局配置中的变量 -- 平台配置可以使用自身、工具以及全局配置中的变量 -- handle可以使用自身、平台、工具以及全局配置中的变量 -- 变量只会解析一次，采取就近解析原则 +- 工具配置可以使用自身内部以及全局配置中的变量 +- 平台配置可以使用自身内部、工具以及全局配置中的变量 +- handl中的操作项可以使用自身内部、平台、工具以及全局配置中的变量 +- 变量解析优先级为：自身内部配置 > 平台配置 > 工具配置 > 全局配置 diff --git a/prebuilts_service/common_utils.py b/prebuilts_service/common_utils.py index acb566c456..8af62b9561 100644 --- a/prebuilts_service/common_utils.py +++ b/prebuilts_service/common_utils.py @@ -20,6 +20,7 @@ import pathlib import time import json import importlib +import re def get_code_dir(): @@ -52,6 +53,12 @@ def import_rich_module(): return progress +def save_data(file_path: str, data): + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "w") as f: + json.dump(data, f, indent=4) + + def load_config(config_file: str): with open(config_file, "r", encoding="utf-8") as r: config = json.load(r) @@ -145,6 +152,33 @@ def is_system_component() -> bool: ) +def check_hpm_version(hpm_path: str, npm_path: str) -> bool: + if not os.path.exists(hpm_path): + print(f"hpm not found at {hpm_path}, now install.") + return False + local_hpm_version = subprocess.run([hpm_path, "-V"], capture_output=True, text=True).stdout.strip() + cmd = npm_path + " search hpm-cli --registry https://registry.npmjs.org/" + cmd = cmd.split() + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + out, _ = proc.communicate(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + if proc.returncode == 0: + latest_hpm_version = "" + pattern = r'^@ohos/hpm-cli\s*\|(?:[^|]*\|){3}([^|]*)' + for line in out.splitlines(): + match = re.match(pattern, line) + if match: + latest_hpm_version = match.group(1).strip() + break + if latest_hpm_version and latest_hpm_version == local_hpm_version: + print(f"local hpm version: {local_hpm_version}, remote latest hpm version: {latest_hpm_version}") + return True + print(f"local hpm version: {local_hpm_version}, remote latest hpm version: {latest_hpm_version}") + return False + + def install_hpm_in_other_platform(name: str, operate: dict): download_dir = operate.get("download_dir") package_path = operate.get("package_path") diff --git a/prebuilts_service/config_parser.py b/prebuilts_service/config_parser.py index 38a488786f..2db4c1653f 100644 --- a/prebuilts_service/config_parser.py +++ b/prebuilts_service/config_parser.py @@ -156,7 +156,10 @@ class ConfigParser: class Filter: - def __init__(self, configs=[]): + def __init__(self, configs): + if configs is None: + self.input_configs = [] + return self.input_configs = copy.deepcopy(configs) @classmethod diff --git a/prebuilts_service/operater.py b/prebuilts_service/operater.py index 5e11411ae2..f227607141 100644 --- a/prebuilts_service/operater.py +++ b/prebuilts_service/operater.py @@ -25,38 +25,72 @@ from common_utils import ( install_hpm_in_other_platform, npm_install, is_system_component, + check_hpm_version, + save_data, + load_config, ) import re +from collections import OrderedDict class OperateHanlder: global_args = None @staticmethod - def run(operate_list: list, global_args, unchanged_list: tuple = ()): - ignore_list = [] - OperateHanlder.global_args = global_args - pre_process_tool = "" - for operate in operate_list: + def process_step(process_item: str, step_list: list, unchanged_list: list, processed_dict: dict): + process_result_file = os.path.join(OperateHanlder.global_args.code_dir, "prebuilts/.local_data/processed.json") + for step in step_list: try: - current_tool = re.match(r"(.*)_\d$", operate.get("step_id")).group(1) - shot_name = re.sub(r"(\.[A-Za-z]+)+$", "", current_tool).strip("_") - - if current_tool != pre_process_tool: - print(f"\n==> process {shot_name}") - pre_process_tool = current_tool - - if current_tool in ignore_list: - continue - - getattr(OperateHanlder, "_" + operate.get("type"))(operate) + getattr(OperateHanlder, "_" + step.get("type"))(step) except Exception as e: - if current_tool in unchanged_list: - ignore_list.append(current_tool) - print(f"<== ignore process {shot_name}") + # if the process item is already being processed, but not being recorded(that means prebuilts/processed.json is not exist), + # in this situation, we just check if the process item is in unchanged_list, + # if it is, then we don't need to process it again, we can just mark it as processed. + if process_item in unchanged_list: + processed_dict[process_item] = True + break + # If an error occurs, save the processed status + processed_dict[process_item] = False + save_data(process_result_file, processed_dict) + raise e + + @staticmethod + def run(operate_list: list, global_args, unchanged_list: tuple = ()): + OperateHanlder.global_args = global_args + # read and reset processed record + process_result_file = os.path.join(global_args.code_dir, "prebuilts/.local_data/processed.json") + if os.path.exists(process_result_file): + processed_dict = load_config(process_result_file) + else: + processed_dict = dict() + for key in processed_dict.keys(): + if key not in unchanged_list: + processed_dict[key] = False + + # group operate_list by process item + item_steps_dict = OrderedDict() + for current_operate in operate_list: + # current_process_item = ${tool_name}_${tar_name} + current_process_item = re.match(r"(.*)_\d$", current_operate.get("step_id")).group(1) + if current_process_item not in item_steps_dict: + item_steps_dict[current_process_item] = [current_operate] + else: + item_steps_dict[current_process_item].append(current_operate) + + # process each item + for process_item, step_list in item_steps_dict.items(): + process_item_without_suffix = re.sub(r"(\.[A-Za-z]+)+$", "", process_item).strip("_") + # If the process item is in unchanged_list and has been processed, skip it + if process_item in unchanged_list: + if process_item in processed_dict and processed_dict[process_item]: + print(f"==> {process_item_without_suffix} is unchanged, skip") continue - else: - raise e + print(f"\n==> process {process_item_without_suffix}") + processed_dict[process_item] = False + OperateHanlder.process_step(process_item, step_list, unchanged_list, processed_dict) + processed_dict[process_item] = True + # save the processed status of each item + save_data(process_result_file, processed_dict) @staticmethod def _symlink(operate: dict): @@ -109,9 +143,13 @@ class OperateHanlder: @staticmethod def _hpm_download(operate: dict): + hpm_path = os.path.join(OperateHanlder.global_args.code_dir, "prebuilts/hpm/node_modules/.bin/hpm") + npm_tool_path = os.path.join(OperateHanlder.global_args.code_dir, "prebuilts/build-tools/common/nodejs/current/bin/npm") + if check_hpm_version(hpm_path, npm_tool_path): + print("hpm version is ok, skip hpm download") + return name = operate.get("name") download_dir = operate.get("download_dir") - npm_tool_path = os.path.join(OperateHanlder.global_args.code_dir, "prebuilts/build-tools/common/nodejs/current/bin/npm") symlink_dest = operate.get("symlink") if "@ohos/hpm-cli" == name: install_hpm(npm_tool_path, download_dir) -- Gitee From 810cde9f4f6ef0d9c052db24b063f992a03c43ed Mon Sep 17 00:00:00 2001 From: liutuantuan Date: Wed, 16 Jul 2025 09:36:46 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E9=A2=84=E4=B8=8B=E8=BD=BD=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: liutuantuan --- hb/services/hpm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hb/services/hpm.py b/hb/services/hpm.py index e621bfc12f..4cd75bd49a 100644 --- a/hb/services/hpm.py +++ b/hb/services/hpm.py @@ -46,9 +46,9 @@ class Hpm(BuildFileGeneratorInterface): def __init__(self): super().__init__() - self._regist_hpm_path() def run(self): + self._regist_hpm_path() self.execute_hpm_cmd(CMDTYPE.BUILD) @throw_exception -- Gitee From 0dccd6ec2ce676cfa2479312a5ef5ccc4e934770 Mon Sep 17 00:00:00 2001 From: liutuantuan Date: Wed, 16 Jul 2025 17:22:54 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E5=AE=9A=E4=B9=89=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: liutuantuan --- hb/services/prebuilts.py | 48 +++++++++++++++++------------------ prebuilts_service/operater.py | 1 - 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/hb/services/prebuilts.py b/hb/services/prebuilts.py index a77a4324dd..9690887c42 100644 --- a/hb/services/prebuilts.py +++ b/hb/services/prebuilts.py @@ -62,29 +62,6 @@ class PreuiltsService(BuildFileGeneratorInterface): print(f"{cmd} execute failed: {e.returncode}") raise e - def _get_part_names(self): - part_name_list = [] - if len(sys.argv) > 2 and not sys.argv[2].startswith("-"): - for name in sys.argv[2:]: - if not name.startswith("-"): - part_name_list.append(name) - else: - break - return part_name_list - - def _convert_flags(self) -> list: - flags_list = [] - for key in self.flags_dict.keys(): - if isinstance(self.flags_dict[key], bool) and self.flags_dict[key]: - flags_list.append(f"--{key}") - if isinstance(self.flags_dict[key], str) and self.flags_dict[key]: - flags_list.append(f"--{key}") - flags_list.append(f"{self.flags_dict[key]}") - if isinstance(self.flags_dict[key], list) and self.flags_dict[key]: - flags_list.append(f"--{key}") - flags_list.extend(self.flags_dict[key]) - return flags_list - def check_whether_need_update(self) -> bool: last_update = self.read_last_update().get("last_update", 0) if not last_update: @@ -152,4 +129,27 @@ class PreuiltsService(BuildFileGeneratorInterface): for _, mtime in current_mtimes.items(): if mtime > last_update: return True - return False \ No newline at end of file + return False + + def _get_part_names(self): + part_name_list = [] + if len(sys.argv) > 2 and not sys.argv[2].startswith("-"): + for name in sys.argv[2:]: + if not name.startswith("-"): + part_name_list.append(name) + else: + break + return part_name_list + + def _convert_flags(self) -> list: + flags_list = [] + for key in self.flags_dict.keys(): + if isinstance(self.flags_dict[key], bool) and self.flags_dict[key]: + flags_list.append(f"--{key}") + if isinstance(self.flags_dict[key], str) and self.flags_dict[key]: + flags_list.append(f"--{key}") + flags_list.append(f"{self.flags_dict[key]}") + if isinstance(self.flags_dict[key], list) and self.flags_dict[key]: + flags_list.append(f"--{key}") + flags_list.extend(self.flags_dict[key]) + return flags_list \ No newline at end of file diff --git a/prebuilts_service/operater.py b/prebuilts_service/operater.py index a7df87e7d6..f27787efd0 100644 --- a/prebuilts_service/operater.py +++ b/prebuilts_service/operater.py @@ -72,7 +72,6 @@ class OperateHanlder: # group operate_list by process item item_steps_dict = OrderedDict() for current_operate in operate_list: - # current_process_item = ${tool_name}_${tar_name} current_process_item = re.match(r"(.*)_\d$", current_operate.get("step_id")).group(1) if current_process_item not in item_steps_dict: item_steps_dict[current_process_item] = [current_operate] -- Gitee