From becb38fc1da5bf41fb606c88ffa26fc64838b5e2 Mon Sep 17 00:00:00 2001 From: Medvedev Aleksandr Date: Thu, 24 Jul 2025 10:05:34 +0300 Subject: [PATCH 1/3] Add koala_mirror node modules path Change-Id: Ia8b38e300b905285696c8ca18ac366510d375c7c --- prebuilts_download_config.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/prebuilts_download_config.json b/prebuilts_download_config.json index f997077d76..9ad2ed68fd 100644 --- a/prebuilts_download_config.json +++ b/prebuilts_download_config.json @@ -14,8 +14,13 @@ "arkcompiler/ets_frontend/ets2panda/bindings", "arkcompiler/runtime_core/static_core/plugins/ets/tools/declgen_ts2sts", "developtools/ace_ets2bundle/koala-wrapper", - "developtools/ace_ets2bundle/arkui-plugins" - + "developtools/ace_ets2bundle/arkui-plugins", + "foundation/arkui/ace_engine/frameworks/bridge/arkts_frontend/koala_mirror/incremental", + "foundation/arkui/ace_engine/frameworks/bridge/arkts_frontend/koala_mirror/interop", + "foundation/arkui/ace_engine/frameworks/bridge/arkts_frontend/koala_mirror/ui2abc/fast-arktsc", + "foundation/arkui/ace_engine/frameworks/bridge/arkts_frontend/koala_mirror/ui2abc/libarkts", + "foundation/arkui/ace_engine/frameworks/bridge/arkts_frontend/koala_mirror/ui2abc", + "foundation/arkui/ace_engine/frameworks/bridge/arkts_frontend/koala_mirror/arkoala-arkts" ], "uninstalled_tools": [ "prebuilts/build-tools/common/restool" -- Gitee From 1da1d479fbbfd41516621536d9f1e8789263e4cc Mon Sep 17 00:00:00 2001 From: Medvedev Aleksandr Date: Thu, 24 Jul 2025 12:57:58 +0300 Subject: [PATCH 2/3] Retry for npm install was added Change-Id: I65bfdbcab884c6493b3473f8cdfc4edf9be6d91b --- prebuilts_download.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/prebuilts_download.py b/prebuilts_download.py index 3d3489a22c..da8f5a70c4 100644 --- a/prebuilts_download.py +++ b/prebuilts_download.py @@ -272,8 +272,12 @@ def _npm_install(args): for index, proc in enumerate(procs): out, err = proc.communicate() if proc.returncode: - print("in dir:{}, executing:{}".format(full_code_paths[index], ' '.join(install_cmd[index]))) - return False, err.decode() + print("in dir:{}, executing:{}".format(full_code_paths[index], ' '.join(install_cmd))) + print("Retrying npm install in parallel mode...") + retry_proc = subprocess.Popen(install_cmds[index], cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + retry_out, retry_err = retry_proc.communicate() + if retry_proc.returncode: + return False, retry_err.decode() args.success_installed.append(full_code_paths[index]) else: @@ -283,8 +287,12 @@ def _npm_install(args): time.sleep(0.1) out, err = proc.communicate() if proc.returncode: - print("in dir:{}, executing:{}".format(full_code_paths[index], ' '.join(install_cmd[index]))) - return False, err.decode() + print("in dir:{}, executing:{}".format(full_code_paths[index], ' '.join(install_cmd))) + print("Retrying npm install in non-parallel mode...") + retry_proc = subprocess.Popen(install_cmds[index], cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + retry_out, retry_err = retry_proc.communicate() + if retry_proc.returncode: + return False, retry_err.decode() args.success_installed.append(full_code_paths[index]) return True, None -- Gitee From fb5ea35d8db42e732a81a4911fe0298d63a3b018 Mon Sep 17 00:00:00 2001 From: Medvedev Aleksandr Date: Thu, 24 Jul 2025 15:19:37 +0300 Subject: [PATCH 3/3] Refactor npm install process: add environment setup, support sequential/parallel install, and improve retry logic Change-Id: Ieb95f3b43d167e46a2fcb136858c9d26a3216107 --- prebuilts_download.py | 132 ++++++++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 57 deletions(-) diff --git a/prebuilts_download.py b/prebuilts_download.py index da8f5a70c4..207f732601 100644 --- a/prebuilts_download.py +++ b/prebuilts_download.py @@ -220,6 +220,15 @@ def _hwcloud_download(args, config: dict, bin_dir: str, code_dir: str, glibc_ver def _npm_install(args): + _setup_npm_environment(args) + install_cmds, full_code_paths = _prepare_install_commands(args) + if args.parallel_install: + return _execute_parallel_install(install_cmds, full_code_paths, args) + else: + return _execute_sequential_install(install_cmds, full_code_paths, args) + + +def _setup_npm_environment(args): node_path = 'prebuilts/build-tools/common/nodejs/current/bin' os.environ['PATH'] = '{}/{}:{}'.format(args.code_dir, node_path, os.environ.get('PATH')) npm = os.path.join(args.code_dir, node_path, 'npm') @@ -227,76 +236,85 @@ def _npm_install(args): skip_ssl_cmd = '{} config set strict-ssl false;'.format(npm) out, err, retcode = _run_cmd(skip_ssl_cmd) if retcode != 0: - return False, err.decode() + raise Exception(err.decode()) npm_clean_cmd = '{} cache clean -f'.format(npm) npm_package_lock_cmd = '{} config set package-lock true'.format(npm) - out, err, retcode = _run_cmd(npm_clean_cmd) - if retcode != 0: - return False, err.decode() - out, err, retcode = _run_cmd(npm_package_lock_cmd) - if retcode != 0: - return False, err.decode() + _run_cmd(npm_clean_cmd) + _run_cmd(npm_package_lock_cmd) + + +def _prepare_install_commands(args): install_cmds = [] full_code_paths = [] - print('start npm install, please wait.') for install_info in args.npm_install_config: full_code_path = os.path.join(args.code_dir, install_info) if full_code_path in args.success_installed: print('{} has been installed, skip'.format(full_code_path)) continue - basename = os.path.basename(full_code_path) - node_modules_path = os.path.join(full_code_path, "node_modules") - npm_cache_dir = os.path.join('~/.npm/_cacache', basename) - if os.path.exists(node_modules_path): - print('remove node_modules %s' % node_modules_path) - _run_cmd(('rm -rf {}'.format(node_modules_path))) - if os.path.exists(full_code_path): - cmd = ['timeout', '-s', '9', '90s', npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] - if args.host_platform == 'darwin': - cmd = [npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] - if args.unsafe_perm: - cmd.append('--unsafe-perm') - install_cmds.append(cmd) - full_code_paths.append(full_code_path) - else: - raise Exception("{} not exist, it shouldn't happen, pls check...".format(full_code_path)) - - if args.parallel_install: - procs = [] - for index, install_cmd in enumerate(install_cmds): - proc = subprocess.Popen(install_cmd, cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print('run npm install in {}'.format(full_code_paths[index])) - time.sleep(0.1) - procs.append(proc) - for index, proc in enumerate(procs): - out, err = proc.communicate() - if proc.returncode: - print("in dir:{}, executing:{}".format(full_code_paths[index], ' '.join(install_cmd))) - print("Retrying npm install in parallel mode...") - retry_proc = subprocess.Popen(install_cmds[index], cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - retry_out, retry_err = retry_proc.communicate() - if retry_proc.returncode: - return False, retry_err.decode() - args.success_installed.append(full_code_paths[index]) + _prepare_install_directory(full_code_path) + install_cmds.append(_build_install_command(args, full_code_path)) + full_code_paths.append(full_code_path) + return install_cmds, full_code_paths + + +def _prepare_install_directory(full_code_path): + node_modules_path = os.path.join(full_code_path, "node_modules") + if os.path.exists(node_modules_path): + print('remove node_modules %s' % node_modules_path) + _run_cmd(('rm -rf {}'.format(node_modules_path))) + if not os.path.exists(full_code_path): + raise Exception("{} not exist, it shouldn't happen, pls check...".format(full_code_path)) + + +def _build_install_command(args, full_code_path): + npm = os.path.join(args.code_dir, 'prebuilts/build-tools/common/nodejs/current/bin', 'npm') + npm_cache_dir = os.path.join('~/.npm/_cacache', os.path.basename(full_code_path)) + cmd = ['timeout', '-s', '9', '90s', npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] + if args.host_platform == 'darwin': + cmd = [npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] + if args.unsafe_perm: + cmd.append('--unsafe-perm') + return cmd + + +def _execute_parallel_install(install_cmds, full_code_paths, args): + procs = [] + for index, install_cmd in enumerate(install_cmds): + proc = subprocess.Popen(install_cmd, cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print('run npm install in {}'.format(full_code_paths[index])) + time.sleep(0.1) + procs.append(proc) + for index, proc in enumerate(procs): + if not _handle_install_process(proc, install_cmds[index], full_code_paths[index]): + return False, proc.stderr.read().decode() + args.success_installed.append(full_code_paths[index]) + return True, None - else: - for index, install_cmd in enumerate(install_cmds): - proc = subprocess.Popen(install_cmd, cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print('run npm install in {}'.format(full_code_paths[index])) - time.sleep(0.1) - out, err = proc.communicate() - if proc.returncode: - print("in dir:{}, executing:{}".format(full_code_paths[index], ' '.join(install_cmd))) - print("Retrying npm install in non-parallel mode...") - retry_proc = subprocess.Popen(install_cmds[index], cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - retry_out, retry_err = retry_proc.communicate() - if retry_proc.returncode: - return False, retry_err.decode() - args.success_installed.append(full_code_paths[index]) + +def _execute_sequential_install(install_cmds, full_code_paths, args): + for index, install_cmd in enumerate(install_cmds): + proc = subprocess.Popen(install_cmd, cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print('run npm install in {}'.format(full_code_paths[index])) + time.sleep(0.1) + if not _handle_install_process(proc, install_cmds[index], full_code_paths[index]): + return False, proc.stderr.read().decode() + args.success_installed.append(full_code_paths[index]) return True, None +def _handle_install_process(proc, install_cmd, full_code_path): + out, err = proc.communicate() + if proc.returncode: + print("in dir:{}, executing:{}".format(full_code_path, ' '.join(install_cmd))) + print("Retrying npm install...") + retry_proc = subprocess.Popen(install_cmd, cwd=full_code_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + retry_proc.communicate() + if retry_proc.returncode: + return False + return True + + def _node_modules_copy(config: dict, code_dir: str, enable_symlink: bool): for config_info in config: src_dir = os.path.join(code_dir, config_info.get('src')) @@ -456,7 +474,7 @@ def main(): retry_times = 0 max_retry_times = 2 args.success_installed = [] - args.parallel_install = True + args.parallel_install = False while retry_times <= max_retry_times: result, error = _npm_install(args) if result: -- Gitee