diff --git a/prebuilts_config.json b/prebuilts_config.json index 2ae636310e7dd97128df8e10afcecb6dfd05d70d..9a6371dbbf1a891318a9e7b420e5a96edfe6c537 100644 --- a/prebuilts_config.json +++ b/prebuilts_config.json @@ -1,409 +1,399 @@ { - "npm": [ - { - "name": "@ohos/hpm-cli", - "download": { - "version": "1.6.1", - "package_path": "", - "package_lock_path": "", - "download_dir": ".prebuilts_cache/hpm", - "symlink": "prebuilts/hpm/node_modules" - } - } - ], - "tar": [ - { - "name": "nodejs", - "tag": "base", - "download": [ + "download_root": "${code_dir}/../openharmony_prebuilts", + "tool_list": [ { - "target_os": "linux", - "target_cpu": "", - "version": "v14.21.1", - "url": "nodejs/v14.21.1/node-v14.21.1-linux-x64.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/common/nodejs", - "symlink": "prebuilts/build-tools/common/nodejs/node-v14.21.1-linux-x64", - "type": "dir" + "name": "ark_js_prebuilts", + "tag": "base", + "type": "src, indep", + "config": { + "linux": { + "arm64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_20231215.tar.gz" + }, + "x86_64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_20230713.tar.gz" + } + }, + "darwin": { + "arm64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_darwin_arm64_20230209.tar.gz" + }, + "x86_64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_darwin_x64_20230209.tar.gz" + } + } + }, + "unzip_dir": "${code_dir}/prebuilts/ark_tools", + "unzip_filename": "ark_js_prebuilts" }, { - "target_os": "darwin", - "target_cpu": "", - "version": "v14.21.1", - "url": "nodejs/v14.21.1/node-v14.21.1-darwin-x64.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/common/nodejs", - "symlink": "prebuilts/build-tools/common/nodejs/node-v14.21.1-darwin-x86", - "type": "dir" - } - ] - }, - { - "name": "gn", - "tag": "base", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "20250509", - "url": "openharmony/compiler/gn/20250509/gn-linux-x86-20250509.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/linux-x86/bin/gn", - "symlink": "prebuilts/build-tools/linux-x86/bin/gn", - "type": "file" - }, - { - "target_os": "linux", - "target_cpu": "arm64", - "version": "20240510", - "url": "openharmony/compiler/gn/20240510/gn-linux-x86-20240510.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/linux-x86/bin/gn", - "symlink": "prebuilts/build-tools/linux-x86/bin/gn", - "type": "file" - }, - { - "target_os": "darwin", - "target_cpu": "x86_64", - "version": "20240416", - "url": "openharmony/compiler/gn/20240416/gn-darwin-x86-20240416.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/darwin-x86/bin/gn", - "symlink": "prebuilts/build-tools/darwin-x86/bin/gn", - "type": "file" - }, - { - "target_os": "darwin", - "target_cpu": "arm64", - "version": "20240416", - "url": "openharmony/compiler/gn/20240416/gn-darwin-x86-20240416.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/darwin-x86/bin/gn", - "symlink": "prebuilts/build-tools/darwin-x86/bin/gn", - "type": "file" - } - ] - }, - { - "name": "packing_tool", - "tag": "base", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "20241022", - "url": "harmonyos/compiler/packing_tool/packing_tool_libs_20250215.zip", - "download_dir": ".prebuilts_cache/packing_tool", - "symlink": "prebuilts/packing_tool", - "type": "dir" - }, - { - "target_os": "linux", - "target_cpu": "arm64", - "version": "20241022", - "url": "harmonyos/compiler/packing_tool/packing_tool_libs_20250215.zip", - "download_dir": ".prebuilts_cache/packing_tool", - "symlink": "prebuilts/packing_tool", - "type": "dir" - }, - { - "target_os": "darwin", - "target_cpu": "x86_64", - "version": "20241022", - "url": "/harmonyos/compiler/packing_tool/packing_tool_libs_20250215.zip", - "download_dir": ".prebuilts_cache/packing_tool", - "symlink": "prebuilts/packing_tool", - "type": "dir" - }, - { - "target_os": "darwin", - "target_cpu": "arm64", - "version": "20241022", - "url": "/harmonyos/compiler/packing_tool/packing_tool_libs_20250215.zip", - "download_dir": ".prebuilts_cache/packing_tool", - "symlink": "prebuilts/packing_tool", - "type": "dir" - } - ] - }, - { - "name": "ninja", - "tag": "base", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "20240416", - "url": "openharmony/compiler/ninja/20240416/ninja-linux-x86-20240416.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/linux-x86/bin/ninja", - "symlink": "prebuilts/build-tools/linux-x86/bin/ninja", - "type": "file" + "name": "llvm", + "tag": "base", + "type": "src, indep", + "unzip_filename": "llvm", + "config": { + "linux": { + "arm64": { + "remote_url": "/openharmony/compiler/clang/15.0.4-da9259/linux_aarch64/clang_linux_aarch64-da9259-20250425.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/linux-aarch64", + "rename_src": "${unzip_dir}/clang_linux_aarch64-da9259-20250425", + "type": "src" + }, + "x86_64": [ + { + "remote_url": "/openharmony/compiler/clang/15.0.4-da9259/ohos_arm64/clang_ohos-arm64-da9259-20250425.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/ohos-arm64", + "rename_src": "${unzip_dir}/clang_ohos-arm64-da9259-20250425", + "type": "src" + }, + { + "remote_url": "/openharmony/compiler/clang/15.0.4-da9259/windows/clang_windows-x86_64-da9259-20250425.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/windows-x86_64", + "rename_src": "${unzip_dir}/clang_windows-x86_64-da9259-20250425", + "type": "src" + }, + { + "remote_url": "/openharmony/compiler/clang/15.0.4-da9259/linux/clang_linux-x86_64-da9259-20250425.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/linux-x86_64", + "rename_src": "${unzip_dir}/clang_linux-x86_64-da9259-20250425" + } + ] + }, + "darwin": { + "arm64": { + "remote_url": "/openharmony/compiler/clang/15.0.4-da9259/darwin_arm64/clang_darwin-arm64-da9259-20250425.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/darwin-arm64", + "rename_src": "${unzip_dir}/clang_darwin-arm64-da9259-20250425" + }, + "x86_64": { + "remote_url": "/openharmony/compiler/clang/15.0.4-da9259/darwin_x86_64/clang_darwin-x86_64-da9259-20250425.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/darwin-x86_64", + "rename_src": "${unzip_dir}/clang_darwin-x86_64-da9259-20250425" + } + } + }, + "handle": [ + { + "type": "move", + "src": "${rename_src}", + "dest": "${unzip_dir}/llvm" + }, + { + "type": "symlink", + "src": "${unzip_dir}/llvm/lib/clang/15.0.4", + "dest": "${unzip_dir}/llvm/lib/clang/current" + } + ] }, { - "target_os": "linux", - "target_cpu": "arm64", - "version": "20240416", - "url": "openharmony/compiler/ninja/20240416/ninja-linux-x86-20240416.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/linux-x86/bin/ninja", - "symlink": "prebuilts/build-tools/linux-x86/bin/ninja", - "type": "file" + "name": "gn", + "tag": "base", + "type": "src, indep", + "unzip_filename": "gn", + "config": { + "linux": { + "x86_64": { + "remote_url": "/openharmony/compiler/gn/20250509/gn-linux-x86-20250509.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/linux-x86/bin" + }, + "arm64": { + "remote_url": "/openharmony/compiler/gn/20240530/gn-linux-aarch64-20240530.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/linux-aarch64/bin" + } + }, + "darwin": { + "x86_64, arm64": { + "remote_url": "/openharmony/compiler/gn/20240416/gn-darwin-x86-20240416.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/darwin-x86/bin" + } + } + } }, { - "target_os": "darwin", - "target_cpu": "x86_64", - "version": "1.11.0", - "url": "openharmony/compiler/ninja/1.11.0/darwin/ninja-darwin-x86-1.11.0.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/darwin-x86/bin/ninja", - "symlink": "prebuilts/build-tools/darwin-x86/bin/ninja", - "type": "file" + "name": "hc-gen", + "tag": "base", + "type": "src, indep", + "config": { + "linux": { + "x86_64": { + "remote_url": "/openharmony/compiler/hc-gen/linux/hc-gen-20240926.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/hc_gen", + "unzip_filename": "hc-gen" + } + } + } }, { - "target_os": "darwin", - "target_cpu": "arm64", - "version": "1.11.0", - "url": "openharmony/compiler/ninja/1.11.0/darwin/ninja-darwin-x86-1.11.0.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/darwin-x86/bin/ninja", - "symlink": "prebuilts/build-tools/darwin-x86/bin/ninja", - "type": "file" + "name": "OpenHarmonyApplication.pem", + "tag": "base", + "type": "indep", + "config": { + "linux": { + "x86_64": { + "remote_url": "/openharmony/compiler/certificates/20241025/OpenHarmonyApplication.zip", + "unzip_dir": "${code_dir}/prebuilts/ohos-sdk", + "unzip_filename": "OpenHarmonyApplication.pem" + } + } + } }, { - "target_os": "ohos", - "target_cpu": "arm64", - "version": "1.12.0", - "url": "openharmony/compiler/ninja/1.12.0/ohos/ninja-ohos-1.12.0-20240827.tar.gz", - "download_dir": ".prebuilts_cache/build-tools/ohos/bin/ninja", - "symlink": "prebuilts/build-tools/ohos/bin/ninja", - "type": "file" - } - ] - }, - { - "name": "llvm", - "tag": "base", - "download": [ - { - "target_os": "linux", - "target_cpu": "arm64", - "version": "20250425", - "url": "openharmony/compiler/clang/15.0.4-da9259/linux/clang_linux-x86_64-da9259-20250425.tar.gz", - "download_dir": ".prebuilts_cache/clang/ohos/linux-x86_64/llvm", - "symlink": "prebuilts/clang/ohos/linux-x86_64/llvm", - "type": "dir", - "copy_src":"lib/clang/15.0.4", - "copy_dest":"lib/clang/current" + "name": "ninja", + "tag": "base", + "type": "src, indep", + "unzip_filename": "ninja", + "config": { + "linux": { + "arm64": { + "remote_url": "/openharmony/compiler/ninja/1.12.0/linux/ninja-linux-aarch64-1.12.0-20241210.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/linux-aarch64/bin" + }, + "x86_64": [ + { + "remote_url": "/openharmony/compiler/ninja/1.12.0/linux/ninja-linux-x86-1.12.0-20240523.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/linux-x86/bin" + }, + { + "remote_url": "/openharmony/compiler/ninja/1.12.0/windows/ninja-windows-x86-1.12.0-20240523.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/windows-x86/bin", + "type": "src" + }, + { + "remote_url": "/openharmony/compiler/ninja/1.12.0/ohos/ninja-ohos-1.12.0-20240827.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/ohos/bin", + "type": "src" + } + ] + }, + "darwin": { + "arm64": { + "remote_url": "/openharmony/compiler/ninja/1.12.0/darwin/arm/ninja-darwin-arm-1.12.0-20240829.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/darwin-x86/bin" + }, + "x86_64": { + "remote_url": "/openharmony/compiler/ninja/1.12.0/darwin/x86/ninja-darwin-x86-1.12.0-20240829.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/build-tools/darwin-x86/bin" + } + } + } }, { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "20250425", - "url": "openharmony/compiler/clang/15.0.4-da9259/linux/clang_linux-x86_64-da9259-20250425.tar.gz", - "download_dir": ".prebuilts_cache/clang/ohos/linux-x86_64/llvm", - "symlink": "prebuilts/clang/ohos/linux-x86_64/llvm", - "type": "dir", - "copy_src":"lib/clang/15.0.4", - "copy_dest":"lib/clang/current" + "name": "node", + "tag": "base", + "type": "src, indep", + "unzip_dir": "${code_dir}/prebuilts/build-tools/common/nodejs", + "config": { + "linux": { + "arm64": { + "remote_url": "/openharmony/compiler/nodejs/v14.21.1/node-v14.21.1-linux-aarch64.tar.gz", + "unzip_filename": "node-v14.21.1-linux-aarch64", + "symlink_src": "${code_dir}/prebuilts/build-tools/common/nodejs/node-v14.21.1-linux-aarch64" + }, + "x86_64": [ + { + "remote_url": "/nodejs/v14.21.1/node-v14.21.1-linux-x64.tar.gz", + "unzip_filename": "node-v14.21.1-linux-x64", + "symlink_src": "${code_dir}/prebuilts/build-tools/common/nodejs/node-v14.21.1-linux-x64" + }, + { + "remote_url": "/nodejs/v16.20.2/node-v16.20.2-linux-x64.tar.gz", + "unzip_filename": "node-v16.20.2-linux-x64" + } + ] + }, + "darwin": [ + { + "remote_url": "/nodejs/v14.21.1/node-v14.21.1-darwin-x64.tar.gz", + "unzip_filename": "node-v14.21.1-darwin-x64", + "symlink_src": "${code_dir}/prebuilts/build-tools/common/nodejs/node-v14.21.1-darwin-x64" + }, + { + "remote_url": "/nodejs/v16.20.2/node-v16.20.2-darwin-x64.tar.gz", + "unzip_filename": "node-v16.20.2-darwin-x64" + } + ] + }, + "handle": [ + { + "type": "symlink", + "src": "${symlink_src}", + "dest": "${code_dir}/prebuilts/build-tools/common/nodejs/current" + } + ] }, { - "target_os": "darwin", - "target_cpu": "arm64", - "version": "20250425", - "url": "openharmony/compiler/clang/15.0.4-da9259/darwin_x86/clang_darwin-x86_64-da9259-20250425.tar.gz", - "download_dir": ".prebuilts_cache/clang/ohos/darwin-x86_64/llvm", - "symlink": "prebuilts/clang/ohos/darwin-x86_64/llvm", - "type": "dir", - "copy_src":"lib/clang/15.0.4", - "copy_dest":"lib/clang/current" + "name": "packing_tool", + "tag": "base", + "type": "src, indep", + "config": { + "linux, darwin": { + "arm64, x86_64": { + "remote_url": "/harmonyos/compiler/packing_tool/packing_tool_libs_20250215.zip", + "unzip_dir": "${code_dir}/prebuilts/packing_tool", + "unzip_filename": "packing_tool" + } + } + } }, { - "target_os": "darwin", - "target_cpu": "x86_64", - "version": "20250425", - "url": "openharmony/compiler/clang/15.0.4-da9259/darwin_x86/clang_darwin-x86_64-da9259-20250425.tar.gz", - "download_dir": ".prebuilts_cache/clang/ohos/darwin-x86_64/llvm", - "symlink": "prebuilts/clang/ohos/darwin-x86_64/llvm", - "type": "dir", - "copy_src":"lib/clang/15.0.4", - "copy_dest":"lib/clang/current" - } - ] - }, - { - "name": "python", - "tag": "base", - "download": [ - { - "target_os": "linux", - "target_cpu": "arm64", - "version": "20240715", - "url": "openharmony/compiler/python/3.11.4/linux/python-linux-x86-GLIBC2.27-3.11.4_20250219.tar.gz", - "download_dir": ".prebuilts_cache/python/linux-x86", - "symlink": "prebuilts/python/linux-x86", - "type": "dir", - "copy_src": "3.11.4", - "copy_dest": "current", - "pip_install": [ - "pyyaml", - "requests>=2.32.1", - "prompt_toolkit==1.0.14", - "asn1crypto", - "cryptography", - "json5==0.9.6" - ] + "name": "python", + "tag": "base", + "type": "src, indep", + "unzip_dir": "${code_dir}/prebuilts/python", + "config": { + "linux": { + "arm64": { + "remote_url": "/openharmony/compiler/python/3.11.4/linux/python-linux-arm64-3.11.4_20240715.tar.gz", + "unzip_filename": "linux-arm64" + }, + "x86_64": { + "remote_url": "/openharmony/compiler/python/3.11.4/linux/python-linux-x86-GLIBC2.27-3.11.4_20250219.tar.gz", + "unzip_filename": "linux-x86" + } + }, + "darwin": { + "x86_64": { + "remote_url": "/openharmony/compiler/python/3.11.4/darwin/python-darwin-x86-3.11.4_20250228.tar.gz", + "unzip_filename": "darwin-x86" + }, + "arm64": { + "remote_url": "/openharmony/compiler/python/3.11.4/darwin/python-darwin-arm64-3.11.4_20250228.tar.gz", + "unzip_filename": "darwin-arm64" + } + } + }, + "handle": [ + { + "type": "symlink", + "src": "${unzip_dir}/${unzip_filename}/3.11.4", + "dest": "${unzip_dir}/${unzip_filename}/current" + }, + { + "type": "shell", + "python_path": "${unzip_dir}/${unzip_filename}/current/bin/python3", + "pip_path": "${unzip_dir}/${unzip_filename}/current/bin/pip3", + "cmd": [ + "${python_path}", + "${pip_path}", + "install", + "--trusted-host", + "repo.huaweicloud.com", + "-i", + "http://repo.huaweicloud.com/repository/pypi/simple", + "pyyaml", + "requests>=2.32.1", + "prompt_toolkit==1.0.14", + "asn1crypto", + "cryptography", + "json5==0.9.6" + ] + } + ] }, { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "20240715", - "url": "openharmony/compiler/python/3.11.4/linux/python-linux-x86-GLIBC2.27-3.11.4_20250219.tar.gz", - "download_dir": ".prebuilts_cache/python/linux-x86", - "symlink": "prebuilts/python/linux-x86", - "type": "dir", - "copy_src": "3.11.4", - "copy_dest": "current", - "pip_install": [ - "pyyaml", - "requests>=2.32.1", - "prompt_toolkit==1.0.14", - "asn1crypto", - "cryptography", - "json5==0.9.6" - ] + "name": "hpm", + "tag": "base", + "type": "indep", + "handle": [ + { + "type": "hpm_download", + "name": "@ohos/hpm-cli", + "version": "1.6.1", + "package_path": "${code_dir}", + "package_lock_path": "${code_dir}", + "download_dir": "${download_root}/hpm", + "symlink": "${code_dir}/prebuilts/hpm/node_modules" + } + ] }, { - "target_os": "darwin", - "target_cpu": "arm64", - "version": "20240715", - "url": "openharmony/compiler/python/3.11.4/linux/python_darwin-arm64-3.11.4_20250228.tar.gz", - "download_dir": ".prebuilts_cache/python/darwin-arm64", - "symlink": "prebuilts/python/darwin-arm64", - "type": "dir", - "copy_src": "3.11.4", - "copy_dest": "current", - "pip_install": [ - "pyyaml", - "requests>=2.32.1", - "prompt_toolkit==1.0.14", - "asn1crypto", - "cryptography", - "json5==0.9.6" - ] - }, - { - "target_os": "darwin", - "target_cpu": "x86_64", - "version": "20240729", - "url": "openharmony/compiler/python/3.11.4/linux/python-darwin-x86-3.11.4_20250228.tar.gz", - "download_dir": ".prebuilts_cache/python/darwin-x86", - "symlink": "prebuilts/python/darwin-x86", - "type": "dir", - "copy_src": "3.11.4", - "copy_dest": "current", - "pip_install": [ - "pyyaml", - "requests>=2.32.1", - "prompt_toolkit==1.0.14", - "asn1crypto", - "cryptography", - "json5==0.9.6" - ] - } - ] - }, - { - "name": "hcgen", - "tag": "base", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "linux", - "url": "openharmony/compiler/hc-gen/linux/hc-gen-20240926.tar.gz", - "download_dir": ".prebuilts_cache/hc_gen", - "symlink": "prebuilts/hc_gen/hc-gen", - "type": "dir" - } - ] - }, - { - "name": "OpenHarmonyApplication.pem", - "tag": "base", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "linux", - "url": "openharmony/compiler/certificates/20241025/OpenHarmonyApplication.zip", - "download_dir": ".prebuilts_cache/OpenHarmonyApplication.pem", - "symlink": "prebuilts/ohos-sdk/OpenHarmonyApplication.pem", - "type": "file" - } - ] - }, - { - "name": "rustc", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "1.72.0", - "url": "openharmony/compiler/rust/1.72.0-20250515/rust-nightly-x86_64-unknown-linux-gnu.tar.gz", - "download_dir": ".prebuilts_cache/rustc/rust-nightly-x86_64-unknown-linux-gnu", - "script": "install_ext.sh", - "args": [{ - "prefix": "''" - }], - "destdir": "prebuilts/rustc/linux-x86_64/current", - "type": "dir" - } - ] - }, - { - "name": "rustc-aarch64-std", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "1.72.0", - "url": "openharmony/compiler/rust/1.72.0-20250515/rust-std-nightly-aarch64-unknown-linux-ohos.tar.gz", - "download_dir": ".prebuilts_cache/rustc/rust-nightly-x86_64-unknown-linux-gnu", - "script": "install_ext.sh", - "args": [{ - "prefix": "''" - }], - "destdir": "prebuilts/rustc/linux-x86_64/current", - "type": "dir" - } - ] - }, - { - "name": "rustc-arm-std", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "1.72.0", - "url": "openharmony/compiler/rust/1.72.0-20250515/rust-std-nightly-armv7-unknown-linux-ohos.tar.gz", - "download_dir": ".prebuilts_cache/rustc/rust-nightly-x86_64-unknown-linux-gnu", - "script": "install_ext.sh", - "args": [{ - "prefix": "''" - }], - "destdir": "prebuilts/rustc/linux-x86_64/current", - "type": "dir" - } - ] - }, - { - "name": "rustc-x86_64-std", - "download": [ - { - "target_os": "linux", - "target_cpu": "x86_64", - "version": "1.72.0", - "url": "openharmony/compiler/rust/1.72.0-20250515/rust-std-nightly-x86_64-unknown-linux-ohos.tar.gz", - "download_dir": ".prebuilts_cache/rustc/rust-nightly-x86_64-unknown-linux-gnu", - "script": "install_ext.sh", - "args": [{ - "prefix": "''" - }], - "destdir": "prebuilts/rustc/linux-x86_64/current", - "type": "dir" + "name": "rustc", + "tag": "base", + "type": "src, indep", + "unzip_dir": "${code_dir}/prebuilts/rustc", + "config": { + "linux": { + "arm64": { + "remote_url": "/openharmony/compiler/rust/20240528/rust-std-nightly-aarch64-unknown-linux-ohos_20240528.tar.gz", + "unzip_filename": "rustc-aarch64-std", + "install_dir": "${code_dir}/prebuilts/rustc/rust-std-nightly-aarch64-unknown-linux-ohos_20240528", + "destdir": "${code_dir}/prebuilts/rustc/linux-x86_64/current" + }, + "x86_64": [ + { + "remote_url": "/openharmony/compiler/rust/20240528/rust-nightly-x86_64-unknown-linux-gnu_20240528.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-nightly-x86_64-unknown-linux-gnu_20240528", + "destdir": "${code_dir}/prebuilts/rustc/linux-x86_64/current", + "unzip_filename": "rustc" + }, + { + "remote_url": "/openharmony/compiler/rust/20240528/rust-std-nightly-aarch64-unknown-linux-ohos_20240528.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-std-nightly-aarch64-unknown-linux-ohos_20240528", + "destdir": "${code_dir}/prebuilts/rustc/linux-x86_64/current", + "unzip_filename": "rustc-aarch64-std" + }, + { + "remote_url": "/openharmony/compiler/rust/20240528/rust-std-nightly-armv7-unknown-linux-ohos_20240528.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-std-nightly-armv7-unknown-linux-ohos_20240528", + "destdir": "${code_dir}/prebuilts/rustc/linux-x86_64/current", + "unzip_filename": "rustc-arm-std" + }, + { + "remote_url": "/openharmony/compiler/rust/20240528/rust-std-nightly-x86_64-unknown-linux-ohos_20240528.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-std-nightly-x86_64-unknown-linux-ohos_20240528", + "destdir": "${code_dir}/prebuilts/rustc/linux-x86_64/current", + "unzip_filename": "rustc-x86_64-std" + }, + { + "remote_url": "/openharmony/compiler/rust/20240528/rust-std-nightly-x86_64-pc-windows-gnullvm_20240528.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-std-nightly-x86_64-pc-windows-gnullvm_20240528", + "destdir": "${code_dir}/prebuilts/rustc/linux-x86_64/current", + "unzip_filename": "rustc-x86_64-windows-std", + "type": "src" + } + ] + }, + "darwin": { + "arm64": [ + { + "remote_url": "/openharmony/compiler/rust/20240429/rust-nightly-aarch64-apple-darwin_20240429.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-nightly-aarch64-apple-darwin_20240429", + "destdir": "${code_dir}/prebuilts/rustc/darwin-aarch64/current", + "unzip_filename": "rustc" + }, + { + "remote_url": "/openharmony/compiler/rust/20240429/rust-std-nightly-aarch64-apple-darwin_20240429.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-std-nightly-aarch64-apple-darwin_20240429", + "destdir": "${code_dir}/prebuilts/rustc/darwin-aarch64/current", + "unzip_filename": "rustc-aarch64-std" + } + ], + "x86_64": [ + { + "remote_url": "/openharmony/compiler/rust/20240429/rust-nightly-x86_64-apple-darwin_20240429.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-nightly-x86_64-apple-darwin_20240429", + "destdir": "${code_dir}/prebuilts/rustc/darwin-x86_64/current", + "unzip_filename": "rustc" + }, + { + "remote_url": "/openharmony/compiler/rust/20240429/rust-std-nightly-x86_64-apple-darwin_20240429.tar.gz", + "install_dir": "${code_dir}/prebuilts/rustc/rust-std-nightly-x86_64-apple-darwin_20240429", + "destdir": "${code_dir}/prebuilts/rustc/darwin-x86_64/current", + "unzip_filename": "rustc-x86_64-std" + } + ] + } + }, + "handle": [ + { + "type": "shell", + "cmd": [ + "${install_dir}/install.sh", + "--prefix=''", + "--destdir=${destdir}" + ] + } + ] } - ] - } - ] -} + ] +} \ No newline at end of file diff --git a/prebuilts_config.py b/prebuilts_config.py index eca5e23b5d8708acceffbc58f2a4d07d7a8535de..cbd2f5cf99cf48032787b8a0d5b01f1498758a30 100644 --- a/prebuilts_config.py +++ b/prebuilts_config.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# -# Copyright (c) 2024 Huawei Device Co., Ltd. +# Copyright (c) 2025 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,335 +12,66 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -import json import os -import subprocess -import shutil import argparse +import ssl import sys -import urllib.request -import socket - - -def _run_cmd(cmd): - res = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - sout, serr = res.communicate(timeout=10000) - return sout.rstrip().decode('utf-8'), serr, res.returncode - - -def _check_sha256(check_url: str, local_file: str) -> bool: - check_sha256_cmd = ('curl -s -k ' + check_url + '.sha256').split(' ') - local_sha256_cmd = ['sha256sum', local_file] - check_sha256, err, returncode = _run_cmd(check_sha256_cmd) - local_sha256, err, returncode = _run_cmd(local_sha256_cmd) - local_sha256 = local_sha256.split(' ')[0] - if check_sha256 != local_sha256: - print('remote file {}.sha256 is not found, begin check SHASUMS256.txt'.format(check_url)) - check_sha256 = _obtain_sha256_by_sha_sums256(check_url) - return check_sha256 == local_sha256 - - -def _obtain_sha256_by_sha_sums256(check_url: str) -> str: - sha_sums256 = 'SHASUMS256.txt' - sha_sums256_path = os.path.join(os.path.dirname(check_url), sha_sums256) - file_name = os.path.basename(check_url) - cmd = ('curl -s -k ' + sha_sums256_path).split(' ') - data_sha_sums256, err, returncode = _run_cmd(cmd) - check_sha256 = None - for line in data_sha_sums256.split('\n'): - if file_name in line: - check_sha256 = line.split(' ')[0] - return check_sha256 - - -def npm_install(dir_name): - result = subprocess.run(["npm", "install", "--prefix", dir_name], capture_output=True, text=True) - if result.returncode == 0: - print("{}目录下npm install完毕".format(dir_name)) - else: - print("依赖项安装失败:", result.stderr) - - -def copy_file(src, dest): - if not os.path.exists(dest): - os.makedirs(dest) - shutil.copy(src, dest) - - -def copy_folder(src, dest): - if os.path.exists(dest): - if os.path.islink(dest): - os.unlink(dest) - else: - shutil.rmtree(dest) - shutil.copytree(src, dest) - - -def symlink_src2dest(src_dir, dest_dir): - if os.path.exists(dest_dir) or os.path.islink(dest_dir): - if os.path.islink(dest_dir): - os.unlink(dest_dir) - elif os.path.isdir(dest_dir): - shutil.rmtree(dest_dir) - else: - os.remove(dest_dir) - else: - os.makedirs(os.path.dirname(dest_dir), exist_ok=True) - - print("symlink {} ---> {}".format(src_dir, dest_dir)) - os.symlink(src_dir, dest_dir) - - -def install_hpm(code_path, download_dir, symlink_dir, home_path): - content = """\ -package-lock=true -registry=http://repo.huaweicloud.com/repository/npm -strict-ssl=false -lockfile=false -""" - with os.fdopen(os.open(os.path.join(home_path, '.npmrc'), os.O_WRONLY | os.O_CREAT, mode=0o640), 'w') as f: - os.truncate(f.fileno(), 0) - f.write(content) - if not os.path.exists(download_dir): - os.makedirs(download_dir) - with os.fdopen(os.open(os.path.join(download_dir, 'package.json'), os.O_WRONLY | os.O_CREAT, mode=0o640), - 'w') as f: - os.truncate(f.fileno(), 0) - f.write('{}\n') - npm_path = os.path.join(code_path, "prebuilts/build-tools/common/nodejs/current/bin/npm") - node_bin_path = os.path.join(code_path, "prebuilts/build-tools/common/nodejs/current/bin") - os.environ['PATH'] = f"{node_bin_path}:{os.environ['PATH']}" - subprocess.run( - [npm_path, 'install', '@ohos/hpm-cli', '--registry', 'https://repo.huaweicloud.com/repository/npm/', '--prefix', - download_dir]) - symlink_src2dest(os.path.join(download_dir, 'node_modules'), symlink_dir) - - -def process_npm(npm_dict, args): - code_path = args.code_path - home_path = args.home_path - name = npm_dict.get('name') - npm_download = npm_dict.get('download') - package_path = os.path.join(code_path, npm_download.get('package_path')) - package_lock_path = os.path.join(code_path, npm_download.get('package_lock_path')) - hash_value = \ - subprocess.run(['sha256sum', package_lock_path], capture_output=True, text=True).stdout.strip().split(' ')[0] - download_dir = os.path.join(home_path, npm_download.get('download_dir'), hash_value) - - if '@ohos/hpm-cli' == name: - symlink = os.path.join(code_path, npm_download.get('symlink')) - install_hpm(code_path, os.path.join(home_path, download_dir), os.path.join(code_path, symlink), home_path) - return - - copy_file(package_path, download_dir) - copy_file(package_lock_path, download_dir) - - if not os.path.exists(os.path.join(download_dir, "npm-install.js")): - npm_install_script = os.path.join(os.path.dirname(package_path), "npm-install.js") - if os.path.exists(npm_install_script): - shutil.copyfile(npm_install_script, os.path.join(download_dir, "npm-install.js")) - - npm_install(download_dir) - - if name == 'legacy_bin': - for link in npm_download.get('symlink', []): - symlink_src2dest(os.path.join(download_dir, "node_modules"), os.path.join(code_path, link)) - return - - symlink = os.path.join(code_path, npm_download.get('symlink')) - - if name in ['parse5']: - copy_folder(os.path.join(download_dir, "node_modules"), symlink) - return - - copy_folder(os.path.join(download_dir, "node_modules"), symlink) - - for copy_entry in npm_download.get('copy', []): - copy_folder(os.path.join(code_path, copy_entry['src']), os.path.join(code_path, copy_entry['dest'])) - - for copy_ext_entry in npm_download.get('copy_ext', []): - copy_folder(os.path.join(code_path, copy_ext_entry['src']), os.path.join(code_path, copy_ext_entry['dest'])) - +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "prebuilts_service")) +from prebuilts_service.operater import OperateHanlder +from prebuilts_service.pool_downloader import PoolDownloader +from prebuilts_service.config_parser import ConfigParser +import json +from prebuilts_service.common_utils import get_code_dir -def download_url(url, folder_path): - filename = url.split('/')[-1] - file_path = os.path.join(folder_path, filename) - if os.path.exists(file_path): - if _check_sha256(url, file_path): - return - else: - os.remove(file_path) - if not os.path.exists(folder_path): - os.makedirs(folder_path) - try: - print("Downloading {}".format(url)) - with urllib.request.urlopen(url) as response, os.fdopen( - os.open(file_path, os.O_WRONLY | os.O_CREAT, mode=0o640), 'wb') as out_file: - total_size = int(response.headers['Content-Length']) - chunk_size = 16 * 1024 - downloaded_size = 0 - print_process(chunk_size, downloaded_size, out_file, response, total_size) - print("\n{} downloaded successfully".format(url)) - except urllib.error.URLError as e: - print("Error:", e.reason) - except socket.timeout: - print("Timeout error: Connection timed out") +global_args = None -def print_process(chunk_size, downloaded_size, out_file, response, total_size): - while True: - chunk = response.read(chunk_size) - if not chunk: - break - out_file.write(chunk) - downloaded_size += len(chunk) - if total_size != 0: - if "DISABLE_DOWNLOAD_PROGRESS" not in os.environ: - progress = downloaded_size / total_size * 50 - print(f'\r[{"=" * int(progress)}{" " * (50 - int(progress))}] {progress * 2:.2f}%', end='', flush=True) - else: - print("\r[Error] Total size is zero, unable to calculate progress.", end='', flush=True) +def _parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--skip-ssl', action='store_true', help='skip ssl authentication') + parser.add_argument('--unsafe-perm', action='store_true', help='add "--unsafe-perm" for npm install') + parser.add_argument('--disable-rich', action='store_true', help='disable the rich module') + parser.add_argument('--enable-symlink', action='store_true', help='enable symlink while copying node_modules') + parser.add_argument('--build-arkuix', action='store_true', help='build ArkUI-X SDK') + parser.add_argument('--tool-repo', default='https://repo.huaweicloud.com', help='prebuilt file download source') + parser.add_argument('--npm-registry', default='https://repo.huaweicloud.com/repository/npm/', + help='npm download source') + parser.add_argument('--host-cpu', help='host cpu', required=True) + parser.add_argument('--host-platform', help='host platform', required=True) + parser.add_argument('--glibc-version', help='glibc version', required=False) + parser.add_argument('--config-file', help='prebuilts download config file') + parser.add_argument('--type', help='prebuilts download type', default="indep") + parser.add_argument('--tag', help='prebuilts download tag', default="base") + return parser -def extract_compress_files(source_file, destination_folder): - if not os.path.exists(destination_folder): - os.makedirs(destination_folder, exist_ok=True) - if source_file.endswith('.zip'): - command = ['unzip', '-qq', '-o', '-d', destination_folder, source_file] - elif source_file.endswith('.tar.gz'): - command = ['tar', '-xzf', source_file, '-C', destination_folder] - elif source_file.endswith('.tar.xz'): - command = ['tar', '-xJf', source_file, '-C', destination_folder] - else: - print("暂不支持解压此类型压缩文件！") - command = [] - try: - subprocess.run(command, check=True) - print(f"{source_file} extracted to {destination_folder}") - except subprocess.CalledProcessError as e: - print(f"Error: {e}") +def main(): + parser = _parse_args() + global global_args + global_args = parser.parse_args() + if global_args.skip_ssl: + global_args._create_default_https_context = ssl._create_unverified_context + global_args.code_dir = get_code_dir() -def install_python(version, download_dir, one_type, code_path): - after_extract_folder = os.listdir(os.path.join(download_dir, version))[0] - copy_folder(os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_src')), - os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_dest'))) - python3_path = os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_dest'), - 'bin', 'python3') - pip3_path = os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_dest'), 'bin', - 'pip3') - subprocess.run( - [python3_path, pip3_path, "install", "--trusted-host", "repo.huaweicloud.com", "-i", - "http://repo.huaweicloud.com/repository/pypi/simple"] + one_type.get( - 'pip_install')) - copy_folder(os.path.join(download_dir, version, after_extract_folder), - os.path.join(code_path, one_type.get('symlink'))) + config_file = os.path.join(global_args.code_dir,"build", "prebuilts_config.json") + if global_args.config_file: + config_file = global_args.config_file + config_parser = ConfigParser(config_file, global_args) + download_operate, other_operate = config_parser.get_operate() + + prebuilts_path = os.path.join(global_args.code_dir, "prebuilts") + if not os.path.exists(prebuilts_path): + os.makedirs(prebuilts_path) + + # 使用线程池下载 + pool_downloader = PoolDownloader(download_operate, global_args) + unchanged = pool_downloader.start() + + OperateHanlder.run(other_operate, global_args, unchanged) -def install_rustc(url, download_dir, version, one_type, code_path): - part_file_name = url.split('/')[-1].split('linux')[0] - after_extract_file_list = os.listdir(os.path.join(download_dir, version)) - for file_name in after_extract_file_list: - if part_file_name in file_name: - script_path = os.path.join(download_dir, version, file_name, "install.sh") - subprocess.run([script_path, "--prefix=''", - "--destdir='{}'".format(os.path.join(code_path, one_type.get('destdir')))]) - break - - -def process_tar(tar_dict, args): - code_path = args.code_path - home_path = args.home_path - os_type = args.target_os - cpu_type = args.target_cpu - tar_name = tar_dict.get('name') - print(tar_name) - tar_download = tar_dict.get('download') - for one_type in tar_download: - download_flag = download_or_not(cpu_type, one_type, os_type) - if download_flag == True: - version = one_type.get('version') - url = os.path.join(args.repo_https, one_type.get('url')) - download_dir = os.path.join(home_path, one_type.get('download_dir')) - download_url(url, download_dir) - extract_compress_files(os.path.join(download_dir, url.split('/')[-1]), - os.path.join(download_dir, version)) - if tar_name == 'python': - install_python(version, download_dir, one_type, code_path) - continue - if 'rustc' in tar_name: - install_rustc(url, download_dir, version, one_type, code_path) - continue - if tar_name == 'llvm': - after_extract_folder = os.listdir(os.path.join(download_dir, version))[0] - symlink_src2dest(os.path.join(download_dir, version, after_extract_folder), - os.path.join(code_path, one_type.get('symlink'))) - symlink_src2dest(os.path.join(code_path, one_type.get('symlink'), one_type.get('copy_src')), - os.path.join(code_path, one_type.get('symlink'), one_type.get('copy_dest'))) - continue - if tar_name in ['ark_tools']: - after_extract_folder = os.listdir(os.path.join(download_dir, version))[0] - copy_folder(os.path.join(download_dir, version, after_extract_folder), - os.path.join(code_path, one_type.get('symlink'))) - continue - after_extract_file_list = os.listdir(os.path.join(download_dir, version)) - symlink = os.path.join(code_path, one_type.get('symlink')) - if one_type.get('type') == 'dir': - deal_tar_dir(after_extract_file_list, download_dir, symlink, tar_name, version) - else: - file_name = after_extract_file_list[0] - symlink_src2dest(os.path.join(download_dir, version, file_name), symlink) - if tar_name == 'nodejs' and url == os.path.join(args.repo_https, - "nodejs/v14.21.1/node-v14.21.1-linux-x64.tar.gz"): - symlink_src2dest(symlink, os.path.join(os.path.dirname(symlink), 'current')) - - -def download_or_not(cpu_type, one_type, os_type): - download_flag = False - if cpu_type == 'any': - if one_type.get('target_os') == os_type: - download_flag = True - else: - if one_type.get('target_os') == os_type and ( - one_type.get('target_cpu') == cpu_type or one_type.get('target_cpu') == ""): - download_flag = True - return download_flag - - -def deal_tar_dir(after_extract_file_list, download_dir, symlink, tar_name, version): - for one_file in after_extract_file_list: - if tar_name == 'packing_tool': - symlink_src2dest(os.path.join(download_dir, version, one_file), - os.path.join(symlink, one_file)) - else: - symlink_src2dest(os.path.join(download_dir, version, one_file), - symlink) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--code_path', required=True, type=str, help='path of openharmony code') - parser.add_argument('--home_path', required=True, type=str, help='path of home') - parser.add_argument('--config_file', required=True, type=str, help='path of prebuilts_config.json') - parser.add_argument('--repo_https', required=True, type=str, default='https://repo.huaweicloud.com', - help='path of prebuilts_config.json') - parser.add_argument('--target_os', required=True, type=str, help='type of os') - parser.add_argument('--target_cpu', type=str, default='any', help='type of cpu') - args = parser.parse_args() - config_file = args.config_file - with open(config_file, 'r', encoding='utf-8') as r: - config = json.load(r) - tar = config['tar'] - for one_tar in tar: - process_tar(one_tar, args) - npm = config['npm'] - for one_npm in npm: - process_npm(one_npm, args) - - -if __name__ == '__main__': - sys.exit(main()) +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/prebuilts_config.sh b/prebuilts_config.sh index 68e1db4122414e40a59dfc47146dcc6ab886a334..61823270055b6026848feade35a938b1aeb92031 100644 --- a/prebuilts_config.sh +++ b/prebuilts_config.sh @@ -13,19 +13,49 @@ # limitations under the License. set -e - script_path=$(cd $(dirname $0);pwd) code_dir=$(dirname ${script_path}) home_path=$HOME config_file="$script_path/prebuilts_config.json" -# 获取当前系统类型和CPU类型 -target_os=$(uname -s | tr '[:upper:]' '[:lower:]') -target_cpu=$(uname -m | tr '[:upper:]' '[:lower:]') + +case $(uname -s) in + Linux) + host_platform=linux + glibc_version=$(getconf GNU_LIBC_VERSION | grep -oE '[0-9].[0-9]{2}') + ;; + Darwin) + host_platform=darwin + ;; + *) + echo "Unsupported host platform: $(uname -s)" + exit 1 +esac + +case $(uname -m) in + arm64) + host_cpu=arm64 + host_cpu_prefix=arm64 + ;; + aarch64) + host_cpu=arm64 + host_cpu_prefix=aarch64 + ;; + *) + host_cpu=x86_64 + host_cpu_prefix=x86 +esac + +if [[ "${glibc_version}" < "2.35" ]]; then + glibc_version="--glibc-version GLIBC2.27" +else + glibc_version="--glibc-version GLIBC2.35" +fi + # 运行Python命令 -python3 "${script_path}/prebuilts_config.py" --code_path $code_dir --home_path $home_path --config_file $config_file --repo_https https://repo.huaweicloud.com --target_os $target_os --target_cpu $target_cpu +python3 "${script_path}/prebuilts_config.py" $glibc_version --config-file $config_file --host-platform $host_platform --host-cpu $host_cpu -PYTHON_PATH=$(realpath ${code_dir}/prebuilts/python/${target_os}-*/*/bin | tail -1) +PYTHON_PATH=$(realpath $code_dir/prebuilts/python/${host_platform}-${host_cpu_prefix}/*/bin | tail -1) while [ $# -gt 0 ]; do case "$1" in diff --git a/prebuilts_service/README_zh.md b/prebuilts_service/README_zh.md new file mode 100644 index 0000000000000000000000000000000000000000..64615a0223cda6f68db90a9fd1132202a2bb6a33 --- /dev/null +++ b/prebuilts_service/README_zh.md @@ -0,0 +1,215 @@ +# 预编译工具配置指南 +- [工具下载配置](#section-download-01) + 1. [核心配置说明](#section-download-core-01) + 2. [基础配置示例](#section-download-basic-demo) + 3. [高级配置示例](#section-download-advanced-demo) +- [后续处理配置](#advanced-process) +- [变量查找规则](#value-search) + +## 工具下载配置 +下载配置用于配置下载和解压参数 +### 核心配置项说明 + +|参数|描述| +|--|--| +remote_url|远程包下载地址（HTTP/HTTPS）| +unzip_dir|解压目标路径（绝对或相对路径）| +unzip_filename|解压后的顶层目录名（用于版本管理和旧文件清理）| + +### 基础配置示例 +#### 场景1：指定操作系统与CPU架构 +以 ark_js_prebuilts 工具为例，在 Linux x86_64 环境下的配置如下： +```json +{ + "name": "ark_js_prebuilts", + "tag": "base", + "type": "src, indep", + "config": { + "linux": { + "x86_64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_20230713.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/ark_tools", + "unzip_filename": "ark_js_prebuilts" + } + } + } +} +``` + + +#### 场景2：CPU架构无关配置 +若工具包不依赖CPU架构（如纯脚本工具），可省略架构标识 +``` json +{ + "name": "ark_js_prebuilts", + "tag": "base", + "type": "src, indep", + "config": { + "linux": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_20230713.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/ark_tools", + "unzip_filename": "ark_js_prebuilts" + } + } +} +``` + + +#### 场景3：跨平台配置 +若工具包同时兼容多操作系统和CPU架构，配置进一步简化： +```json +{ + "name": "ark_js_prebuilts", + "tag": "base", + "type": "src, indep", + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_20230713.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/ark_tools", + "unzip_filename": "ark_js_prebuilts" +} +``` + +### 高级配置场景 +#### 多版本并行下载（以LLVM为例） +若需在同一平台下安装多个版本，配置项改为列表形式： +```json +{ + "name": "llvm", + "tag": "base", + "type": "src, indep", + "config": { + "linux": { + "x86_64": [ + { + "remote_url": "/openharmony/compiler/clang/15.0.4-3cec00/ohos_arm64/clang_ohos-arm64-3cec00-20250320.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/ohos-arm64"， + "unzip_filename": "llvm", + }, + { + "remote_url": "/openharmony/compiler/clang/15.0.4-3cec00/windows/clang_windows-x86_64-3cec00-20250320.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/windows-x86_64"， + "unzip_filename": "llvm", + }, + { + "remote_url": "/openharmony/compiler/clang/15.0.4-3cec00/linux/clang_linux-x86_64-3cec00-20250320.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/clang/ohos/linux-x86_64"， + "unzip_filename": "llvm", + } + ] + } + } +} +``` + + + +#### 使用公共变量 +当配置中存在值相同的配置项时，可提取公共变量避免冗余：
+**原始冗余配置** +```json +{ + "name": "ark_js_prebuilts", + "tag": "base", + "type": "src, indep", + "config": { + "linux": { + "x86_64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_20230713.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/ark_tools", + "unzip_filename": "ark_js_prebuilts" + } + }, + "darwin": { + "x86_64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_darwin_x64_20230209.tar.gz", + "unzip_dir": "${code_dir}/prebuilts/ark_tools", + "unzip_filename": "ark_js_prebuilts" + } + } + } +} +``` + +**优化后配置** +```json +{ + "name": "ark_js_prebuilts", + "tag": "base", + "type": "src, indep", + "unzip_dir": "${code_dir}/prebuilts/ark_tools", + "unzip_filename": "ark_js_prebuilts", + "config": { + "linux": { + "x86_64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_20230713.tar.gz" + } + }, + "darwin": { + "x86_64": { + "remote_url": "/openharmony/compiler/llvm_prebuilt_libs/ark_js_prebuilts_darwin_x64_20230209.tar.gz" + + } + } + } +} +``` + +#### 配置继承规则 +- 工具配置会继承全局配置 +- 平台配置会继承工具配置 +- 内部配置优于继承配置 + +## 后续处理配置 +工具下载解压完成后可能需要进行后续处理，该部分在handle中配置，handle是一个列表，其中的每一项都代表一个操作 +### handle配置特点 +- 顺序执行：操作按配置顺序依次执行。 +- 变量继承：操作中可引用config和外部的配置参数 +- 灵活控制：可通过handle_index指定执行的操作序号。 +- 容错机制：若操作中的变量解析失败，跳过当前操作。 + +### 公共操作列表 + +|操作类型|参数|用途| +|-|-|-| +|download| remote_url: 远程下载地
unzip_dir: 本地解压目
unzip_filename: 用于哈希校验和清理
**注：该操作通常而言无需显示声明，脚本会根据平台配置的remote_url自动生成对应的下载作 **| 下载和解压 | +|symlink| src: 链接源
dest: 目的链接地址| 生成符号链接 +|copy | src: 源
dest: 目的| 复制文件或文件夹 | +|remove | path:要删除的路径, 可以是字符串，也可以是一个列表 | 删除文件或文件夹 | +|move | src: 源路径
dest: 目标路径
filetype: 该参数默认不填写，若填写，则只会移动src目录中以filetype为后缀的文件 | 移动文件,若dest是个已存在的目录，则会移动到目录中 | +|shell | cmd: 命令(列表形式) |执行shell命令 + +### handle配置示例 +#### 场景： 解压Node工具后创建符号链接: +```json +{ + "name": "node", + "tag": "base", + "type": "src, indep", + "unzip_dir": "${code_dir}/prebuilts/build-tools/common/nodejs", + "config": { + "linux": { + "x86_64": [ + { + "remote_url": "/nodejs/v14.21.1/node-v14.21.1-linux-x64.tar.gz", + "unzip_filename": "node-v14.21.1-linux-x64", + "symlink_src": "${code_dir}/prebuilts/build-tools/common/nodejs/node-v14.21.1-linux-x64" + } + ] + } + }, + "handle": [ + { + "type": "symlink", + "src": "${symlink_src}", + "dest": "${code_dir}/prebuilts/build-tools/common/nodejs/current" + } + ] +} +``` + + +## 变量查找规则 +- 变量只能使用${var_name}的方式指定 +- 工具配置可以使用自身以及全局配置中的变量 +- 平台配置可以使用自身、工具以及全局配置中的变量 +- handle可以使用自身、平台、工具以及全局配置中的变量 +- 变量只会解析一次，采取就近解析原则 diff --git a/prebuilts_service/common_utils.py b/prebuilts_service/common_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7ae16a14c10020aeccfbaed42b5cace4528027c4 --- /dev/null +++ b/prebuilts_service/common_utils.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import subprocess +import pathlib +import time +import json +import importlib + + +def get_code_dir(): + current_dir = os.path.dirname(__file__) + while True: + check_path = os.path.join(current_dir, "build", "ohos.gni") + if os.path.exists(check_path): + return current_dir + else: + new_dir = os.path.dirname(current_dir) + if new_dir == current_dir: + raise Exception(f"file {__file__} not in ohos source directory") + else: + current_dir = new_dir + + +def import_rich_module(): + module = importlib.import_module("rich.progress") + progress = module.Progress( + module.TextColumn("[bold blue]{task.fields[filename]}", justify="right"), + module.BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + module.DownloadColumn(), + "•", + module.TransferSpeedColumn(), + "•", + module.TimeRemainingColumn(), + ) + return progress + + +def load_config(config_file: str): + with open(config_file, "r", encoding="utf-8") as r: + config = json.load(r) + return config + + +def copy_file(src: str, dest: str): + if not os.path.exists(dest): + os.makedirs(dest) + shutil.copy(src, dest) + + +def remove_dest_path(dest_path: str): + if os.path.exists(dest_path) or os.path.islink(dest_path): + if os.path.islink(dest_path): + os.unlink(dest_path) + elif os.path.isdir(dest_path): + shutil.rmtree(dest_path) + else: + os.remove(dest_path) + + +def copy_folder(src: str, dest: str): + remove_dest_path(dest) + shutil.copytree(src, dest) + + +def symlink_src2dest(src_dir: str, dest_dir: str): + remove_dest_path(dest_dir) + os.makedirs(os.path.dirname(dest_dir), exist_ok=True) + os.symlink(src_dir, dest_dir) + print("symlink {} ---> {}".format(src_dir, dest_dir)) + + +def run_cmd_live(cmd: list): + cmd_str = " ".join(cmd) + print(f"run command: {cmd_str}\n") + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True + ) + + while True: + output = process.stdout.readline() + if output == '' and process.poll() is not None: + break + if output: + print(output.strip()) + + return_code = process.poll() + if return_code != 0: + print(f"命令执行失败，返回码: {return_code}") + return return_code, "" + except Exception as e: + print(f"执行命令时出错: {e}") + return 1, "" + + +def run_cmd(cmd: list) -> tuple: + res = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + sout, serr = res.communicate() + return sout.rstrip().decode("utf-8"), serr, res.returncode + + +def is_system_component() -> bool: + root_dir = get_code_dir() + return any( + pathlib.Path(root_dir, *components).exists() + for components in [ + ("interface", "sdk-js"), + ("foundation", "arkui"), + ("arkcompiler",) + ] + ) + + +def install_hpm_in_other_platform(name: str, operate: dict): + download_dir = operate.get("download_dir") + package_path = operate.get("package_path") + package_lock_path = operate.get("package_lock_path") + hash_value = ( + subprocess.run( + ["sha256sum", package_lock_path], capture_output=True, text=True + ) + .stdout.strip() + .split(" ")[0] + ) + hash_dir = os.path.join(download_dir, hash_value) + copy_file(package_path, hash_dir) + copy_file(package_lock_path, hash_dir) + + if not os.path.exists(os.path.join(hash_dir, "npm-install.js")): + npm_install_script = os.path.join( + os.path.dirname(package_path), "npm-install.js" + ) + if os.path.exists(npm_install_script): + shutil.copyfile( + npm_install_script, os.path.join(hash_dir, "npm-install.js") + ) + + result = subprocess.run( + ["npm", "install", "--prefix", hash_dir], capture_output=True, text=True + ) + if result.returncode == 0: + print("npm install completed in the {} directory.".format(hash_dir)) + else: + print("npm dependency installation failed:", result.stderr) + + symlink_src = os.path.join(hash_dir, "node_modules") + symlink_dest = operate.get("symlink") + + if name == "legacy_bin": + for link in operate.get("symlink", []): + symlink_src2dest(symlink_src, link) + return + + if name in ["parse5"]: + copy_folder(symlink_src, symlink_dest) + return + + copy_folder(symlink_src, symlink_dest) + + for copy_entry in operate.get("copy", []): + copy_folder( + copy_entry["src"], copy_entry["dest"] + ) + + for copy_ext_entry in operate.get("copy_ext", []): + copy_folder( + copy_ext_entry["src"], copy_ext_entry["dest"] + ) + + +def install_hpm(npm_tool_path: str, hpm_install_dir: str): + content = """\ +package-lock=true +registry=http://repo.huaweicloud.com/repository/npm +strict-ssl=false +lockfile=false +""" + with os.fdopen( + os.open( + os.path.join(os.path.expanduser("~"), ".npmrc"), + os.O_WRONLY | os.O_CREAT, + mode=0o640, + ), + "w", + ) as f: + os.truncate(f.fileno(), 0) + f.write(content) + if not os.path.exists(hpm_install_dir): + os.makedirs(hpm_install_dir) + with os.fdopen( + os.open( + os.path.join(hpm_install_dir, "package.json"), + os.O_WRONLY | os.O_CREAT, + mode=0o640, + ), + "w", + ) as f: + os.truncate(f.fileno(), 0) + f.write("{}\n") + node_bin_path = os.path.dirname(npm_tool_path) + os.environ["PATH"] = f"{node_bin_path}:{os.environ['PATH']}" + subprocess.run( + [ + npm_tool_path, + "install", + "@ohos/hpm-cli", + "--registry", + "https://repo.huaweicloud.com/repository/npm/", + "--prefix", + hpm_install_dir, + ] + ) + + +def npm_config(npm_tool_path: str, global_args: object) -> tuple: + node_path = os.path.dirname(npm_tool_path) + os.environ["PATH"] = "{}:{}".format(node_path, os.environ.get("PATH")) + if global_args.skip_ssl: + skip_ssl_cmd = "{} config set strict-ssl false;".format(npm_tool_path).split() + _, err, retcode = run_cmd(skip_ssl_cmd) + if retcode != 0: + return False, err.decode() + npm_clean_cmd = "{} cache clean -f".format(npm_tool_path).split() + npm_package_lock_cmd = "{} config set package-lock true".format(npm_tool_path).split() + _, err, retcode = run_cmd(npm_clean_cmd) + if retcode != 0: + return False, err.decode() + _, err, retcode = run_cmd(npm_package_lock_cmd) + if retcode != 0: + return False, err.decode() + return True, None + + +def npm_install(operate: dict, global_args: object, success_installed_npm_config: list) -> tuple: + install_list = operate.get("npm_install_path") + npm_tool_path = os.path.join(global_args.code_dir, "prebuilts/build-tools/common/nodejs/current/bin/npm") + + preset_is_ok, err = npm_config(npm_tool_path, global_args) + if not preset_is_ok: + return preset_is_ok, err + + print("start npm install, please wait.") + for install_path in install_list: + if install_path in success_installed_npm_config: + continue + full_code_path = install_path + basename = os.path.basename(full_code_path) + node_modules_path = os.path.join(full_code_path, "node_modules") + npm_cache_dir = os.path.join("~/.npm/_cacache", basename) + + if os.path.exists(node_modules_path): + print("remove node_modules %s" % node_modules_path) + run_cmd(("rm -rf {}".format(node_modules_path)).split()) + + if os.path.exists(full_code_path): + cmd = ["timeout", "-s", "9", "90s", npm_tool_path, "install", "--registry", global_args.npm_registry, + "--cache", npm_cache_dir] + if global_args.host_platform == "darwin": + cmd = [npm_tool_path, "install", "--registry", global_args.npm_registry, "--cache", npm_cache_dir] + if global_args.unsafe_perm: + cmd.append("--unsafe-perm") + proc = subprocess.Popen( + cmd, cwd=full_code_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + # wait proc Popen with 0.1 second + time.sleep(0.1) + _, err = proc.communicate() + if proc.returncode: + print("in dir:{}, executing:{}".format(full_code_path, " ".join(cmd))) + return False, err.decode() + else: + success_installed_npm_config.append(install_path) + else: + raise Exception( + "{} not exist, it shouldn't happen, pls check...".format(full_code_path) + ) + print(f"{node_modules_path} install over!") + return True, None \ No newline at end of file diff --git a/prebuilts_service/config_parser.py b/prebuilts_service/config_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..50254416e285a05f6eb7a8ec5ab40ced1e3feee0 --- /dev/null +++ b/prebuilts_service/config_parser.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import re +import os +from common_utils import load_config + + +class ConfigParser: + def __init__(self, config_file: str, global_args): + self.data = load_config(config_file) + self.current_cpu = global_args.host_cpu + self.current_os = global_args.host_platform + self.input_tag = global_args.tag + self.input_type = global_args.type + self.global_config = { + "code_dir": global_args.code_dir, + "download_root": self.data["download_root"] + } + VarParser.parse_vars(self.global_config, []) + download_root = self.global_config["download_root"] + self.global_config["download_root"] = os.path.abspath(os.path.expanduser(download_root)) + + def get_operate(self) -> tuple: + download_op = [] + other_op = [] + tool_list = self.data["tool_list"] + for tool in tool_list: + _download, _other = self._get_tool_operate(tool) + download_op.extend(_download) + other_op.extend(_other) + return download_op, other_op + + def _get_tool_operate(self, tool) -> tuple: + tool_matched, unified_tool_basic_config = self._is_tool_matched(tool) + if not tool_matched: + return [], [] + + matched_platform_configs = Filter.filter_platform(self.current_os, self.current_cpu, tool.get("config")) + for config in matched_platform_configs: + VarParser.parse_vars(config, [unified_tool_basic_config, self.global_config]) + unified_platform_configs = [] + for conf in matched_platform_configs: + unified_platform_configs.append(self._unify_config(self.global_config, unified_tool_basic_config, conf)) + unified_platform_configs = Filter(unified_platform_configs).apply_filters(self.input_tag, self.input_type) + + handle = tool.get("handle", []) + + if unified_platform_configs: + # 有平台配置则只使用平台配置 + download_operate, other_operate = self._generate_tool_operate(unified_platform_configs, handle) + else: + # 没有平台配置则使用工具配置 + download_operate, other_operate = self._generate_tool_operate([unified_tool_basic_config], handle) + + # 删除存在未知变量的配置 + return VarParser.remove_undefined(download_operate), VarParser.remove_undefined(other_operate) + + def _is_tool_matched(self, tool): + tool_basic_config = {key: tool[key] for key in tool if key not in {"config", "handle"}} + VarParser.parse_vars(tool_basic_config, [self.global_config]) + unified_tool_basic_config = self._unify_config(self.global_config, tool_basic_config) + if not Filter([unified_tool_basic_config]).apply_filters(self.input_tag, self.input_type): + return False, [] + else: + return True, unified_tool_basic_config + + def _generate_tool_operate(self, outer_configs: list, handles: list) -> tuple: + if not outer_configs: + return [], [] + + download_operate = [] + other_operate = [] + + # 根据配置，自动生成下载操作 + for config in outer_configs: + if config.get("remote_url"): + download_config = self._generate_download_config(config) + download_operate.append(download_config) + + # 如果没有其他操作，则返回 + if not handles: + return download_operate, [] + + operates = self._generate_handles(outer_configs, handles) + # 区分下载操作和其他操作 + other_operate = [] + for operate in operates: + if operate["type"] == "download": + download_operate.append(operate) + else: + other_operate.append(operate) + + return download_operate, other_operate + + def _generate_handles(self, outer_configs: list, handles: list): + """ + 为每个配置生成对应的操作列表 + :param configs: 配置列表 + :param handles: 操作列表 + """ + operate_list = [] + for config in outer_configs: + special_handle = config.get("handle_index") + count = 0 + for index, handle in enumerate(handles): + if special_handle and index not in special_handle: + continue + step_id = "_".join([config.get("name"), os.path.basename(config.get("remote_url", "")), str(count)]) + count += 1 + # 不能改变原来的handle + new_handle = copy.deepcopy(handle) + # 解析handle中的变量 + VarParser.parse_vars(new_handle, [config]) + # 生成操作id + new_handle["tool_name"] = config.get("name") + new_handle["step_id"] = step_id + operate_list.append(new_handle) + return operate_list + + def _generate_download_config(self, config): + try: + return { + "remote_url": config["remote_url"], + "unzip_dir": config["unzip_dir"], + "unzip_filename": config["unzip_filename"], + "download_dir": config.get("download_dir", config["download_root"]), + "operate_type": "download", + "name": config.get("name"), + } + except KeyError as e: + print(f"error config: {config}") + raise e + + def _unify_config(self, *additional_configs) -> dict: + unified_config = dict() + for config in additional_configs: + unified_config.update(config) + return unified_config + + +class Filter: + def __init__(self, configs=[]): + self.input_configs = copy.deepcopy(configs) + + @classmethod + def filter_platform(cls, current_os: str, current_cpu: str, config: dict) -> list: + """获取匹配当前操作系统的配置""" + if not config: + return [] + + filtered = [] + + for os_key, os_config in config.items(): + # 逗号分割操作系统名 + configured_os_list = [o.strip() for o in os_key.split(",")] + if current_os in configured_os_list: + # 不配cpu场景 + if isinstance(os_config, list): + filtered.extend(os_config) + continue + # 不配cpu, 仅有一个配置项场景 + if isinstance(os_config, dict) and "remote_url" in os_config: + filtered.extend(os_config) + continue + # 配cpu场景 + filtered.extend(cls.filter_cpu(current_cpu, os_config)) + return filtered + + @classmethod + def filter_cpu(cls, current_cpu: str, os_config: dict) -> list: + filtered = [] + for cpu_str in os_config: + configured_cpu_list = [c.strip() for c in cpu_str.split(",")] + if current_cpu in configured_cpu_list: + cpu_config = os_config[cpu_str] + # cpu配置内部可以是一个配置，也可以是一个配置列表 + if not isinstance(cpu_config, list): + cpu_config = [cpu_config] + filtered.extend(cpu_config) + return filtered + + def apply_filters(self, input_tag: str, input_type: str): + return self.filter_tag(input_tag).filter_type(input_type).result() + + def filter_tag(self, input_tag: str) -> 'Filter': + """过滤tag字段""" + filtered = [] + for config in self.input_configs: + tag = config.get("tag") + if not tag: + filtered.append(config) + continue + # 配置的tag，转set + if isinstance(tag, str): + configured_tags = set([t.strip() for t in tag.split(",")]) + else: + configured_tags = set(tag) + # 输入的tag，转set + input_tags = set([t.strip() for t in input_tag.split(",")]) + + # 检查二者是否有交集，有则添加 + if not input_tags.isdisjoint(configured_tags): + filtered.append(config) + self.input_configs = filtered + return self + + def filter_type(self, input_type: str) -> 'Filter': + """过滤type字段""" + filtered = [] + for config in self.input_configs: + _type = config.get("type") + if not _type: + filtered.append(config) + continue + # 配置的type，转set + if isinstance(_type, str): + configured_types = set([t.strip() for t in _type.split(",")]) + else: + configured_types = set(_type) + # 输入的type，转set + input_types = set([t.strip() for t in input_type.split(",")]) + + # 检查二者是否有交集，有则添加 + if not input_types.isdisjoint(configured_types): + filtered.append(config) + self.input_configs = filtered + return self + + def result(self): + return self.input_configs + + +class VarParser: + var_pattern: re.Pattern = re.compile(r'\$\{.*?\}') # 正则表达式 + + @classmethod + def remove_undefined(cls, configs: list) -> list: + useful_config = [] + for config in configs: + if not cls.has_undefined_var(config): + useful_config.append(config) + return useful_config + + @classmethod + def has_undefined_var(cls, data): + try: + if isinstance(data, str): + return bool(cls.var_pattern.findall(data)) + elif isinstance(data, list): + return any(cls.has_undefined_var(item) for item in data) + elif isinstance(data, dict): + return any(cls.has_undefined_var(value) for value in data.values()) + else: + return False + except AttributeError: + print("var_pattern不是有效的正则表达式对象") + return False + + @classmethod + def parse_vars(cls, data: dict, dictionarys: list): + """ + 解析config中的变量, 先自解析, 再按顺序查字典 + :param config: 需要进行变量解析的配置 + :param dictionarys: 字典列表 + """ + cls.replace_vars_in_data(data, data) + for dic in dictionarys: + cls.replace_vars_in_data(data, dic) + + @classmethod + def replace_vars_in_data(cls, data: any, dictionary: dict) -> any: + """用dictionary字典中的值替换data中的变量,data可以为列表、字典、字符串等类型, 变量使用${var_name}形式""" + if isinstance(data, str): + return cls.replace_vars_in_string(data, dictionary) + elif isinstance(data, dict): + for k in list(data.keys()): + original_value = data[k] + new_value = cls.replace_vars_in_data(original_value, dictionary) + if new_value is not original_value: # 仅当original_value为字符串时成立 + data[k] = new_value + elif isinstance(data, list): + for i in range(len(data)): + original_value = data[i] + new_value = cls.replace_vars_in_data(original_value, dictionary) + if new_value is not original_value: + data[i] = new_value + else: + return data + return data + + @classmethod + def replace_vars_in_string(cls, s: str, dictionary: dict) -> str: + + """用dictionary字典中的值替换字符串s中的变量, 变量使用${var_name}形式""" + + replaced_var_names = set() # 避免循环依赖 + + while True: + try: + replaced = cls.var_pattern.sub( + lambda matched_var: cls._replace_var_with_dict_value(matched_var, dictionary, replaced_var_names), + s) + if replaced == s: + break + s = replaced + except ValueError as e: + print(f"replace var in string {s} failed") + raise e + return s + + @classmethod + def _replace_var_with_dict_value(cls, matched_var, dictionary, replaced_var_names): + var_name = matched_var.group()[2:-1] + if var_name in replaced_var_names: + raise ValueError(f"Variable \"{var_name}\" is being replaced again.") + if dictionary.get(var_name): + replaced_var_names.add(var_name) + return dictionary.get(var_name, matched_var.group()) # 找得到就替换，找不到就保留原始值 \ No newline at end of file diff --git a/prebuilts_service/download_util.py b/prebuilts_service/download_util.py new file mode 100644 index 0000000000000000000000000000000000000000..2fd4b58f761ec60cf68b070d1be3a2b345378b87 --- /dev/null +++ b/prebuilts_service/download_util.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import os +from common_utils import run_cmd +import threading +import hashlib + + +remote_sha256_cache = dict() +_cache_lock = threading.Lock() + + +def import_rich_module(): + module = importlib.import_module("rich.progress") + progress = module.Progress( + module.TextColumn("[bold blue]{task.fields[filename]}", justify="right"), + module.BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + module.DownloadColumn(), + "•", + module.TransferSpeedColumn(), + "•", + module.TimeRemainingColumn(), + ) + return progress + + +def check_sha256_by_mark(remote_url, unzip_dir: str, unzip_filename: str) -> tuple: + """ + 检查是否存在和远程文件哈希值匹配的解压完成标记文件 + 标记文件名：远程文件哈希值 + '.' + unzip_filename + '.mark' + """ + + + remote_sha256 = get_remote_sha256(remote_url) + mark_file_name = remote_sha256 + '.' + unzip_filename + '.mark' + mark_file_path = os.path.join(unzip_dir, mark_file_name) + return os.path.exists(mark_file_path), mark_file_path + + +def file_sha256(file_path): + sha = hashlib.sha256() + with open(file_path, "rb") as f: + while chunk := f.read(8192): # 分块读取大文件，避免内存问题 + sha.update(chunk) + return sha.hexdigest() + + +def check_sha256(remote_url: str, local_file: str) -> bool: + """ + 检查本地文件的SHA256值是否与远程文件一致 + """ + + remote_sha256 = get_remote_sha256(remote_url) + local_sha256 = file_sha256(local_file) + if remote_sha256 != local_sha256: + print( + "remote file {}.sha256 is not found, begin check SHASUMS256.txt".format( + remote_url + ) + ) + remote_sha256 = obtain_sha256_by_sha_sums256(remote_url) + return remote_sha256 == local_sha256 + + +def get_remote_sha256(remote_url: str) -> str: + """ + 从远程.sha256文件中获取哈希值 + """ + with _cache_lock: # 加锁检查缓存 + if remote_url in remote_sha256_cache: + return remote_sha256_cache[remote_url] + + # 在锁外执行耗时操作（如网络请求） + check_sha256_cmd = f"curl -s -k {remote_url}.sha256" + remote_sha256, _, _ = run_cmd(check_sha256_cmd.split()) + + with _cache_lock: # 加锁更新缓存 + remote_sha256_cache[remote_url] = remote_sha256 + return remote_sha256 + + +def obtain_sha256_by_sha_sums256(remote_url: str) -> str: + """ + 从远程的SHASUMS256.txt中获取SHA256值 + """ + sha_sums256 = "SHASUMS256.txt" + sha_sums256_path = os.path.join(os.path.dirname(remote_url), sha_sums256) + file_name = os.path.basename(remote_url) + cmd = "curl -s -k " + sha_sums256_path + data_sha_sums256, _, _ = run_cmd(cmd.split()) + remote_sha256 = None + for line in data_sha_sums256.split("\n"): + if file_name in line: + remote_sha256 = line.split(" ")[0] + return remote_sha256 + + +def get_local_path(download_root: str, remote_url: str): + """根据远程URL生成本地路径,本地文件名为url的MD5值+远程文件名 + """ + remote_file_name = os.path.basename(remote_url) + remote_url_md5_value = hashlib.md5((remote_url + '\n').encode()).hexdigest() + local_path = os.path.join(download_root, '{}.{}'.format(remote_url_md5_value, remote_file_name)) + return local_path + + +def extract_compress_files_and_gen_mark(source_file: str, unzip_dir: str, mark_file_path: str): + """ + 解压缩文件并生成解压完成标记文件 + 标记文件名：远程文件哈希值 + '.' + unzip_filename + '.mark' + """ + if not os.path.exists(unzip_dir): + os.makedirs(unzip_dir, exist_ok=True) + if source_file.endswith(".zip"): + command = ["unzip", "-qq", "-o", "-d", unzip_dir, source_file] + elif source_file.endswith(".tar.gz"): + command = ["tar", "-xzf", source_file, "-C", unzip_dir] + elif source_file.endswith(".tar.xz"): + command = ["tar", "-xJf", source_file, "-C", unzip_dir] + elif source_file.endswith(".tar"): + command = ["tar", "-xvf", source_file, "-C", unzip_dir] + else: + print("暂不支持解压此类型压缩文件！") + return + + _, err, retcode = run_cmd(command) + if retcode != 0: + print("解压失败，错误信息：", err) + return + else: + with open(mark_file_path, "w") as f: + f.write("0") diff --git a/prebuilts_service/operater.py b/prebuilts_service/operater.py new file mode 100644 index 0000000000000000000000000000000000000000..d7cfe0ca22529f19d62ef0e82b54b8b232ca4a70 --- /dev/null +++ b/prebuilts_service/operater.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import subprocess +from common_utils import ( + symlink_src2dest, + copy_folder, + remove_dest_path, + run_cmd_live, + install_hpm, + install_hpm_in_other_platform, + npm_install, + is_system_component, +) +import re + + +class OperateHanlder: + global_args = None + + @staticmethod + def run(operate_list: list, global_args, unchanged_list: tuple = ()): + ignore_list = [] + OperateHanlder.global_args = global_args + pre_process_tool = "" + for operate in operate_list: + try: + current_tool = re.match(r"(.*)_\d$", operate.get("step_id")).group(1) + shot_name = re.sub(r"(\.[A-Za-z]+)+$", "", current_tool).strip("_") + + if current_tool != pre_process_tool: + print(f"\n==> process {shot_name}") + pre_process_tool = current_tool + + if current_tool in ignore_list: + continue + + getattr(OperateHanlder, "_" + operate.get("type"))(operate) + except Exception as e: + if current_tool in unchanged_list: + ignore_list.append(current_tool) + print(f"<== ignore process {shot_name}") + continue + else: + raise e + + @staticmethod + def _symlink(operate: dict): + src = operate.get("src") + dest = operate.get("dest") + symlink_src2dest(src, dest) + + @staticmethod + def _copy(operate: dict): + src = operate.get("src") + dest = operate.get("dest") + try: + shutil.copy2(src, dest) + except IsADirectoryError: + copy_folder(src, dest) + print(f"copy {src} ---> dest: {dest}") + + @staticmethod + def _remove(operate: dict): + path = operate.get("path") + if isinstance(path, list): + for p in path: + remove_dest_path(p) + else: + remove_dest_path(path) + print(f"remove {path}") + + @staticmethod + def _move(operate: dict): + src = operate.get("src") + dest = operate.get("dest") + + filetype = operate.get("filetype", None) + if filetype: + file_list = os.listdir(src) + for file in file_list: + if file.endswith(filetype): + file_path = os.path.join(src, file) + shutil.move(file_path, dest) + print(f"move {file_path} ---> dest: {dest}") + else: + shutil.move(src, dest) + print(f"move {src} ---> dest: {dest}") + + @staticmethod + def _shell(operate: dict): + cmd = operate.get("cmd") + run_cmd_live(cmd) + + + @staticmethod + def _hpm_download(operate: dict): + name = operate.get("name") + download_dir = operate.get("download_dir") + npm_tool_path = os.path.join(OperateHanlder.global_args.code_dir, "prebuilts/build-tools/common/nodejs/current/bin/npm") + symlink_dest = operate.get("symlink") + if "@ohos/hpm-cli" == name: + install_hpm(npm_tool_path, download_dir) + symlink_src2dest(os.path.join(download_dir, "node_modules"), symlink_dest) + return + else: + install_hpm_in_other_platform(name, operate) + + @staticmethod + def _npm_install(operate: dict, max_retry_times=2): + # 若不是系统组件，直接返回 + if not is_system_component(): + return + success_installed_npm_config = [] + + for retry_times in range(max_retry_times + 1): + try: + result, error = npm_install(operate, OperateHanlder.global_args, success_installed_npm_config) + if result: + return + print("npm install error, error info: %s", error) + except Exception as e: + print("An unexpected error occurred during npm install: %s", str(e)) + error = str(e) + + # 重试次数超过最大限制，处理错误日志 + for error_info in error.split("\n"): + if error_info.endswith("debug.log"): + log_path = error_info.split()[-1] + try: + # 读取日志文件内容 + result = subprocess.run( + ["cat", log_path], + capture_output=True, + text=True, + timeout=60 + ) + print("npm debug log content:\n%s", result.stdout) + except subprocess.TimeoutExpired: + print("Reading npm debug log timed out after 60 seconds.") + except Exception as e: + print("Error reading npm debug log: %s", str(e)) + break + + # 抛出最终异常 + raise Exception("npm install error with three times, prebuilts download exit") + + @staticmethod + def _node_modules_copy(operate: dict): + if not is_system_component(): + return + + copy_list = operate.get("copy_list") + for copy_config in copy_list: + src_dir = copy_config.get("src") + if not os.path.exists(src_dir): + print(f"{src_dir} not exist, skip node_modules copy.") + continue + dest_dir = copy_config.get("dest") + use_symlink = copy_config.get("use_symlink") + if os.path.exists(os.path.dirname(dest_dir)): + print("remove", os.path.dirname(dest_dir)) + shutil.rmtree(os.path.dirname(dest_dir)) + if use_symlink == "True" and OperateHanlder.global_args.enable_symlink == True: + os.makedirs(os.path.dirname(dest_dir), exist_ok=True) + os.symlink(src_dir, dest_dir) + print(f"symlink {src_dir} ---> dest: {dest_dir}") + else: + shutil.copytree(src_dir, dest_dir, symlinks=True) + print(f"copy {src_dir} ---> dest: {dest_dir}") + diff --git a/prebuilts_service/pool_downloader.py b/prebuilts_service/pool_downloader.py new file mode 100644 index 0000000000000000000000000000000000000000..454c864e78eed280777e357e66d4b6aa269099b8 --- /dev/null +++ b/prebuilts_service/pool_downloader.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from download_util import ( + check_sha256, + check_sha256_by_mark, + extract_compress_files_and_gen_mark, + get_local_path, + run_cmd, + import_rich_module, +) +import os +import sys +import re +import glob +import traceback +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed +from multiprocessing import cpu_count +from urllib.request import urlopen +from functools import partial + + +class PoolDownloader: + def __init__(self, download_configs: list, global_args: object = None): + if not global_args.disable_rich: + self.progress = import_rich_module() + else: + self.progress = None + self.global_args = global_args + self.download_configs = download_configs + self.lock = threading.Lock() + self.unchanged_tool_list = [] + + def start(self) -> list: + if self.progress: + with self.progress: + self._run_download_in_thread_pool() + else: + self._run_download_in_thread_pool() + return self.unchanged_tool_list + + def _run_download_in_thread_pool(self): + try: + cnt = cpu_count() + except Exception as e: + cnt = 1 + with ThreadPoolExecutor(max_workers=cnt) as pool: + tasks = dict() + for config_item in self.download_configs: + task = pool.submit(self._process, config_item) + tasks[task] = os.path.basename(config_item.get("remote_url")) + self._wait_for_download_tasks_complete(tasks) + + def _wait_for_download_tasks_complete(self, tasks: dict): + for task in as_completed(tasks): + try: + _ = task.result() + except Exception as e: + self._adaptive_print(f"Task {task} generated an exception: {e}", style="red") + self._adaptive_print(traceback.format_exc()) + else: + self._adaptive_print( + "{}, download and decompress completed".format(tasks.get(task)), + style="green", + ) + + def _adaptive_print(self, msg: str, **kwargs): + if self.progress: + self.progress.console.log(msg, **kwargs) + else: + print(msg) + + def _process(self, operate: dict): + global_args = self.global_args + remote_url = operate.get("remote_url") + if "python" in remote_url and global_args.glibc_version is not None: + remote_url = re.sub(r"GLIBC[0-9]\.[0-9]{2}", global_args.glibc_version, remote_url) + remote_url = global_args.tool_repo + remote_url + + download_root = operate.get("download_dir") + unzip_dir = operate.get("unzip_dir") + unzip_filename = operate.get("unzip_filename") + local_path = get_local_path(download_root, remote_url) + + mark_file_exist, mark_file_path = check_sha256_by_mark(remote_url, unzip_dir, unzip_filename) + # 检查解压的文件是否和远程一致 + if mark_file_exist: + self._adaptive_print( + "{}, Sha256 markword check OK.".format(remote_url), style="green" + ) + with self.lock: + self.unchanged_tool_list.append(operate.get("name") + "_" + os.path.basename(remote_url)) + else: + # 不一致则先删除产物 + run_cmd(["rm", "-rf"] + glob.glob(f"{unzip_dir}/*.{unzip_filename}.mark", recursive=False)) + run_cmd(["rm", "-rf", '{}/{}'.format(unzip_dir, unzip_filename)]) + # 校验压缩包 + if os.path.exists(local_path): + check_result = check_sha256(remote_url, local_path) + if check_result: + self._adaptive_print( + "{}, Sha256 check download OK.".format(local_path), + style="green", + ) + else: + # 压缩包不一致则删除压缩包，重新下载 + os.remove(local_path) + self._try_download(remote_url, local_path) + else: + # 压缩包不存在则下载 + self._try_download(remote_url, local_path) + + # 解压缩包 + self._adaptive_print("Start decompression {}".format(local_path)) + extract_compress_files_and_gen_mark(local_path, unzip_dir, mark_file_path) + self._adaptive_print(f"{local_path} extracted to {unzip_dir}") + + def _try_download(self, remote_url: str, local_path: str): + max_retry_times = 3 + # 创建下载目录 + download_dir = os.path.dirname(local_path) + os.makedirs(download_dir, exist_ok=True) + + # 获取进度条和任务 ID + progress = self.progress + progress_task_id = progress.add_task( + "download", filename=os.path.basename(remote_url), start=False + ) if progress else None + self._adaptive_print(f"Downloading {remote_url}") + for retry_times in range(max_retry_times): + try: + self._download_remote_file(remote_url, local_path, progress_task_id) + return + except Exception as e: + error_message = getattr(e, 'code', str(e)) + self._adaptive_print( + f"Failed to open {remote_url}, Error: {error_message}", + style="red" + ) + + # 重试次数达到上限，下载失败 + self._adaptive_print( + f"{local_path}, download failed after {max_retry_times} retries, " + "please check network status. Prebuilts download exit." + ) + sys.exit(1) + + def _download_remote_file(self, remote_url: str, local_path: str, progress_task_id): + buffer_size = 32768 + progress = self.progress + with urlopen(remote_url) as response: + total_size = int(response.info().get("Content-Length", 0)) + + if progress: + progress.update(progress_task_id, total=total_size) + progress.start_task(progress_task_id) + + with open(local_path, "wb") as dest_file: + for data in iter(partial(response.read, buffer_size), b""): + dest_file.write(data) + self._update_progress(progress_task_id, len(data)) + self._adaptive_print(f"Downloaded {local_path}") + + def _update_progress(self, task_id, advance): + if self.progress: + self.progress.update(task_id, advance=advance)