diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..e86e31e5309c9ffe308b0ca0984612c64cf32c78 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +testcase/command-execute-chroot.sh text eol=lf +testcase/benchmark-command-execute-chroot.sh text eol=lf +testcase/lib/chroot-package-worker.sh text eol=lf +testcase/tests/test-command-execute-chroot.sh text eol=lf diff --git a/README.md b/README.md index 8597cde318ca878902ff2980c0e1c9c1ef26c70a..d4125704b145b544fdd2d8d2d6c26e1e58967d76 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ -# TencentOS发行版检查 -检查项目包括 -- KABI -- 软件包安装 -- 内核模块加载 -- systemd服务启动 \ No newline at end of file +# TencentOS发行版检查 +检查项目包括 +- KABI +- 软件包安装 +- 内核模块加载 +- systemd服务启动 + +## chroot 并发软件包测试 + +`testcase/command-execute-chroot.sh` 可在相互隔离的 chroot 中并发执行软件包安装、命令、服务与卸载检查。使用方法、结果格式与隔离模型见 [testcase/README-command-execute-chroot.md](testcase/README-command-execute-chroot.md);OpenCloudOS 8.10、9.4 和 Stream 23 的真实结果见 [testcase/VALIDATION-command-execute-chroot.md](testcase/VALIDATION-command-execute-chroot.md)。 + +无需 root 的回归测试: + +```bash +make -C testcase check-command-execute-chroot +``` diff --git a/testcase/Makefile b/testcase/Makefile index 1b4db5bcbf65a82a67d32917028e601b2dd629ba..8bdca4cc2425ce1b95a7e1bc1f9a833ed69820bb 100644 --- a/testcase/Makefile +++ b/testcase/Makefile @@ -7,8 +7,13 @@ CFLAGS += LDFLAGS += LDFLAGS += +.PHONY: check-command-execute-chroot + all: $(ALL_BIN) +check-command-execute-chroot: + bash tests/test-command-execute-chroot.sh + $(ALL_BIN):%.test:%.o $(CC) $^ -o $@ $(LDFLAGS) diff --git a/testcase/README-command-execute-chroot.md b/testcase/README-command-execute-chroot.md new file mode 100644 index 0000000000000000000000000000000000000000..c81ed58fb3a1b0ed18b4e245420142bb6ab71252 --- /dev/null +++ b/testcase/README-command-execute-chroot.md @@ -0,0 +1,264 @@ +# 基于 chroot 的并发软件包测试 + +`command-execute-chroot.sh` 是 `command-execute-whitelist.sh` 的独立并发入口,不修改现有串行用例。它为每个待测包创建独立 rootfs,在其中完成: + +```text +install → command → service → remove → 原子结果落盘 → 分类汇总 +``` + +实现对应犀牛鸟任务 IJV5MW,保留现有 `package-command-result-*.txt` 的核心列,同时增加详细原因、阶段耗时、运行清单和清理审计。 + +OpenCloudOS 8.10、9.4、Stream 23 的真实运行、`-j 1/2/4/8` 数据、清理证据和局限说明见 [VALIDATION-command-execute-chroot.md](VALIDATION-command-execute-chroot.md)。 + +## 设计目标 + +- 隔离:一个包的安装、脚本或卸载异常不会污染其他包。 +- 并发:`-j N` 严格限制同时运行的包级 worker 数量。 +- 可判定:每一阶段只输出 `PASS`、`FAIL` 或 `SKIP`,跳过必须带原因。 +- 可追踪:每个包有独立日志,报告按输入顺序合并,不受完成先后影响。 +- 可清理:正常结束、超时、`SIGINT`、`SIGTERM`、`SIGHUP` 都会回收进程、挂载和 rootfs。 +- 可复现:运行清单记录 OS、内核、Bash、参数、输入摘要和执行时间。 +- 向后兼容:不使用 Bash 5.1 的 `wait -n -p`,支持 OpenCloudOS 8 常见的 Bash 4.4。 + +## 隔离模型 + +默认 `--isolation auto` 会探测内核能力: + +1. 可用时,每个 worker 进入独立 mount namespace 和 PID namespace; +2. 在私有 mount namespace 中挂载 `/dev`、独立 `/proc`、`/sys` 和结果目录; +3. PID namespace 的 init 退出时,内核回收其中所有后代进程; +4. rootfs 优先使用独立 overlay upper/work,内核不支持时自动回退为 reflink/copy; +5. namespace 不可用时回退到共享挂载模式,并逐一跟踪、反向卸载和审计。 + +因此隔离不是只有目录层面的 `chroot`:包内进程无法看到其他 worker 的 PID,包级挂载也不会进入宿主的全局 mount namespace。网络仍与宿主共享,保证 DNF 可以访问软件源;测试 rootfs 不包含任何宿主业务目录。 + +## 环境要求 + +- OpenCloudOS 8、OpenCloudOS 9 或 OpenCloudOS Stream 23; +- root 权限; +- Bash 4 或更高版本; +- DNF、util-linux(`unshare`/`mount`/`findmnt`/`setsid`)、procps-ng、coreutils; +- 能访问所测版本的软件仓库; +- overlay 可选,不支持时可用 `--root-mode copy`。 + +建议在专用 QEMU/测试机中执行。脚本会运行软件包提供的 ELF 命令,因此不应在承载业务的生产主机上直接运行。 + +## 输入 + +必须传入三个文件: + +1. 待测软件包列表; +2. 桌面软件包白名单; +3. 需要特殊配置的服务软件包列表。 + +每行只读取第一列;空行和 `#` 注释被忽略。例如: + +```text +# pkg.list +bc +grep +openssh-server +nonexistent-package-for-negative-test +``` + +白名单既可以写请求值,也可以写安装后从 RPM 数据库取得的真实包名: + +```text +# desktop.list +firefox + +# service.list +openssh-server +``` + +待测列表保留重复项,用于验证同一包的重复独立执行;两个白名单会自动去重。 + +## 快速开始 + +使用宿主当前版本的软件源创建一次临时基础 rootfs: + +```bash +sudo bash testcase/command-execute-chroot.sh \ + -j 4 \ + --releasever 9 \ + --output-dir ./artifacts/oc9-j4 \ + ./pkg.list ./desktop.list ./service.list +``` + +显式选择 copy 模式: + +```bash +sudo bash testcase/command-execute-chroot.sh \ + -j 2 --root-mode copy \ + ./pkg.list ./desktop.list ./service.list +``` + +复用已经准备好的、与目标软件源匹配的基础 rootfs: + +```bash +sudo bash testcase/command-execute-chroot.sh \ + -j 8 \ + --base-rootfs /var/lib/yum-ci/base-oc9 \ + --repo-dir /etc/yum.repos.d \ + ./pkg.list ./desktop.list ./service.list +``` + +基础 rootfs 只作为 overlay lowerdir 或 copy 来源使用,脚本不会修改 `--base-rootfs` 指定的目录。 + +## 主要选项 + +| 选项 | 默认值 | 说明 | +| --- | --- | --- | +| `-j, --jobs N` | `min(CPU, 8)` | 同时运行的包级 worker 数量 | +| `--package-timeout SEC` | `1200` | 单包完整生命周期上限 | +| `--stage-timeout SEC` | `300` | 安装/卸载阶段上限 | +| `--command-timeout SEC` | `3` | 单个命令探测上限 | +| `--service-timeout SEC` | `30` | 单次 systemd 操作上限 | +| `--bootstrap-timeout SEC` | `1200` | 创建基础 rootfs 的上限 | +| `--root-mode` | `auto` | `auto`、`overlay` 或 `copy` | +| `--isolation` | `auto` | `auto`、`namespace` 或 `shared` | +| `--service-mode` | `auto` | `auto`、`skip` 或 `require` | +| `--base-rootfs DIR` | 无 | 复用只读基础 rootfs | +| `--releasever VERSION` | `auto` | 创建基础 rootfs 使用的版本 | +| `--repo-dir DIR` | `/etc/yum.repos.d` | 目标版本 `.repo` 文件目录 | +| `--work-dir DIR` | `/var/tmp` | 临时 rootfs 的父目录 | +| `--output-dir DIR` | 当前目录 | 报告与持久日志目录 | + +## 阶段判定 + +### 安装 + +- 普通包执行 `dnf/yum install`; +- 基础 rootfs 已安装的包执行 `reinstall`,避免把“本来就存在”误记为安装成功; +- 模块包会尝试识别并启用对应 module stream; +- 包管理器非零、超时、scriptlet/file 错误或安装后 RPM 查询失败均为 `FAIL`; +- 记录 source RPM、完整 binary NEVRA 和失败日志中的关键原因。 + +### 命令 + +- 桌面白名单命中时为 `SKIP`; +- 从 RPM 文件清单中提取 `/usr/bin`、`/usr/sbin` 下的 ELF 可执行文件; +- 依次尝试 `--help`、`--version` 和 `strace execve`,复用原串行用例的判定思路; +- 被测命令的标准输入固定为 `/dev/null`,命令列表通过独立文件描述符读取,避免 `funzip` 等程序吞掉后续路径; +- 缺动态库、损坏软链接、SIGSEGV/SIGABRT/SIGBUS 或 execve 失败为 `FAIL`; +- 包没有 ELF 命令时为 `PASS`,原因会明确写入详细报告。 + +### 服务 + +通过 RPM 文件清单识别 `.service`、`.socket`、`.target`: + +- 没有 unit:`PASS`; +- 位于参数 3:`SKIP`,原因是需要特殊配置; +- template unit:`SKIP`,原因是缺少实例名/配置; +- `--service-mode skip`:全部明确 `SKIP`; +- `--service-mode auto`:仅当 chroot 内确实能连接 systemd 时才执行 `start → is-active → stop`;普通 chroot 无 PID1 时为 `SKIP`,不会误报 `FAIL`; +- `--service-mode require`:要求 systemd 可用,否则为 `FAIL`,适用于预先准备好的 systemd-capable 测试环境。 + +每个 unit 的 start/status/stop 结论写在对应包的 `service.log`。 + +### 卸载 + +- 执行 `dnf/yum remove` 后再次用 RPM 数据库确认包已不存在; +- 非零、超时、scriptlet 错误或包仍存在均为 `FAIL`; +- 基础 rootfs 预装包为保护测试工具链而标记 `SKIP`,不会把无法安全卸载伪装成 `PASS`;该 worker 的 rootfs 仍会整体丢弃,不污染下一包。 + +## 输出 + +每次运行产生: + +- `package-command-result-chroot-*.txt`:兼容现有格式的主报告和中文分类汇总; +- `package-command-detail-chroot-*.tsv`:状态、原因、阶段耗时、命令数、unit 数和预装标记; +- `package-command-metrics-chroot-*.tsv`:按输入顺序的轻量性能数据; +- `yum-ci-chroot-*/packages/<序号>-<包名>/`:逐包日志; +- `yum-ci-chroot-*/run-manifest.txt`:环境、参数、输入 SHA-256 和执行时间; +- `yum-ci-chroot-*/cleanup-audit.txt`:挂载、worker 和临时 rootfs 的最终审计。 + +主报告前六列保持: + +```text +# source-package binary_package install command service remove +``` + +汇总使用包级优先级 `FAIL > SKIP > PASS`,并强制验证: + +```text +通过 + 失败 + 跳过 = 待测总数 +``` + +若任一阶段失败,程序返回 `1`;参数或基础设施错误返回 `2`;仅有 PASS/SKIP 时返回 `0`。 + +## 并发性能验证 + +性能对比必须复用同一基础 rootfs、同一软件源、同一包列表,并至少运行 `-j 1/2/4/8`。辅助脚本会读取每轮 manifest 中排除 bootstrap 的包测试耗时: + +```bash +sudo JOBS_LIST="1 2 4 8" BENCHMARK_REPEAT=3 \ + bash testcase/benchmark-command-execute-chroot.sh \ + /var/lib/yum-ci/base-oc9 \ + /etc/yum.repos.d \ + ./pkg.list ./desktop.list ./service.list \ + ./artifacts/benchmark-oc9 +``` + +输出 `benchmark.tsv` 和 `benchmark-environment.txt`。前者包括并发度、重复轮次、退出码、执行毫秒数、PASS/FAIL/SKIP、相对 `-j 1` 加速比和并行效率;后者自动记录内核、CPU、内存、文件系统和输入哈希。每次应使用新的输出目录,脚本会拒绝复用已有结果的轮次目录。正式报告建议同时注明: + +- 首轮/热缓存状态; +- 每个并发度至少 3 次的中位数; +- 包列表 SHA-256; +- 测试前后 `findmnt`、`pgrep` 和工作目录检查结果。 + +## 跨版本验证矩阵 + +建议在真实 OpenCloudOS 环境分别执行: + +| 环境 | 建议验证 | +| --- | --- | +| OpenCloudOS 8.x | Bash 4.4 调度、copy/overlay 回退、DNF installroot | +| OpenCloudOS 9.x | namespace + overlay 主路径、服务 SKIP 原因、并发基准 | +| OpenCloudOS Stream 23 | releasever/repo 适配、安装/卸载、信号清理 | + +每个环境至少包含:普通 PASS 包、桌面白名单、特殊服务包、不存在包、命令失败注入、`SIGTERM` 中断和 `-j 1/2/4/8`。 + +## 回归测试 + +无需 root、无需修改宿主软件环境: + +```bash +make -C testcase check-command-execute-chroot +``` + +测试覆盖: + +- 参数和不安全包名拒绝; +- 基准目录防覆盖和既有汇总文件保留; +- worker 六类阶段组合; +- 8 包混合并发和并发上限; +- 报告顺序、20 字段明细和分类原因; +- 生命周期超时后的合成失败结果; +- 超时后子孙进程回收,以及会读取 stdin 的命令枚举回归; +- `PASS + FAIL + SKIP` 对账; +- `/dev`、私有 `/proc` 等挂载路径; +- 临时 rootfs、挂载点和 worker 零残留,并验证清理失败不会覆盖原始阶段结果。 + +建议提交前额外执行: + +```bash +bash -n testcase/command-execute-chroot.sh +bash -n testcase/lib/chroot-package-worker.sh +bash -n testcase/tests/test-command-execute-chroot.sh +bash -n testcase/benchmark-command-execute-chroot.sh +shellcheck testcase/command-execute-chroot.sh \ + testcase/lib/chroot-package-worker.sh \ + testcase/tests/test-command-execute-chroot.sh \ + testcase/benchmark-command-execute-chroot.sh +git diff --check +``` + +## 故障排查 + +- `overlay ... failed`:使用 `--root-mode copy`,并确认临时目录空间足够。 +- `namespace ... unavailable`:确认以 root 运行且内核允许 `unshare`;必要时使用 `--isolation shared`。 +- `base rootfs preparation failed`:检查 `base-rootfs-bootstrap.log`、releasever 和 `.repo` 是否匹配。 +- service 全部 SKIP:普通 chroot 的预期行为;详见逐包 `service.log`,不要把无 PID1 当作服务缺陷。 +- 报告出现 lifecycle timeout:查看该包的 `chroot.log`;超时结果会覆盖被中断 worker 留下的部分结果。 +- cleanup audit 失败:脚本会保留无法安全清理的目录并返回非零,禁止对未知目录做递归删除。 diff --git a/testcase/VALIDATION-command-execute-chroot.md b/testcase/VALIDATION-command-execute-chroot.md new file mode 100644 index 0000000000000000000000000000000000000000..2e6055afeccfb81fd7e2ca280abc30f7e2bf0157 --- /dev/null +++ b/testcase/VALIDATION-command-execute-chroot.md @@ -0,0 +1,98 @@ +# `command-execute-chroot.sh` 真实环境验证报告 + +验证日期:2026-07-26。本报告记录的数据来自真实 OpenCloudOS 用户态、DNF/RPM 仓库和 Linux namespace/overlay 内核路径,不是 mock 结果。无 root 回归测试另用于覆盖故障注入和边界条件。 + +## 环境与可复现性 + +三个官方 GenericCloud 基础镜像保持只读,每个版本使用独立 qcow2 overlay。虚拟机使用 QEMU 11.0.3 TCG、4 vCPU、4 GiB 内存;镜像下载后与官方校验文件逐一核对。 + +| 环境 | 镜像校验 | 内核 | Bash | DNF / RPM | namespace / overlay | 原生回归 | +| --- | --- | --- | --- | --- | --- | --- | +| OpenCloudOS 8.10 | MD5 `05a5de885310a87e5495a5144aa187ab` | `5.4.241-30.17.27` | 4.4.20 | 4.7.0 / 4.14.3 | PASS / PASS | PASS, 18.7 s | +| OpenCloudOS 9.4 | SHA-256 `b4bd8ce239e5f942b76dc9e8061f27c3a71402d29122aab9be05809694f37e04` | `6.6.119-47.8.oc9` | 5.2.15 | 4.16.2 / 4.18.2 | PASS / PASS | PASS, 24.8 s | +| OpenCloudOS Stream 23 | SHA-256 `1da444d247231fd47333280f16fbc228dd6ee5858019dca6803cfc37c5fd7425` | `6.6.117-45.ocs23` | 5.2.15 | 4.16.2 / 4.18.2 | PASS / PASS | PASS, 17.8 s | + +原生回归命令: + +```bash +bash testcase/tests/test-command-execute-chroot.sh +``` + +真实包测试的命令形式: + +```bash +sudo bash testcase/command-execute-chroot.sh \ + -j 2 --repo-dir /etc/yum.repos.d --releasever "${VERSION_ID}" \ + --root-mode auto --isolation namespace \ + --bootstrap-timeout 1200 --package-timeout 1200 --stage-timeout 900 \ + --command-timeout 5 --service-timeout 20 \ + --output-dir ./artifacts \ + ./packages.list ./desktop.list ./service.list +``` + +控制器在 8.10 和 Stream 23 中自行通过当前系统 repo bootstrap rootfs,因此同时覆盖了 `dnf --installroot`、GPG 签名、releasever 和 repo 复制路径。9.4 功能和性能测试复用同一个已验证的只读基础 rootfs。 + +## 跨版本功能结果 + +| 环境 | 请求包 | 安装 | ELF 命令 | service | 卸载 | 整体 | 要点 | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 8.10 | `bc` | PASS | PASS (2) | PASS (0 unit) | PASS | PASS | 普通命令包 | +| 8.10 | `chrony` | PASS | PASS (2) | SKIP (3 units) | PASS | SKIP | 普通 chroot 无 systemd/PID1,明确记录未执行 start/status/stop | +| 9.4 | `bc` | PASS | PASS (2) | PASS (0 unit) | PASS | PASS | 普通命令包 | +| 9.4 | `tree` | PASS | PASS (1) | PASS (0 unit) | PASS | PASS | 普通命令包 | +| 9.4 | `chrony` | PASS | PASS (2) | SKIP (2 units) | PASS | SKIP | 无 PID1 的明确 SKIP | +| 9.4 | `openssh-server` | PASS | PASS (1) | SKIP (5 units) | PASS | SKIP | 命中特殊服务配置列表 | +| Stream 23 | `tree` | PASS | PASS (1) | PASS (0 unit) | PASS | PASS | Stream repo/releasever 路径 | +| Stream 23 | `openssh-server` | PASS | PASS (1) | SKIP (5 units) | PASS | SKIP | 命中特殊服务配置列表 | + +三个环境的汇总均满足 `PASS + FAIL + SKIP = total`,真实功能矩阵合计 5 PASS、0 FAIL、3 SKIP。SKIP 是显式的可审计结果,没有把未执行的 systemd 操作伪装为 PASS。 + +## 清理与隔离审计 + +| 环境/运行 | 挂载残留 | worker 残留 | 工作 rootfs 已删除 | audit | +| --- | ---: | ---: | ---: | --- | +| 8.10 功能矩阵 | 0 | 0 | 1 | PASS | +| 9.4 四包矩阵 | 0 | 0 | 1 | PASS | +| 9.4 基准 `-j 1` | 0 | 0 | 1 | PASS | +| 9.4 基准 `-j 2` | 0 | 0 | 1 | PASS | +| 9.4 基准 `-j 4` | 0 | 0 | 1 | PASS | +| 9.4 基准 `-j 8` | 0 | 0 | 1 | PASS | +| Stream 23 功能矩阵 | 0 | 0 | 1 | PASS | + +每次运行结束后还从虚拟机宿主视角再次执行 `findmnt`、`/var/tmp/yum-ci-chroot.*` 目录检查和进程检查,未发现额外残留。 + +## OpenCloudOS 9.4 并发基准 + +包列表为 `bc tree jq patch zip lsof time rsync`,SHA-256 为 `413be3611d1ef2fa8d7737d0ad7720c97e7146321506dc7678062f987e354096`。四轮共用同一基础 rootfs、repo 和输入,每轮 8 PASS、0 FAIL、0 SKIP: + +| `-j` | 执行时间 | 相对 `-j 1` 加速 | 并行效率 | 结果 | +| ---: | ---: | ---: | ---: | --- | +| 1 | 472.205 s | 1.000x | 100.0% | 8/0/0 | +| 2 | 278.373 s | 1.696x | 84.8% | 8/0/0 | +| 4 | 192.414 s | 2.454x | 61.4% | 8/0/0 | +| 8 | 213.285 s | 2.214x | 27.7% | 8/0/0 | + +`-j 4` 是该 4 vCPU TCG 环境的最佳点,比串行减少约 59.3% 耗时。`-j 8` 因 CPU 超订阅和 qcow2/DNF 争用回落,说明 `-j` 不应简单设得越大越好。此数据在软件模拟器上每个并发度运行 1 次,用于证明加速曲线和调度上限,不宣称为生产硬件容量结论;正式容量报告应按说明文档每点至少运行 3 次并取中位数。 + +## 实测发现与回归锁定 + +OpenCloudOS 9.4 的 `unzip-6.0-7.oc9` 测试先暴露了 worker 的一个输入隔离缺陷:`funzip` 会读取 stdin,而旧实现正用 stdin 枚举命令,它因此吞掉下一行 `/usr/bin/unzip` 的 `/usr/b`。修复后: + +- 所有被测命令从 `/dev/null` 读取; +- 命令列表使用独立 fd 3; +- 无 root 回归中增加了会主动吞 6 字节的模拟命令。 + +修复后能正确枚举 `funzip`、`unzip`、`unzipsfx`、`zipinfo` 四个 ELF,并捕获到 `zipinfo --help` 的真实崩溃: + +```text +execve("/usr/bin/zipinfo", ["/usr/bin/zipinfo", "--help"], ...) = 0 +--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0x1} --- +``` + +因此该包的 command 阶段保留为 FAIL,而不是为了报表好看而放宽判定。这是特定镜像与软件包下的测试观察,本报告不将它延伸声明为安全漏洞。 + +回归测试还锁定了整个 package session 超时后的子孙进程回收、挂载/rootfs 零残留、清理失败不覆盖原始阶段结果、报告顺序确定性、并发上限和基准目录防覆盖。 + +## 原始证据 + +未将逐包日志提交到源码树,以避免污染仓库。PR 附件 `IJV5MW-evidence-20260726.zip` 包含 8.10、9.4、Stream 23 的 774 个原始运行文件和一份索引:兼容报告、20 字段明细、metrics、manifest、cleanup audit、bootstrap 日志、逐命令 strace 与 service 日志。附件大小 342,402 字节,SHA-256 为 `4bdccfe4c857798a92dbb1bfe0f71bd04437dafb0693eeb018e348b55e3f3ba1`。 diff --git a/testcase/benchmark-command-execute-chroot.sh b/testcase/benchmark-command-execute-chroot.sh new file mode 100755 index 0000000000000000000000000000000000000000..b9ef577edbfeff57f7e692e6a320345c2339c82c --- /dev/null +++ b/testcase/benchmark-command-execute-chroot.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Run a reproducible -j 1/2/4/8 comparison against one immutable base rootfs. + +set -u +set -o pipefail +export LC_ALL=C + +usage() { + cat <<'EOF' +Usage: + benchmark-command-execute-chroot.sh \ + + +Environment: + JOBS_LIST="1 2 4 8" Concurrency values to measure + BENCHMARK_REPEAT=1 Repetitions for each value + EXTRA_ARGS="..." Extra controller options + +The base rootfs and input lists are shared read-only by every run. The resulting +benchmark.tsv reports the controller's package-execution time (base preparation +is excluded), exit status, accounting totals, speedup, and parallel efficiency. +EOF +} + +[ "$#" -eq 6 ] || { usage >&2; exit 2; } + +g_script_dir="$(cd "$(dirname "$0")" && pwd)" +g_controller="$g_script_dir/command-execute-chroot.sh" +g_base_root="$1" +g_repo_dir="$2" +g_package_list="$3" +g_desktop_list="$4" +g_service_list="$5" +g_output="$6" +g_jobs_list="${JOBS_LIST:-1 2 4 8}" +g_repeat="${BENCHMARK_REPEAT:-1}" +g_extra_args="${EXTRA_ARGS:-}" + +[[ "$g_repeat" =~ ^[1-9][0-9]*$ ]] || { echo "BENCHMARK_REPEAT must be positive" >&2; exit 2; } +[ -d "$g_base_root" ] || { echo "base rootfs not found: $g_base_root" >&2; exit 2; } +[ -d "$g_repo_dir" ] || { echo "repo directory not found: $g_repo_dir" >&2; exit 2; } +for file in "$g_package_list" "$g_desktop_list" "$g_service_list"; do + [ -f "$file" ] || { echo "input file not found: $file" >&2; exit 2; } +done + +mkdir -p "$g_output" +g_output="$(realpath "$g_output")" +g_table="$g_output/benchmark.tsv" +g_environment="$g_output/benchmark-environment.txt" + +# Validate the complete destination set before truncating any aggregate file. +for jobs in $g_jobs_list; do + [[ "$jobs" =~ ^[1-9][0-9]*$ ]] || { echo "invalid jobs value: $jobs" >&2; exit 2; } + repeat=1 + while [ "$repeat" -le "$g_repeat" ]; do + run_output="$g_output/j${jobs}-r${repeat}" + if [ -e "$run_output" ] && [ -n "$(find "$run_output" -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "refusing to reuse non-empty benchmark run directory: $run_output" >&2 + exit 2 + fi + repeat=$((repeat + 1)) + done +done + +printf '# jobs\trepeat\texit-code\texecution-ms\tpass\tfail\tskip\tspeedup\tefficiency\tmanifest\n' >"$g_table" +g_baseline_ms="" + +{ + echo "generated_at=$(date --iso-8601=seconds)" + echo "kernel=$(uname -srmo)" + echo "bash=$BASH_VERSION" + echo "base_rootfs=$(realpath "$g_base_root")" + echo "repo_dir=$(realpath "$g_repo_dir")" + echo "jobs_list=$g_jobs_list" + echo "repeat=$g_repeat" + if command -v sha256sum >/dev/null 2>&1; then + echo "packages_sha256=$(sha256sum "$g_package_list" | awk '{ print $1 }')" + echo "desktop_sha256=$(sha256sum "$g_desktop_list" | awk '{ print $1 }')" + echo "service_sha256=$(sha256sum "$g_service_list" | awk '{ print $1 }')" + fi + echo + echo '[os-release]' + tr '\n' ' ' /dev/null || true + echo + if command -v lscpu >/dev/null 2>&1; then echo; echo '[lscpu]'; lscpu; fi + if command -v free >/dev/null 2>&1; then echo; echo '[memory]'; free -h; fi + if command -v df >/dev/null 2>&1; then echo; echo '[filesystem]'; df -T "$g_base_root" "$g_output"; fi +} >"$g_environment" + +for jobs in $g_jobs_list; do + repeat=1 + while [ "$repeat" -le "$g_repeat" ]; do + run_output="$g_output/j${jobs}-r${repeat}" + mkdir -p "$run_output" + # EXTRA_ARGS is intentionally split so callers may pass normal CLI flags. + # shellcheck disable=SC2086 + bash "$g_controller" -j "$jobs" --base-rootfs "$g_base_root" \ + --repo-dir "$g_repo_dir" --output-dir "$run_output" $g_extra_args \ + "$g_package_list" "$g_desktop_list" "$g_service_list" \ + >"$run_output/console.log" 2>&1 + rc=$? + manifest="$(find "$run_output" -mindepth 2 -maxdepth 2 -name run-manifest.txt -print -quit)" + if [ ! -s "$manifest" ]; then + echo "run j=$jobs repeat=$repeat produced no manifest" >&2 + exit 2 + fi + execution_ms="$(sed -n 's/^execution_ms=//p' "$manifest")" + pass="$(sed -n 's/^pass=//p' "$manifest")" + fail="$(sed -n 's/^fail=//p' "$manifest")" + skip="$(sed -n 's/^skip=//p' "$manifest")" + [[ "$execution_ms" =~ ^[1-9][0-9]*$ ]] || { echo "invalid execution time in $manifest" >&2; exit 2; } + [ -n "$g_baseline_ms" ] || g_baseline_ms="$execution_ms" + speedup="$(awk -v base="$g_baseline_ms" -v current="$execution_ms" 'BEGIN { printf "%.3f", base / current }')" + efficiency="$(awk -v value="$speedup" -v jobs="$jobs" 'BEGIN { printf "%.1f%%", 100 * value / jobs }')" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$jobs" "$repeat" "$rc" "$execution_ms" "$pass" "$fail" "$skip" \ + "$speedup" "$efficiency" "$manifest" >>"$g_table" + repeat=$((repeat + 1)) + done +done + +cat "$g_table" +echo "Benchmark report: $g_table" +echo "Benchmark environment: $g_environment" diff --git a/testcase/command-execute-chroot.sh b/testcase/command-execute-chroot.sh new file mode 100755 index 0000000000000000000000000000000000000000..c7bb4bc0b4b3bfe5ac29f743ff546214fa4b0530 --- /dev/null +++ b/testcase/command-execute-chroot.sh @@ -0,0 +1,1125 @@ +#!/bin/bash +############################################################################### +# @用例ID: 20260726-000000-000000001 +# @用例名称: command-execute-chroot +# @用例级别: 2 +# @用例类型: 功能测试 +# @自动化: 1 +############################################################################### +# Concurrent install -> command -> service -> remove checks in isolated chroots. + +set -u +set -o pipefail +export LC_ALL=C + +g_script_dir="$(cd "$(dirname "$0")" && pwd)" +g_script_path="$g_script_dir/$(basename "$0")" +g_worker="$g_script_dir/lib/chroot-package-worker.sh" + +g_cpu_count="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)" +[[ "$g_cpu_count" =~ ^[1-9][0-9]*$ ]] || g_cpu_count=1 +[ "$g_cpu_count" -le 8 ] || g_cpu_count=8 +g_jobs="$g_cpu_count" +g_stage_timeout=300 +g_package_timeout=1200 +g_bootstrap_timeout=1200 +g_command_timeout=3 +g_service_timeout=30 +g_service_mode=auto +g_releasever=auto +g_root_mode=auto +g_isolation=auto +g_base_root_input="" +g_base_root="" +g_repo_dir=/etc/yum.repos.d +g_output_parent="$(pwd)" +g_work_parent=/var/tmp +g_work_root="" +g_run_dir="" +g_run_id="yum-ci-chroot-$(date '+%Y%m%d%H%M%S')-$$" +g_execution_started_ms=0 +g_execution_finished_ms=0 +g_total=0 +g_pass_count=0 +g_fail_count=0 +g_skip_count=0 +g_cleanup_done=0 +g_signal_code=0 +g_launch_pid="" +g_pids=() +g_session_ids=() + +g_dnf_bin="${YUM_CI_DNF_BIN:-dnf}" +g_chroot_bin="${YUM_CI_CHROOT_BIN:-chroot}" +g_mount_bin="${YUM_CI_MOUNT_BIN:-mount}" +g_umount_bin="${YUM_CI_UMOUNT_BIN:-umount}" +g_findmnt_bin="${YUM_CI_FINDMNT_BIN:-findmnt}" +g_unshare_bin="${YUM_CI_UNSHARE_BIN:-unshare}" + +usage() { + cat <<'EOF' +Usage: + command-execute-chroot.sh [options] + +Required arguments: + pkg.list Packages to test, one package spec per line + desktop.list Packages whose binary-command phase must be skipped + service.list Packages whose services need special configuration + +Options: + -j, --jobs N Concurrent package workers (default: min(CPUs, 8)) + --stage-timeout SEC Install/remove operation timeout (default: 300) + --package-timeout SEC Whole package lifecycle timeout (default: 1200) + --bootstrap-timeout SEC Base-rootfs creation timeout (default: 1200) + --command-timeout SEC One command probe timeout (default: 3) + --service-timeout SEC One systemd operation timeout (default: 30) + --service-mode MODE auto, skip, or require (default: auto) + --base-rootfs DIR Reuse an existing rootfs as the read-only base + --repo-dir DIR Repository configuration (default: /etc/yum.repos.d) + --releasever VERSION DNF releasever; auto detects the host (default: auto) + --root-mode MODE auto, overlay, or copy (default: auto) + --isolation MODE auto, namespace, or shared (default: auto) + --work-dir DIR Temporary-root parent (default: /var/tmp) + --output-dir DIR Reports and retained per-package logs (default: .) + -h, --help Show this help + +The default isolation path gives every package its own mount and PID namespace +as well as its own chroot. If the kernel denies namespaces, auto mode falls back +to carefully tracked shared mounts. Plain chroots normally have no systemd PID1; +auto mode records service start/status/stop as SKIP instead of a false failure. +EOF +} + +die() { + echo "ERROR: $*" >&2 + exit 2 +} + +is_positive_integer() { + [[ "$1" =~ ^[1-9][0-9]*$ ]] +} + +now_ms() { + local value + value="$(date +%s%3N 2>/dev/null || true)" + if [[ "$value" =~ ^[0-9]+$ ]]; then + printf '%s\n' "$value" + else + printf '%s000\n' "$(date +%s)" + fi +} + +safe_slug() { + printf '%s' "$1" | sed 's/[^A-Za-z0-9_.-]/_/g' | cut -c1-56 +} + +clean_field() { + printf '%s' "$*" | + tr '\t\r\n' ' ' | + sed 's/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//' | + cut -c1-800 +} + +normalize_list() { + local input="$1" + local output="$2" + local deduplicate="$3" + + if [ "$deduplicate" = yes ]; then + sed 's/\r$//; s/[[:space:]]*#.*$//' "$input" | + awk 'NF && !seen[$1]++ { print $1 }' >"$output" + else + sed 's/\r$//; s/[[:space:]]*#.*$//' "$input" | + awk 'NF { print $1 }' >"$output" + fi +} + +package_relative_dir() { + local index="$1" + local package="$2" + local slug + slug="$(safe_slug "$package")" + [ -n "$slug" ] || slug=package + printf '%05d-%s' "$index" "$slug" +} + +status_is_valid() { + case "$1" in + PASS | FAIL | SKIP) return 0 ;; + *) return 1 ;; + esac +} + +write_synthetic_result() { + local package="$1" + local package_dir="$2" + local failed_stage="$3" + local reason="$4" + local install=SKIP command=SKIP service=SKIP remove=SKIP + local install_reason="not executed" command_reason="not executed" + local service_reason="not executed" remove_reason="not executed" + local tmp="$package_dir/.synthetic.$$.tmp" + + mkdir -p "$package_dir" + reason="$(clean_field "$reason")" + case "$failed_stage" in + install) + install=FAIL + install_reason="$reason" + command_reason="installation did not complete" + service_reason="installation did not complete" + remove_reason="installation did not complete" + ;; + command) + install=PASS + install_reason="-" + command=FAIL + command_reason="$reason" + service_reason="command phase did not complete" + remove_reason="command phase did not complete" + ;; + service) + install=PASS + command=PASS + install_reason="-" + command_reason="-" + service=FAIL + service_reason="$reason" + remove_reason="service phase did not complete" + ;; + *) + install=PASS + command=PASS + service=PASS + install_reason="-" + command_reason="-" + service_reason="-" + remove=FAIL + remove_reason="$reason" + ;; + esac + + printf '%s\tUNKNOWN\t%s\t%s\t%s\t%s\t%s\tFAIL\t%s\t%s\t%s\t%s\t0\t0\t0\t0\t0\t0\t0\t0\n' \ + "$package" "$package" "$install" "$command" "$service" "$remove" \ + "$install_reason" "$command_reason" "$service_reason" "$remove_reason" >"$tmp" + mv -f -- "$tmp" "$package_dir/result.tsv" + rm -f -- "$package_dir/worker.complete" +} + +mark_cleanup_failure() { + local package="$1" + local package_dir="$2" + local reason="$3" + local result="$package_dir/result.tsv" + local tmp="$package_dir/.cleanup.$$.tmp" + local fields requested source binary install command service remove worker_overall + local install_reason command_reason service_reason remove_reason + local install_ms command_ms service_ms remove_ms total_ms command_count unit_count preinstalled + + fields="$(awk -F '\t' 'NR == 1 { print NF }' "$result" 2>/dev/null || true)" + if [ "$fields" != 20 ]; then + write_synthetic_result "$package" "$package_dir" remove "$reason" + return 0 + fi + IFS=$'\t' read -r requested source binary install command service remove worker_overall \ + install_reason command_reason service_reason remove_reason \ + install_ms command_ms service_ms remove_ms total_ms command_count unit_count preinstalled \ + <"$result" + remove=FAIL + worker_overall=FAIL + remove_reason="$(clean_field "$reason")" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$requested" "$source" "$binary" "$install" "$command" "$service" "$remove" \ + "$worker_overall" "$install_reason" "$command_reason" "$service_reason" \ + "$remove_reason" "$install_ms" "$command_ms" "$service_ms" "$remove_ms" \ + "$total_ms" "$command_count" "$unit_count" "$preinstalled" >"$tmp" + mv -f -- "$tmp" "$result" +} + +detect_releasever() { + local value="" + if [ -s /etc/yum/vars/releasever ]; then + value="$(head -n 1 /etc/yum/vars/releasever | tr -d '[:space:]')" + fi + if [ -z "$value" ] && [ -r /etc/os-release ]; then + value="$(sed -n 's/^VERSION_ID=["'"']\?\([^"'"']*\)["'"']\?$/\1/p' /etc/os-release | head -n 1)" + fi + [ -n "$value" ] || value=9 + printf '%s\n' "$value" +} + +install_repo_configuration() { + local root="$1" + mkdir -p "$root/etc/yum.repos.d" + rm -f -- "$root/etc/yum.repos.d/"*.repo + cp -a "$g_repo_dir/"*.repo "$root/etc/yum.repos.d/" +} + +validate_base_root() { + local root="$1" + if [ "${YUM_CI_CHROOT_TEST_MODE:-0}" = 1 ]; then + [ -f "$root/bin/bash" ] || die "test base rootfs has no /bin/bash: $root" + [ -f "$root/usr/bin/rpm" ] || die "test base rootfs has no rpm: $root" + [ -f "$root/usr/bin/dnf" ] || [ -f "$root/usr/bin/yum" ] || + die "test base rootfs has neither dnf nor yum: $root" + return 0 + fi + [ -x "$root/bin/bash" ] || die "base rootfs has no executable /bin/bash: $root" + [ -x "$root/usr/bin/rpm" ] || [ -x "$root/bin/rpm" ] || + die "base rootfs has no rpm: $root" + [ -x "$root/usr/bin/dnf" ] || [ -x "$root/usr/bin/yum" ] || + die "base rootfs has neither dnf nor yum: $root" +} + +prepare_base_root() { + if [ -n "$g_base_root_input" ]; then + g_base_root="$(realpath "$g_base_root_input")" + validate_base_root "$g_base_root" + return 0 + fi + + g_base_root="$g_work_root/base-rootfs" + mkdir -p "$g_base_root/etc/yum.repos.d" + if [ -d /etc/pki/rpm-gpg ]; then + mkdir -p "$g_base_root/etc/pki" + cp -a /etc/pki/rpm-gpg "$g_base_root/etc/pki/" + fi + echo "Preparing base rootfs: releasever=$g_releasever repos=$g_repo_dir" + timeout --signal=TERM --kill-after=10 "${g_bootstrap_timeout}s" \ + "$g_dnf_bin" -y --installroot="$g_base_root" --releasever="$g_releasever" \ + --setopt="reposdir=$g_repo_dir" --setopt=install_weak_deps=False \ + --setopt=keepcache=True install \ + bash coreutils dnf rpm file strace grep sed gawk findutils util-linux \ + procps-ng systemd >"$g_run_dir/base-rootfs-bootstrap.log" 2>&1 || + die "base rootfs preparation failed; see $g_run_dir/base-rootfs-bootstrap.log" + install_repo_configuration "$g_base_root" + if [ -r /etc/resolv.conf ]; then + cp -L /etc/resolv.conf "$g_base_root/etc/resolv.conf" + fi + validate_base_root "$g_base_root" +} + +path_has_mounts() { + local prefix="$1" + local target + while IFS= read -r target; do + case "$target" in + "$prefix" | "$prefix"/*) return 0 ;; + esac + done < <("$g_findmnt_bin" -rn -o TARGET 2>/dev/null || true) + return 1 +} + +unmount_target() { + local target="$1" + [ -e "$target" ] || return 0 + + for _ in 1 2 3; do + "$g_umount_bin" -R "$target" >/dev/null 2>&1 || true + path_has_mounts "$target" || return 0 + sleep 0.1 + done + "$g_umount_bin" -R -l "$target" >/dev/null 2>&1 || true + path_has_mounts "$target" && return 1 + return 0 +} + +unmount_package_tree() { + local root="$1" + local failed=0 + local target + for target in \ + "$root/run/yum-ci-output" \ + "$root/sys" \ + "$root/proc" \ + "$root/dev" \ + "$root"; do + unmount_target "$target" || failed=1 + done + return "$failed" +} + +safe_remove_package_root() { + local root_dir="$1" + [ -n "$g_work_root" ] || return 1 + case "$root_dir" in + "$g_work_root"/packages/*) ;; + *) echo "ERROR: refusing to clean untrusted package root: $root_dir" >&2; return 1 ;; + esac + [ -e "$root_dir" ] || return 0 + unmount_package_tree "$root_dir/rootfs" || return 1 + if path_has_mounts "$root_dir"; then + echo "ERROR: mounts remain below $root_dir; directory retained" >&2 + return 1 + fi + rm -rf -- "$root_dir" +} + +cleanup_work_root() { + local root_dir + local failed=0 + [ "$g_cleanup_done" -eq 0 ] || return 0 + g_cleanup_done=1 + [ -n "$g_work_root" ] || return 0 + + if [ -d "$g_work_root/packages" ]; then + while IFS= read -r root_dir; do + safe_remove_package_root "$root_dir" || failed=1 + done < <(find "$g_work_root/packages" -mindepth 1 -maxdepth 1 -type d -print) + fi + if path_has_mounts "$g_work_root"; then + echo "ERROR: mount audit failed below $g_work_root; work tree retained" >&2 + return 1 + fi + [ "$failed" -eq 0 ] || return 1 + case "$g_work_root" in + "$g_work_parent"/yum-ci-chroot.*) rm -rf -- "$g_work_root" ;; + *) echo "ERROR: refusing to remove untrusted work root: $g_work_root" >&2; return 1 ;; + esac +} + +setup_package_root() { + local root_dir="$1" + local package_dir="$2" + local root="$root_dir/rootfs" + + mkdir -p "$root_dir/upper" "$root_dir/overlay-work" "$root" + if [ "$g_root_mode" = overlay ]; then + "$g_mount_bin" -t overlay overlay \ + -o "lowerdir=$g_base_root,upperdir=$root_dir/upper,workdir=$root_dir/overlay-work" \ + "$root" || return 1 + else + cp -a --reflink=auto "$g_base_root/." "$root/" || return 1 + fi + + install_repo_configuration "$root" || return 1 + mkdir -p "$root/dev" "$root/proc" "$root/sys" \ + "$root/run/yum-ci-input" "$root/run/yum-ci-output" "$root/etc" + cp -a "$g_worker" "$root/run/yum-ci-input/worker.sh" || return 1 + cp -a "$g_run_dir/input/desktop.list" "$root/run/yum-ci-input/desktop.list" || return 1 + cp -a "$g_run_dir/input/service.list" "$root/run/yum-ci-input/service.list" || return 1 + chmod 0755 "$root/run/yum-ci-input/worker.sh" + if [ -r /etc/resolv.conf ]; then + cp -L /etc/resolv.conf "$root/etc/resolv.conf" || return 1 + fi + + "$g_mount_bin" --rbind /dev "$root/dev" || return 1 + "$g_mount_bin" --make-rslave "$root/dev" || return 1 + if [ "$g_isolation" = namespace ]; then + "$g_mount_bin" -t proc proc "$root/proc" || return 1 + else + "$g_mount_bin" --rbind /proc "$root/proc" || return 1 + "$g_mount_bin" --make-rslave "$root/proc" || return 1 + fi + "$g_mount_bin" --rbind /sys "$root/sys" || return 1 + "$g_mount_bin" --make-rslave "$root/sys" || return 1 + "$g_mount_bin" --bind "$package_dir" "$root/run/yum-ci-output" || return 1 +} + +cleanup_current_root() { + local rc=$? + local root_dir="${g_current_root:-}" + trap - EXIT + if [ -n "$root_dir" ]; then + safe_remove_package_root "$root_dir" || rc=125 + fi + exit "$rc" +} + +run_package_in_chroot() { + local index="$1" + local package="$2" + local relative package_dir root_dir root rc=0 + + relative="$(package_relative_dir "$index" "$package")" + package_dir="$g_run_dir/packages/$relative" + root_dir="$g_work_root/packages/$relative" + root="$root_dir/rootfs" + mkdir -p "$package_dir" + printf '%s\n' "$package" >"$package_dir/requested-package.txt" + : >"$package_dir/chroot.log" + g_current_root="$root_dir" + trap cleanup_current_root EXIT + trap 'exit 130' INT + trap 'exit 143' TERM HUP + + if setup_package_root "$root_dir" "$package_dir" >>"$package_dir/chroot.log" 2>&1; then + YUM_CI_TEST_PACKAGE_DIR="$package_dir" \ + "$g_chroot_bin" "$root" /bin/bash /run/yum-ci-input/worker.sh \ + "$package" /run/yum-ci-input/desktop.list /run/yum-ci-input/service.list \ + /run/yum-ci-output "$g_stage_timeout" "$g_command_timeout" \ + "$g_service_timeout" "$g_service_mode" >>"$package_dir/chroot.log" 2>&1 + rc=$? + else + rc=125 + fi + + if [ "$rc" -eq 0 ] && [ -s "$package_dir/result.tsv" ]; then + : >"$package_dir/worker.complete" + fi + if ! safe_remove_package_root "$root_dir" >>"$package_dir/chroot.log" 2>&1; then + rc=125 + fi + g_current_root="" + trap - EXIT + return "$rc" +} + +supervise_package() { + local index="$1" + local package="$2" + local relative package_dir root_dir rc cleanup_reason="" + local -a command + + relative="$(package_relative_dir "$index" "$package")" + package_dir="$g_run_dir/packages/$relative" + root_dir="$g_work_root/packages/$relative" + mkdir -p "$package_dir" + printf '%s\n' "$package" >"$package_dir/requested-package.txt" + + command=(/bin/bash "$g_script_path" --internal-run-package "$index" "$package") + if [ "$g_isolation" = namespace ]; then + command=("$g_unshare_bin" --mount --pid --fork --kill-child --propagation private "${command[@]}") + fi + + timeout --signal=TERM --kill-after=5 "${g_package_timeout}s" "${command[@]}" + rc=$? + if ! terminate_current_session_children; then + cleanup_reason="worker descendants remained after TERM/KILL escalation" + fi + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ] || [ "$rc" -eq 143 ]; then + write_synthetic_result "$package" "$package_dir" install \ + "package lifecycle timed out after ${g_package_timeout}s" + elif { [ "$rc" -ne 0 ] && { [ "$rc" -ne 125 ] || [ ! -f "$package_dir/worker.complete" ]; }; } || + [ ! -f "$package_dir/worker.complete" ] || [ ! -s "$package_dir/result.tsv" ]; then + write_synthetic_result "$package" "$package_dir" install \ + "package controller exited with status $rc; see chroot.log" + fi + + if ! safe_remove_package_root "$root_dir" >>"$package_dir/chroot.log" 2>&1; then + cleanup_reason="${cleanup_reason:+$cleanup_reason; }mount or rootfs cleanup failed" + fi + if [ -n "$cleanup_reason" ]; then + mark_cleanup_failure "$package" "$package_dir" \ + "chroot cleanup failed: $cleanup_reason; inspect the retained work tree when present" + else + : >"$package_dir/cleanup.ok" + fi + return 0 +} + +terminate_current_session_children() { + local session_id pid attempt process_stat process_tail process_session + local -a children=() + + if [ "${YUM_CI_CHROOT_TEST_MODE:-0}" = 1 ]; then + [ "${YUM_CI_TEST_SESSION_CLEANUP_FAIL:-0}" != 1 ] + return $? + fi + IFS= read -r process_stat <"/proc/$$/stat" || return 1 + process_tail="${process_stat##*) }" + read -r _ _ _ session_id _ <<<"$process_tail" + [[ "$session_id" =~ ^[0-9]+$ ]] || return 1 + for attempt in 1 2 3 4 5 6 7 8 9 10; do + children=() + for process_stat in /proc/[0-9]*/stat; do + [ -r "$process_stat" ] || continue + pid="${process_stat#/proc/}" + pid="${pid%/stat}" + [[ "$pid" =~ ^[0-9]+$ ]] || continue + [ "$pid" = "$$" ] && continue + IFS= read -r process_tail <"$process_stat" 2>/dev/null || continue + process_tail="${process_tail##*) }" + read -r _ _ _ process_session _ <<<"$process_tail" + [ "$process_session" = "$session_id" ] && children+=("$pid") + done + [ "${#children[@]}" -gt 0 ] || return 0 + if [ "$attempt" -eq 1 ]; then + kill -TERM "${children[@]}" >/dev/null 2>&1 || true + elif [ "$attempt" -eq 6 ]; then + kill -KILL "${children[@]}" >/dev/null 2>&1 || true + fi + sleep 0.1 + done + return 1 +} + +load_internal_environment() { + g_script_path="${YUM_CI_INTERNAL_SCRIPT:?}" + g_worker="${YUM_CI_INTERNAL_WORKER:?}" + g_run_dir="${YUM_CI_INTERNAL_RUN_DIR:?}" + g_work_parent="${YUM_CI_INTERNAL_WORK_PARENT:?}" + g_work_root="${YUM_CI_INTERNAL_WORK_ROOT:?}" + g_base_root="${YUM_CI_INTERNAL_BASE_ROOT:?}" + g_repo_dir="${YUM_CI_INTERNAL_REPO_DIR:?}" + g_root_mode="${YUM_CI_INTERNAL_ROOT_MODE:?}" + g_isolation="${YUM_CI_INTERNAL_ISOLATION:?}" + g_stage_timeout="${YUM_CI_INTERNAL_STAGE_TIMEOUT:?}" + g_package_timeout="${YUM_CI_INTERNAL_PACKAGE_TIMEOUT:?}" + g_command_timeout="${YUM_CI_INTERNAL_COMMAND_TIMEOUT:?}" + g_service_timeout="${YUM_CI_INTERNAL_SERVICE_TIMEOUT:?}" + g_service_mode="${YUM_CI_INTERNAL_SERVICE_MODE:?}" + g_chroot_bin="${YUM_CI_INTERNAL_CHROOT_BIN:?}" + g_mount_bin="${YUM_CI_INTERNAL_MOUNT_BIN:?}" + g_umount_bin="${YUM_CI_INTERNAL_UMOUNT_BIN:?}" + g_findmnt_bin="${YUM_CI_INTERNAL_FINDMNT_BIN:?}" + g_unshare_bin="${YUM_CI_INTERNAL_UNSHARE_BIN:?}" +} + +internal_overlay_probe() { + local probe="$g_work_root/overlay-probe" + local rc=0 + mkdir -p "$probe/lower" "$probe/upper" "$probe/work" "$probe/merged" + : >"$probe/lower/probe" + "$g_mount_bin" -t overlay overlay \ + -o "lowerdir=$probe/lower,upperdir=$probe/upper,workdir=$probe/work" \ + "$probe/merged" >/dev/null 2>&1 || rc=1 + [ "$rc" -ne 0 ] || "$g_umount_bin" "$probe/merged" >/dev/null 2>&1 || rc=1 + return "$rc" +} + +if [ "${1:-}" = --internal-supervise-package ]; then + [ "$#" -eq 3 ] || die "invalid internal supervisor invocation" + load_internal_environment + supervise_package "$2" "$3" + exit 0 +fi + +if [ "${1:-}" = --internal-run-package ]; then + [ "$#" -eq 3 ] || die "invalid internal package invocation" + load_internal_environment + run_package_in_chroot "$2" "$3" + exit $? +fi + +if [ "${1:-}" = --internal-probe-overlay ]; then + [ "$#" -eq 1 ] || die "invalid internal overlay probe invocation" + load_internal_environment + internal_overlay_probe + exit $? +fi + +start_package() { + local index="$1" + local package="$2" + env \ + YUM_CI_INTERNAL_SCRIPT="$g_script_path" \ + YUM_CI_INTERNAL_WORKER="$g_worker" \ + YUM_CI_INTERNAL_RUN_DIR="$g_run_dir" \ + YUM_CI_INTERNAL_WORK_PARENT="$g_work_parent" \ + YUM_CI_INTERNAL_WORK_ROOT="$g_work_root" \ + YUM_CI_INTERNAL_BASE_ROOT="$g_base_root" \ + YUM_CI_INTERNAL_REPO_DIR="$g_repo_dir" \ + YUM_CI_INTERNAL_ROOT_MODE="$g_root_mode" \ + YUM_CI_INTERNAL_ISOLATION="$g_isolation" \ + YUM_CI_INTERNAL_STAGE_TIMEOUT="$g_stage_timeout" \ + YUM_CI_INTERNAL_PACKAGE_TIMEOUT="$g_package_timeout" \ + YUM_CI_INTERNAL_COMMAND_TIMEOUT="$g_command_timeout" \ + YUM_CI_INTERNAL_SERVICE_TIMEOUT="$g_service_timeout" \ + YUM_CI_INTERNAL_SERVICE_MODE="$g_service_mode" \ + YUM_CI_INTERNAL_CHROOT_BIN="$g_chroot_bin" \ + YUM_CI_INTERNAL_MOUNT_BIN="$g_mount_bin" \ + YUM_CI_INTERNAL_UMOUNT_BIN="$g_umount_bin" \ + YUM_CI_INTERNAL_FINDMNT_BIN="$g_findmnt_bin" \ + YUM_CI_INTERNAL_UNSHARE_BIN="$g_unshare_bin" \ + setsid /bin/bash "$g_script_path" --internal-supervise-package "$index" "$package" & + g_launch_pid=$! + g_pids+=("$g_launch_pid") + g_session_ids+=("$g_launch_pid") + g_launch_pid="" +} + +active_jobs() { + echo "${#g_pids[@]}" +} + +tracked_process_count() { + local session_id process_stat process_tail process_session count=0 + if [ "${YUM_CI_CHROOT_TEST_MODE:-0}" = 1 ]; then + echo 0 + return 0 + fi + for process_stat in /proc/[0-9]*/stat; do + [ -r "$process_stat" ] || continue + IFS= read -r process_tail <"$process_stat" 2>/dev/null || continue + process_tail="${process_tail##*) }" + read -r _ _ _ process_session _ <<<"$process_tail" + for session_id in "${g_session_ids[@]}"; do + if [ "$process_session" = "$session_id" ]; then + count=$((count + 1)) + break + fi + done + done + echo "$count" +} + +remove_pid() { + local completed="$1" + local slot pid + for slot in "${!g_pids[@]}"; do + pid="${g_pids[$slot]}" + if [ "$pid" = "$completed" ]; then + unset 'g_pids[slot]' + return 0 + fi + done +} + +reap_one() { + local slot pid + while :; do + for slot in "${!g_pids[@]}"; do + pid="${g_pids[$slot]}" + if ! kill -0 "$pid" 2>/dev/null; then + wait "$pid" 2>/dev/null || true + remove_pid "$pid" + return 0 + fi + done + sleep 0.05 + done +} + +terminate_sessions() { + local pid slot running attempt=0 + trap '' INT TERM HUP + [ -z "$g_launch_pid" ] || kill -TERM "$g_launch_pid" >/dev/null 2>&1 || true + for slot in "${!g_pids[@]}"; do + pid="${g_pids[$slot]}" + pkill -TERM -s "$pid" >/dev/null 2>&1 || kill -TERM -- "-$pid" >/dev/null 2>&1 || true + done + while [ "$attempt" -lt 50 ]; do + attempt=$((attempt + 1)) + running=0 + for slot in "${!g_pids[@]}"; do + pid="${g_pids[$slot]}" + if kill -0 "$pid" 2>/dev/null; then running=1; break; fi + done + [ "$running" -eq 1 ] || break + sleep 0.1 + done + for slot in "${!g_pids[@]}"; do + pid="${g_pids[$slot]}" + pkill -KILL -s "$pid" >/dev/null 2>&1 || kill -KILL -- "-$pid" >/dev/null 2>&1 || true + done + for slot in "${!g_pids[@]}"; do + wait "${g_pids[$slot]}" 2>/dev/null || true + done + g_pids=() +} + +on_signal() { + g_signal_code="$1" + exit "$g_signal_code" +} + +on_exit() { + local rc=$? + trap - EXIT + if [ "$(active_jobs)" -gt 0 ] || [ -n "$g_launch_pid" ]; then + terminate_sessions + fi + cleanup_work_root || rc=2 + exit "$rc" +} + +append_category() { + local file="$1" + local package="$2" + local reason="$3" + printf ' - %-35s 原因: %s\n' "$package" "$reason" >>"$file" +} + +print_category() { + local title="$1" + local file="$2" + local count + count="$(wc -l <"$file")" + printf '【%s】(%d)\n' "$title" "$count" + if [ "$count" -eq 0 ]; then + echo " (无)" + else + cat "$file" + fi + echo +} + +aggregate_results() { + local timestamp="$1" + local legacy="$g_output_parent/package-command-result-chroot-${timestamp}-$$.txt" + local detail="$g_output_parent/package-command-detail-chroot-${timestamp}-$$.tsv" + local metrics="$g_output_parent/package-command-metrics-chroot-${timestamp}-$$.tsv" + local rows="$g_run_dir/legacy.rows" + local install_fail="$g_run_dir/install-fail.list" + local remove_fail="$g_run_dir/remove-fail.list" + local command_fail="$g_run_dir/command-fail.list" + local service_fail="$g_run_dir/service-fail.list" + local skipped="$g_run_dir/skipped.list" + local index=0 package relative result fields display overall skip_reason + local requested source binary install command service remove worker_overall + local install_reason command_reason service_reason remove_reason + local install_ms command_ms service_ms remove_ms total_ms command_count unit_count preinstalled + + : >"$rows" + : >"$install_fail" + : >"$remove_fail" + : >"$command_fail" + : >"$service_fail" + : >"$skipped" + g_pass_count=0 + g_fail_count=0 + g_skip_count=0 + printf '# requested\tsource-package\tbinary-package\tinstall\tcommand\tservice\tremove\toverall\tinstall-reason\tcommand-reason\tservice-reason\tremove-reason\tinstall-ms\tcommand-ms\tservice-ms\tremove-ms\ttotal-ms\tcommand-count\tunit-count\tpreinstalled\n' >"$detail" + printf '# index\trequested\toverall\ttotal-ms\tinstall-ms\tcommand-ms\tservice-ms\tremove-ms\n' >"$metrics" + + while IFS= read -r package; do + index=$((index + 1)) + relative="$(package_relative_dir "$index" "$package")" + result="$g_run_dir/packages/$relative/result.tsv" + fields="$(awk -F '\t' 'NR == 1 { print NF }' "$result" 2>/dev/null || true)" + if [ "$fields" != 20 ]; then + write_synthetic_result "$package" "$g_run_dir/packages/$relative" install \ + "missing or malformed worker result" + fi + IFS=$'\t' read -r requested source binary install command service remove worker_overall \ + install_reason command_reason service_reason remove_reason \ + install_ms command_ms service_ms remove_ms total_ms command_count unit_count preinstalled \ + <"$result" + + if ! status_is_valid "$install" || ! status_is_valid "$command" || + ! status_is_valid "$service" || ! status_is_valid "$remove" || + ! status_is_valid "$worker_overall"; then + write_synthetic_result "$package" "$g_run_dir/packages/$relative" install \ + "worker returned an invalid stage status" + IFS=$'\t' read -r requested source binary install command service remove worker_overall \ + install_reason command_reason service_reason remove_reason \ + install_ms command_ms service_ms remove_ms total_ms command_count unit_count preinstalled \ + <"$result" + fi + + if [ "$install" = FAIL ] || [ "$command" = FAIL ] || [ "$service" = FAIL ] || [ "$remove" = FAIL ]; then + overall=FAIL + g_fail_count=$((g_fail_count + 1)) + elif [ "$install" = SKIP ] || [ "$command" = SKIP ] || [ "$service" = SKIP ] || [ "$remove" = SKIP ]; then + overall=SKIP + g_skip_count=$((g_skip_count + 1)) + else + overall=PASS + g_pass_count=$((g_pass_count + 1)) + fi + display="$binary" + [ "$display" != UNKNOWN ] || display="$requested" + + printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$source" "$binary" "$install" "$command" "$service" "$remove" >>"$rows" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$requested" "$source" "$binary" "$install" "$command" "$service" "$remove" "$overall" \ + "$install_reason" "$command_reason" "$service_reason" "$remove_reason" \ + "$install_ms" "$command_ms" "$service_ms" "$remove_ms" "$total_ms" \ + "$command_count" "$unit_count" "$preinstalled" >>"$detail" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$index" "$requested" "$overall" "$total_ms" "$install_ms" "$command_ms" \ + "$service_ms" "$remove_ms" >>"$metrics" + + [ "$install" != FAIL ] || append_category "$install_fail" "$display" "$install_reason" + [ "$remove" != FAIL ] || append_category "$remove_fail" "$display" "$remove_reason" + [ "$command" != FAIL ] || append_category "$command_fail" "$display" "$command_reason" + [ "$service" != FAIL ] || append_category "$service_fail" "$display" "$service_reason" + skip_reason="" + [ "$install" != SKIP ] || skip_reason="install: $install_reason" + [ "$command" != SKIP ] || skip_reason="${skip_reason:+$skip_reason; }command: $command_reason" + [ "$service" != SKIP ] || skip_reason="${skip_reason:+$skip_reason; }service: $service_reason" + [ "$remove" != SKIP ] || skip_reason="${skip_reason:+$skip_reason; }remove: $remove_reason" + [ -z "$skip_reason" ] || append_category "$skipped" "$display" "$skip_reason" + done <"$g_run_dir/input/packages.list" + + { + printf '# source-package\tbinary_package\tinstall\tcommand\tservice\tremove\n' + cat "$rows" + echo + printf '========== 测试失败汇总 (%s) ==========\n' "$(date '+%Y-%m-%d %H:%M:%S')" + printf '环境: chroot | 隔离: %s | rootfs: %s | 并发度: %d | 待测包总数: %d | 通过: %d | 失败: %d | 跳过: %d\n\n' \ + "$g_isolation" "$g_root_mode" "$g_jobs" "$g_total" \ + "$g_pass_count" "$g_fail_count" "$g_skip_count" + print_category "安装失败" "$install_fail" + print_category "卸载失败" "$remove_fail" + print_category "命令测试失败" "$command_fail" + print_category "服务测试失败" "$service_fail" + print_category "跳过项" "$skipped" + echo "=======================================================" + } >"$legacy" + + if [ $((g_pass_count + g_fail_count + g_skip_count)) -ne "$g_total" ]; then + echo "ERROR: reconciliation failed: PASS + FAIL + SKIP != total" >&2 + return 2 + fi + cat "$legacy" + echo "Legacy report: $legacy" + echo "Detailed report: $detail" + echo "Metrics: $metrics" +} + +write_manifest() { + local phase="$1" + local manifest="$g_run_dir/run-manifest.txt" + if [ "$phase" = start ]; then + { + echo "run_id=$g_run_id" + echo "started_at=$(date --iso-8601=seconds)" + echo "host_os=$(clean_field "$(tr '\n' ' ' /dev/null || true)")" + echo "kernel=$(uname -srmo)" + echo "bash=${BASH_VERSION}" + echo "releasever=$g_releasever" + echo "jobs=$g_jobs" + echo "root_mode=$g_root_mode" + echo "isolation=$g_isolation" + echo "service_mode=$g_service_mode" + echo "package_timeout=$g_package_timeout" + echo "stage_timeout=$g_stage_timeout" + echo "command_timeout=$g_command_timeout" + echo "service_timeout=$g_service_timeout" + echo "package_total=$g_total" + if command -v sha256sum >/dev/null 2>&1; then + echo "packages_sha256=$(sha256sum "$g_run_dir/input/packages.list" | awk '{print $1}')" + echo "desktop_sha256=$(sha256sum "$g_run_dir/input/desktop.list" | awk '{print $1}')" + echo "service_sha256=$(sha256sum "$g_run_dir/input/service.list" | awk '{print $1}')" + fi + } >"$manifest" + else + { + echo "finished_at=$(date --iso-8601=seconds)" + echo "execution_ms=$((g_execution_finished_ms - g_execution_started_ms))" + echo "pass=$g_pass_count" + echo "fail=$g_fail_count" + echo "skip=$g_skip_count" + } >>"$manifest" + fi +} + +probe_namespace_isolation() { + "$g_unshare_bin" --mount --pid --fork --kill-child --propagation private true \ + >/dev/null 2>&1 +} + +detect_isolation() { + case "$g_isolation" in + shared) return 0 ;; + namespace) + probe_namespace_isolation || die "PID/mount namespace isolation was requested but is unavailable" + ;; + auto) + if probe_namespace_isolation; then + g_isolation=namespace + else + g_isolation=shared + echo "WARNING: PID/mount namespaces unavailable; using tracked shared mounts" >&2 + fi + ;; + esac +} + +run_overlay_probe() { + local rc + if [ "$g_isolation" = namespace ]; then + env \ + YUM_CI_INTERNAL_SCRIPT="$g_script_path" \ + YUM_CI_INTERNAL_WORKER="$g_worker" \ + YUM_CI_INTERNAL_RUN_DIR="$g_run_dir" \ + YUM_CI_INTERNAL_WORK_PARENT="$g_work_parent" \ + YUM_CI_INTERNAL_WORK_ROOT="$g_work_root" \ + YUM_CI_INTERNAL_BASE_ROOT="$g_base_root" \ + YUM_CI_INTERNAL_REPO_DIR="$g_repo_dir" \ + YUM_CI_INTERNAL_ROOT_MODE=overlay \ + YUM_CI_INTERNAL_ISOLATION="$g_isolation" \ + YUM_CI_INTERNAL_STAGE_TIMEOUT="$g_stage_timeout" \ + YUM_CI_INTERNAL_PACKAGE_TIMEOUT="$g_package_timeout" \ + YUM_CI_INTERNAL_COMMAND_TIMEOUT="$g_command_timeout" \ + YUM_CI_INTERNAL_SERVICE_TIMEOUT="$g_service_timeout" \ + YUM_CI_INTERNAL_SERVICE_MODE="$g_service_mode" \ + YUM_CI_INTERNAL_CHROOT_BIN="$g_chroot_bin" \ + YUM_CI_INTERNAL_MOUNT_BIN="$g_mount_bin" \ + YUM_CI_INTERNAL_UMOUNT_BIN="$g_umount_bin" \ + YUM_CI_INTERNAL_FINDMNT_BIN="$g_findmnt_bin" \ + YUM_CI_INTERNAL_UNSHARE_BIN="$g_unshare_bin" \ + "$g_unshare_bin" --mount --pid --fork --kill-child --propagation private \ + /bin/bash "$g_script_path" --internal-probe-overlay + rc=$? + else + internal_overlay_probe + rc=$? + fi + rm -rf -- "$g_work_root/overlay-probe" + return "$rc" +} + +detect_root_mode() { + if [[ "$g_base_root" == *','* || "$g_base_root" == *':'* || + "$g_work_root" == *','* || "$g_work_root" == *':'* ]]; then + if [ "$g_root_mode" = overlay ]; then + die "overlay paths may not contain ',' or ':'" + fi + g_root_mode=copy + return 0 + fi + case "$g_root_mode" in + copy) return 0 ;; + overlay) + run_overlay_probe || die "overlay mode was requested but the mount probe failed" + ;; + auto) + if run_overlay_probe; then g_root_mode=overlay; else g_root_mode=copy; fi + ;; + esac +} + +while [ "$#" -gt 0 ]; do + case "$1" in + -j | --jobs) [ "$#" -ge 2 ] || die "$1 requires a value"; g_jobs="$2"; shift 2 ;; + --stage-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; g_stage_timeout="$2"; shift 2 ;; + --package-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; g_package_timeout="$2"; shift 2 ;; + --bootstrap-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; g_bootstrap_timeout="$2"; shift 2 ;; + --command-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; g_command_timeout="$2"; shift 2 ;; + --service-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; g_service_timeout="$2"; shift 2 ;; + --service-mode) [ "$#" -ge 2 ] || die "$1 requires a value"; g_service_mode="$2"; shift 2 ;; + --base-rootfs) [ "$#" -ge 2 ] || die "$1 requires a value"; g_base_root_input="$2"; shift 2 ;; + --repo-dir) [ "$#" -ge 2 ] || die "$1 requires a value"; g_repo_dir="$2"; shift 2 ;; + --releasever) [ "$#" -ge 2 ] || die "$1 requires a value"; g_releasever="$2"; shift 2 ;; + --root-mode) [ "$#" -ge 2 ] || die "$1 requires a value"; g_root_mode="$2"; shift 2 ;; + --isolation) [ "$#" -ge 2 ] || die "$1 requires a value"; g_isolation="$2"; shift 2 ;; + --work-dir) [ "$#" -ge 2 ] || die "$1 requires a value"; g_work_parent="$2"; shift 2 ;; + --output-dir) [ "$#" -ge 2 ] || die "$1 requires a value"; g_output_parent="$2"; shift 2 ;; + -h | --help) usage; exit 0 ;; + --) shift; break ;; + -*) die "unknown option: $1" ;; + *) break ;; + esac +done + +[ "$#" -eq 3 ] || { usage >&2; exit 2; } +g_package_file="$1" +g_desktop_file="$2" +g_service_file="$3" + +[ "${BASH_VERSINFO[0]}" -ge 4 ] || die "Bash 4 or newer is required" +for value in "$g_jobs" "$g_stage_timeout" "$g_package_timeout" "$g_bootstrap_timeout" \ + "$g_command_timeout" "$g_service_timeout"; do + is_positive_integer "$value" || die "jobs and timeout values must be positive integers" +done +case "$g_service_mode" in auto | skip | require) ;; *) die "service mode must be auto, skip, or require" ;; esac +case "$g_root_mode" in auto | overlay | copy) ;; *) die "root mode must be auto, overlay, or copy" ;; esac +case "$g_isolation" in auto | namespace | shared) ;; *) die "isolation must be auto, namespace, or shared" ;; esac + +for input in "$g_package_file" "$g_desktop_file" "$g_service_file"; do + [ -f "$input" ] || die "input file not found: $input" +done +[ -f "$g_worker" ] || die "worker not found: $g_worker" +[ -d "$g_repo_dir" ] || die "repo directory not found: $g_repo_dir" +find "$g_repo_dir" -maxdepth 1 -type f -name '*.repo' -print -quit | grep -q . || + die "repo directory contains no .repo file: $g_repo_dir" +[ -z "$g_base_root_input" ] || [ -d "$g_base_root_input" ] || die "base rootfs not found: $g_base_root_input" + +for command in bash timeout setsid pkill realpath awk sed grep sort find cp; do + command -v "$command" >/dev/null 2>&1 || die "required command not found: $command" +done +[ "$(id -u)" -eq 0 ] || [ "${YUM_CI_CHROOT_TEST_MODE:-0}" = 1 ] || die "root privileges are required; run with sudo" +for command in "$g_chroot_bin" "$g_mount_bin" "$g_umount_bin" "$g_findmnt_bin"; do + command -v "$command" >/dev/null 2>&1 || die "required command not found: $command" +done +if [ "$g_isolation" != shared ]; then + command -v "$g_unshare_bin" >/dev/null 2>&1 || [ "$g_isolation" = auto ] || die "unshare is required for namespace isolation" +fi +if [ -z "$g_base_root_input" ]; then + command -v "$g_dnf_bin" >/dev/null 2>&1 || die "dnf is required to prepare a base rootfs" +fi + +g_repo_dir="$(realpath "$g_repo_dir")" +[ "$g_releasever" != auto ] || g_releasever="$(detect_releasever)" +[[ "$g_releasever" =~ ^[A-Za-z0-9._-]+$ ]] || die "releasever contains unsupported characters" +mkdir -p "$g_output_parent" "$g_work_parent" || die "cannot create output/work parent" +g_output_parent="$(realpath "$g_output_parent")" +g_work_parent="$(realpath "$g_work_parent")" +case "$g_work_parent" in *$'\n'* | *$'\r'*) die "work path contains a newline" ;; esac + +g_work_root="$(mktemp -d "$g_work_parent/yum-ci-chroot.XXXXXX")" || die "cannot create work root" +g_run_dir="$g_output_parent/$g_run_id" +case "$g_run_dir/" in "$g_work_root/"*) die "output directory overlaps temporary work root" ;; esac +case "$g_work_root/" in "$g_run_dir/"*) die "temporary work root overlaps output directory" ;; esac +mkdir "$g_work_root/packages" || die "cannot create package work root" +chmod 0700 "$g_work_root/packages" 2>/dev/null || [ "${YUM_CI_CHROOT_TEST_MODE:-0}" = 1 ] || + die "cannot make package work root private" +mkdir "$g_run_dir" || die "cannot create run output directory" +chmod 0755 "$g_run_dir" 2>/dev/null || [ "${YUM_CI_CHROOT_TEST_MODE:-0}" = 1 ] || + die "cannot set run output directory permissions" +mkdir "$g_run_dir/input" "$g_run_dir/packages" || die "cannot create output subdirectories" + +trap on_exit EXIT +trap 'on_signal 130' INT +trap 'on_signal 143' TERM HUP + +normalize_list "$g_package_file" "$g_run_dir/input/packages.list" no +normalize_list "$g_desktop_file" "$g_run_dir/input/desktop.list" yes +normalize_list "$g_service_file" "$g_run_dir/input/service.list" yes +g_total="$(wc -l <"$g_run_dir/input/packages.list")" +[ "$g_total" -gt 0 ] || die "package list contains no testable entries" +while IFS= read -r package; do + [[ "$package" =~ ^[A-Za-z0-9][A-Za-z0-9._+~:@%-]*$ ]] || die "unsafe or unsupported package spec: $package" +done <"$g_run_dir/input/packages.list" + +detect_isolation +prepare_base_root +detect_root_mode +write_manifest start + +echo "Chroot package test: total=$g_total jobs=$g_jobs isolation=$g_isolation root_mode=$g_root_mode" +echo "Per-package logs: $g_run_dir/packages" +g_execution_started_ms="$(now_ms)" +g_index=0 +while IFS= read -r package; do + g_index=$((g_index + 1)) + start_package "$g_index" "$package" + if [ "$(active_jobs)" -ge "$g_jobs" ]; then + reap_one + fi +done <"$g_run_dir/input/packages.list" +while [ "$(active_jobs)" -gt 0 ]; do + reap_one +done +g_execution_finished_ms="$(now_ms)" + +aggregate_results "$(date '+%Y%m%d-%H%M%S')" +g_report_rc=$? +write_manifest finish + +g_tracked_remaining="$(tracked_process_count)" +g_cleanup_ok=1 +if [ "$g_tracked_remaining" -gt 0 ]; then + echo "ERROR: $g_tracked_remaining process(es) remain in tracked worker sessions; work tree retained" >&2 + g_cleanup_done=1 + g_cleanup_ok=0 +elif ! cleanup_work_root; then + g_cleanup_ok=0 +fi +if [ "$g_cleanup_ok" -eq 1 ]; then + { + echo "run_id=$g_run_id" + echo "mounts_remaining=0" + echo "tracked_workers_remaining=$g_tracked_remaining" + echo "work_root_removed=1" + echo "result=PASS" + } >"$g_run_dir/cleanup-audit.txt" +else + { + echo "run_id=$g_run_id" + echo "tracked_workers_remaining=$g_tracked_remaining" + echo "work_root_removed=0" + echo "result=FAIL" + } >"$g_run_dir/cleanup-audit.txt" + g_report_rc=2 +fi + +trap - EXIT +if [ "$g_report_rc" -ne 0 ]; then + exit "$g_report_rc" +fi +[ "$g_fail_count" -eq 0 ] diff --git a/testcase/lib/chroot-package-worker.sh b/testcase/lib/chroot-package-worker.sh new file mode 100755 index 0000000000000000000000000000000000000000..897863a70f942abcd94adcc2e226dbb34217e10e --- /dev/null +++ b/testcase/lib/chroot-package-worker.sh @@ -0,0 +1,511 @@ +#!/bin/bash +# Execute one package test inside an already prepared chroot. + +set -u +set -o pipefail +export LC_ALL=C + +usage() { + echo "Usage: $0 " >&2 +} + +[ "$#" -eq 8 ] || { + usage + exit 2 +} + +g_requested="$1" +g_desktop_list="$2" +g_service_list="$3" +g_output_dir="$4" +g_stage_timeout="$5" +g_command_timeout="$6" +g_service_timeout="$7" +g_service_mode="$8" + +mkdir -p "$g_output_dir" +umask 022 +ulimit -c 0 >/dev/null 2>&1 || true + +g_source_package="UNKNOWN" +g_binary_package="$g_requested" +g_rpm_name="$g_requested" +g_install="FAIL" +g_command="SKIP" +g_service="SKIP" +g_remove="SKIP" +g_install_reason="worker exited before installation completed" +g_command_reason="installation did not complete" +g_service_reason="installation did not complete" +g_remove_reason="installation did not complete" +g_install_ms=0 +g_command_ms=0 +g_service_ms=0 +g_remove_ms=0 +g_total_ms=0 +g_command_count=0 +g_unit_count=0 +g_preinstalled=0 +g_worker_started_ms=0 +g_result_written=0 + +clean_field() { + printf '%s' "$*" | + tr '\t\r\n' ' ' | + sed 's/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//' | + cut -c1-800 +} + +now_ms() { + local value + value="$(date +%s%3N 2>/dev/null || true)" + if [[ "$value" =~ ^[0-9]+$ ]]; then + printf '%s\n' "$value" + else + printf '%s000\n' "$(date +%s)" + fi +} + +elapsed_ms() { + local started="$1" + local finished + finished="$(now_ms)" + if [ "$finished" -ge "$started" ] 2>/dev/null; then + printf '%s\n' "$((finished - started))" + else + echo 0 + fi +} + +reason_from_log() { + local log_file="$1" + local fallback="$2" + local reason="" + + if [ -s "$log_file" ]; then + reason="$(grep -Ei 'error:|failed|failure|conflict|nothing provides|no match|not found|timed out|timeout|missing|segmentation|core dumped' "$log_file" | tail -n 1 || true)" + [ -n "$reason" ] || reason="$(grep -v '^[[:space:]]*$' "$log_file" | tail -n 1 || true)" + fi + [ -n "$reason" ] || reason="$fallback" + clean_field "$reason" +} + +list_contains() { + local list_file="$1" + local name="$2" + [ -s "$list_file" ] && grep -Fqx -- "$name" "$list_file" +} + +# Called by the EXIT result handler. +# shellcheck disable=SC2329 +overall_status() { + if [ "$g_install" = "FAIL" ] || [ "$g_command" = "FAIL" ] || + [ "$g_service" = "FAIL" ] || [ "$g_remove" = "FAIL" ]; then + echo FAIL + elif [ "$g_install" = "SKIP" ] || [ "$g_command" = "SKIP" ] || + [ "$g_service" = "SKIP" ] || [ "$g_remove" = "SKIP" ]; then + echo SKIP + else + echo PASS + fi +} + +# Invoked by the EXIT trap. +# shellcheck disable=SC2329 +write_result() { + local trap_rc=$? + local overall + local result_tmp="$g_output_dir/.result.$$.tmp" + + [ "$g_result_written" -eq 0 ] || return "$trap_rc" + g_result_written=1 + trap - EXIT + g_total_ms="$(elapsed_ms "$g_worker_started_ms")" + overall="$(overall_status)" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$(clean_field "$g_requested")" \ + "$(clean_field "$g_source_package")" \ + "$(clean_field "$g_binary_package")" \ + "$g_install" "$g_command" "$g_service" "$g_remove" "$overall" \ + "$(clean_field "$g_install_reason")" \ + "$(clean_field "$g_command_reason")" \ + "$(clean_field "$g_service_reason")" \ + "$(clean_field "$g_remove_reason")" \ + "$g_install_ms" "$g_command_ms" "$g_service_ms" "$g_remove_ms" \ + "$g_total_ms" "$g_command_count" "$g_unit_count" "$g_preinstalled" \ + >"$result_tmp" + mv -f -- "$result_tmp" "$g_output_dir/result.tsv" + return "$trap_rc" +} + +trap write_result EXIT +g_worker_started_ms="$(now_ms)" + +if command -v dnf >/dev/null 2>&1; then + g_pm=dnf +elif command -v yum >/dev/null 2>&1; then + g_pm=yum +else + g_install_reason="chroot has neither dnf nor yum" + exit 1 +fi + +run_timed() { + timeout --signal=TERM --kill-after=5 "${g_stage_timeout}s" "$@" +} + +enable_module_if_needed() { + local module_stream="" + local module_log="$g_output_dir/module.log" + + case "$g_requested" in + *module+* | *module_*) ;; + *) return 0 ;; + esac + + module_stream="$("$g_pm" module provides -- "$g_requested" 2>"$module_log" | + awk '$1 == "Module" { print $3; exit }' || true)" + if [ -z "$module_stream" ]; then + echo "No module stream discovered; continuing with normal install" >>"$module_log" + return 0 + fi + run_timed "$g_pm" module enable -y -- "$module_stream" >>"$module_log" 2>&1 +} + +install_package() { + local started rc install_action + local install_log="$g_output_dir/install.log" + + started="$(now_ms)" + if rpm -q -- "$g_requested" >/dev/null 2>&1; then + g_preinstalled=1 + install_action=reinstall + else + install_action=install + fi + + enable_module_if_needed || true + run_timed "$g_pm" "$install_action" -y -- "$g_requested" >"$install_log" 2>&1 + rc=$? + g_install_ms="$(elapsed_ms "$started")" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + g_install_reason="${install_action} timed out after ${g_stage_timeout}s" + return 1 + fi + if [ "$rc" -ne 0 ]; then + g_install_reason="$(reason_from_log "$install_log" "$g_pm $install_action returned $rc")" + return 1 + fi + if grep -Eiq 'scriptlet failed|error:[[:space:]].*No such file or directory' "$install_log"; then + g_install_reason="$(reason_from_log "$install_log" "$install_action reported a scriptlet or file error")" + return 1 + fi + + g_rpm_name="$(rpm -q --qf '%{NAME}\n' -- "$g_requested" 2>>"$install_log" | head -n 1)" + if [ -z "$g_rpm_name" ]; then + g_install_reason="rpm query failed after package-manager success" + return 1 + fi + g_source_package="$(rpm -q --qf '%{SOURCERPM}\n' -- "$g_rpm_name" 2>/dev/null | head -n 1)" + g_binary_package="$(rpm -q --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}\n' -- "$g_rpm_name" 2>/dev/null | head -n 1)" + [ -n "$g_source_package" ] || g_source_package=UNKNOWN + [ -n "$g_binary_package" ] || g_binary_package="$g_requested" + g_install=PASS + if [ "$g_preinstalled" -eq 1 ]; then + g_install_reason="package was present in the base rootfs and was reinstalled successfully" + else + g_install_reason="-" + fi + return 0 +} + +probe_command() { + local command_path="$1" + local command_id="$2" + local command_log="$g_output_dir/commands/${command_id}.log" + local trace_log="$g_output_dir/commands/${command_id}.strace" + + : >"$command_log" + # A tested command must never inherit the file descriptor used to enumerate + # command paths. Utilities such as funzip read stdin even for an invalid + # invocation and would otherwise consume the next path from the loop. + if timeout --signal=KILL "$g_command_timeout" "$command_path" --help \ + "$command_log" 2>&1; then + return 0 + fi + if timeout --signal=KILL "$g_command_timeout" "$command_path" --version \ + >"$command_log" 2>&1; then + return 0 + fi + if grep -Eiq 'error while loading shared libraries|segmentation fault|core dumped' "$command_log"; then + return 1 + fi + if ! realpath "$command_path" >/dev/null 2>&1; then + echo "broken executable path: $command_path" >>"$command_log" + return 1 + fi + + timeout --signal=KILL "$g_command_timeout" \ + strace -f -qq -e trace=execve -o "$trace_log" \ + "$command_path" --help >"$command_log" 2>&1 || true + if grep -Eq 'SIG(SEGV|ABRT|BUS)|core dumped' "$trace_log" "$command_log" 2>/dev/null; then + return 1 + fi + if grep -Eq 'execve\(.*\)[[:space:]]+=[[:space:]]+0' "$trace_log" 2>/dev/null; then + return 0 + fi + grep -Eiq 'invalid|argument|unknown|option|usage' "$command_log" +} + +test_commands() { + local started command_path command_id=0 failures=0 first_failure="" + local prereq_log="$g_output_dir/command-prerequisites.log" + local files_list="$g_output_dir/command-files.list" + local commands_list="$g_output_dir/commands.list" + + started="$(now_ms)" + if list_contains "$g_desktop_list" "$g_rpm_name" || list_contains "$g_desktop_list" "$g_requested"; then + g_command=SKIP + g_command_reason="desktop whitelist: command execution is not applicable" + g_command_ms="$(elapsed_ms "$started")" + return 0 + fi + + if ! command -v file >/dev/null 2>&1 || ! command -v strace >/dev/null 2>&1; then + run_timed "$g_pm" install -y file strace >"$prereq_log" 2>&1 || true + fi + if ! command -v file >/dev/null 2>&1 || ! command -v strace >/dev/null 2>&1; then + g_command=FAIL + g_command_reason="$(reason_from_log "$prereq_log" "file/strace prerequisites are unavailable")" + g_command_ms="$(elapsed_ms "$started")" + return 1 + fi + + rpm -ql -- "$g_rpm_name" 2>"$g_output_dir/rpm-ql-command.log" | + awk '/^\/usr\/(s?bin)\// && $0 !~ /\.so([.]|$)/ && $0 !~ /^\/usr\/bin\/nde-/ { print }' \ + >"$files_list" + : >"$commands_list" + while IFS= read -r command_path; do + [ -x "$command_path" ] || continue + if file -L -- "$command_path" 2>/dev/null | grep -qw ELF; then + echo "$command_path" >>"$commands_list" + fi + done <"$files_list" + + if [ ! -s "$commands_list" ]; then + g_command=PASS + g_command_reason="no ELF executable under /usr/bin or /usr/sbin" + g_command_ms="$(elapsed_ms "$started")" + return 0 + fi + + mkdir -p "$g_output_dir/commands" + while IFS= read -r command_path <&3; do + command_id=$((command_id + 1)) + if ! probe_command "$command_path" "$command_id"; then + failures=$((failures + 1)) + if [ -z "$first_failure" ]; then + first_failure="$command_path: $(reason_from_log "$g_output_dir/commands/${command_id}.log" "execve probe failed")" + fi + fi + done 3<"$commands_list" + g_command_count="$command_id" + g_command_ms="$(elapsed_ms "$started")" + + if [ "$failures" -eq 0 ]; then + g_command=PASS + g_command_reason="all ${command_id} ELF executable(s) passed" + return 0 + fi + g_command=FAIL + g_command_reason="${failures}/${command_id} executable(s) failed; $first_failure" + return 1 +} + +systemd_is_reachable() { + local state + command -v systemctl >/dev/null 2>&1 || return 1 + state="$(timeout --signal=TERM --kill-after=2 "$g_service_timeout" \ + systemctl is-system-running 2>/dev/null || true)" + case "$state" in + running | degraded | starting) return 0 ;; + *) return 1 ;; + esac +} + +record_skipped_units() { + local units_file="$1" + local reason="$2" + local unit + while IFS= read -r unit; do + printf '===== %s =====\nstart=SKIP\nstatus/is-active=SKIP\nstop=SKIP\nreason=%s\n\n' \ + "$unit" "$reason" >>"$g_output_dir/service.log" + done <"$units_file" +} + +test_services() { + local started unit rc failures=0 skipped=0 passed=0 first_failure="" + local units_list="$g_output_dir/systemd-units.list" + local concrete_list="$g_output_dir/systemd-concrete-units.list" + local service_log="$g_output_dir/service.log" + + started="$(now_ms)" + : >"$service_log" + rpm -ql -- "$g_rpm_name" 2>"$g_output_dir/rpm-ql-service.log" | + grep -E '^/(usr/lib|lib|etc)/systemd/system/.*\.(service|socket|target)$' | + awk -F/ '{ print $NF }' | sort -u >"$units_list" || true + g_unit_count="$(wc -l <"$units_list")" + + if [ "$g_unit_count" -eq 0 ]; then + g_service=PASS + g_service_reason="package provides no systemd service/socket/target unit" + g_service_ms="$(elapsed_ms "$started")" + return 0 + fi + if list_contains "$g_service_list" "$g_rpm_name" || list_contains "$g_service_list" "$g_requested"; then + g_service=SKIP + g_service_reason="special-service list: configuration is required before startup" + record_skipped_units "$units_list" "special configuration required" + g_service_ms="$(elapsed_ms "$started")" + return 0 + fi + if [ "$g_service_mode" = skip ]; then + g_service=SKIP + g_service_reason="service mode is skip; start/status/is-active/stop were not executed" + record_skipped_units "$units_list" "service mode is skip" + g_service_ms="$(elapsed_ms "$started")" + return 0 + fi + if ! systemd_is_reachable; then + if [ "$g_service_mode" = require ]; then + g_service=FAIL + g_service_reason="systemd/PID1 is unavailable but service mode require was requested" + record_skipped_units "$units_list" "systemd/PID1 unavailable" + g_service_ms="$(elapsed_ms "$started")" + return 1 + fi + g_service=SKIP + g_service_reason="plain chroot has no reachable systemd/PID1; start/status/is-active/stop were not executed" + record_skipped_units "$units_list" "systemd/PID1 unavailable in chroot" + g_service_ms="$(elapsed_ms "$started")" + return 0 + fi + + : >"$concrete_list" + while IFS= read -r unit; do + case "$unit" in + *@.service | *@.socket | *@.target) + skipped=$((skipped + 1)) + printf '===== %s =====\nstart=SKIP\nstatus/is-active=SKIP\nstop=SKIP\nreason=template unit requires an instance and configuration\n\n' \ + "$unit" >>"$service_log" + ;; + *) echo "$unit" >>"$concrete_list" ;; + esac + done <"$units_list" + + while IFS= read -r unit; do + [ -n "$unit" ] || continue + printf '===== %s =====\n' "$unit" >>"$service_log" + timeout --signal=TERM --kill-after=3 "$g_service_timeout" \ + systemctl --no-pager start "$unit" >>"$service_log" 2>&1 + rc=$? + if [ "$rc" -ne 0 ]; then + echo "start=FAIL (rc=$rc)" >>"$service_log" + echo "status/is-active=SKIP" >>"$service_log" + failures=$((failures + 1)) + [ -n "$first_failure" ] || first_failure="$unit start failed (rc=$rc)" + elif ! timeout --signal=TERM --kill-after=3 "$g_service_timeout" \ + systemctl --no-pager is-active --quiet "$unit" >>"$service_log" 2>&1; then + echo "start=PASS" >>"$service_log" + echo "status/is-active=FAIL" >>"$service_log" + failures=$((failures + 1)) + [ -n "$first_failure" ] || first_failure="$unit was not active after start" + else + echo "start=PASS" >>"$service_log" + echo "status/is-active=PASS" >>"$service_log" + passed=$((passed + 1)) + fi + + timeout --signal=TERM --kill-after=3 "$g_service_timeout" \ + systemctl --no-pager stop "$unit" >>"$service_log" 2>&1 + rc=$? + if [ "$rc" -ne 0 ]; then + echo "stop=FAIL (rc=$rc)" >>"$service_log" + failures=$((failures + 1)) + [ -n "$first_failure" ] || first_failure="$unit stop failed (rc=$rc)" + else + echo "stop=PASS" >>"$service_log" + fi + echo >>"$service_log" + done <"$concrete_list" + g_service_ms="$(elapsed_ms "$started")" + + if [ "$failures" -gt 0 ]; then + g_service=FAIL + g_service_reason="${failures} service lifecycle operation(s) failed; $first_failure" + return 1 + fi + if [ "$skipped" -gt 0 ]; then + g_service=SKIP + g_service_reason="${passed} concrete unit(s) passed; ${skipped} template unit(s) skipped" + return 0 + fi + g_service=PASS + g_service_reason="all ${passed} unit(s) passed start/status/is-active/stop" + return 0 +} + +remove_package() { + local started rc + local remove_log="$g_output_dir/remove.log" + + started="$(now_ms)" + if [ "$g_preinstalled" -eq 1 ]; then + g_remove=SKIP + g_remove_reason="package belongs to the base rootfs; removal is skipped to preserve the test harness" + g_remove_ms="$(elapsed_ms "$started")" + return 0 + fi + + run_timed "$g_pm" remove -y --setopt=clean_requirements_on_remove=False -- "$g_rpm_name" >"$remove_log" 2>&1 + rc=$? + g_remove_ms="$(elapsed_ms "$started")" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + g_remove=FAIL + g_remove_reason="removal timed out after ${g_stage_timeout}s" + return 1 + fi + if [ "$rc" -ne 0 ]; then + g_remove=FAIL + g_remove_reason="$(reason_from_log "$remove_log" "$g_pm remove returned $rc")" + return 1 + fi + if grep -Eiq 'scriptlet failed' "$remove_log"; then + g_remove=FAIL + g_remove_reason="$(reason_from_log "$remove_log" "removal reported a scriptlet error")" + return 1 + fi + if rpm -q -- "$g_rpm_name" >/dev/null 2>&1; then + g_remove=FAIL + g_remove_reason="package remains installed after package-manager success" + return 1 + fi + g_remove=PASS + g_remove_reason="-" + return 0 +} + +if install_package; then + test_commands || true + test_services || true + remove_package || true +else + g_command=SKIP + g_command_reason="installation failed" + g_service=SKIP + g_service_reason="installation failed" + g_remove=SKIP + g_remove_reason="installation failed" +fi + +exit 0 diff --git a/testcase/tests/test-command-execute-chroot.sh b/testcase/tests/test-command-execute-chroot.sh new file mode 100755 index 0000000000000000000000000000000000000000..1a6ecd37799bc2f9185d7192959886e50d4b9d56 --- /dev/null +++ b/testcase/tests/test-command-execute-chroot.sh @@ -0,0 +1,575 @@ +#!/bin/bash +# Scheduler, report, timeout and cleanup regression tests. No root is required. + +set -u +set -o pipefail + +g_test_dir="$(mktemp -d /tmp/yum-ci-chroot-test.XXXXXX)" +g_repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +g_script="$g_repo_root/testcase/command-execute-chroot.sh" +g_fakebin="$g_test_dir/bin" +g_worker_fakebin="$g_test_dir/worker-bin" + +cleanup() { + local rc=$? + trap - EXIT + if [ "${KEEP_TEST_TMP:-0}" = 1 ]; then + echo "Test artifacts retained: $g_test_dir" >&2 + else + rm -rf -- "$g_test_dir" + fi + exit "$rc" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local file="$1" + local expected="$2" + grep -F -- "$expected" "$file" >/dev/null 2>&1 || { + echo "Expected in $file: $expected" >&2 + sed -n '1,220p' "$file" >&2 || true + fail "missing expected text" + } +} + +assert_not_contains() { + local file="$1" + local unexpected="$2" + if grep -F -- "$unexpected" "$file" >/dev/null 2>&1; then + fail "unexpected text in $file: $unexpected" + fi +} + +mkdir -p "$g_fakebin" "$g_worker_fakebin" + +cat >"$g_fakebin/setsid" <<'FAKE' +#!/bin/bash +exec "$@" +FAKE + +cat >"$g_fakebin/pgrep" <<'FAKE' +#!/bin/bash +exit 1 +FAKE + +cat >"$g_fakebin/pkill" <<'FAKE' +#!/bin/bash +if [ "${1:-}" = "-TERM" ] || [ "${1:-}" = "-KILL" ]; then + signal="$1" + shift +else + signal=-TERM +fi +if [ "${1:-}" = "-s" ] && [ "$#" -ge 2 ]; then + kill "$signal" "$2" >/dev/null 2>&1 || true +fi +exit 0 +FAKE + +cat >"$g_fakebin/fake-findmnt" <<'FAKE' +#!/bin/bash +exit 0 +FAKE + +cat >"$g_fakebin/fake-mount" <<'FAKE' +#!/bin/bash +printf '%s\n' "$*" >>"${FAKE_MOUNT_LOG:?}" +if [ "${FAKE_MOUNT_FAIL:-0}" = 1 ]; then + exit 1 +fi +exit 0 +FAKE + +cat >"$g_fakebin/fake-umount" <<'FAKE' +#!/bin/bash +printf '%s\n' "$*" >>"${FAKE_UMOUNT_LOG:?}" +exit 0 +FAKE + +cat >"$g_fakebin/fake-unshare" <<'FAKE' +#!/bin/bash +while [ "$#" -gt 0 ]; do + case "$1" in + --mount | --pid | --fork | --kill-child) shift ;; + --propagation) shift 2 ;; + *) exec "$@" ;; + esac +done +exit 0 +FAKE + +cat >"$g_fakebin/fake-chroot" <<'FAKE' +#!/bin/bash +set -u + +root="$1" +shift +[ "${1:-}" = /bin/bash ] && shift +[ "${1:-}" = /run/yum-ci-input/worker.sh ] && shift +package="$1" +output="${YUM_CI_TEST_PACKAGE_DIR:?}" +state="${FAKE_STATE_DIR:?}" +lock="$state/lock" +cleaned=0 + +with_lock() { + while ! mkdir "$lock" 2>/dev/null; do sleep 0.01; done +} + +unlock() { + rmdir "$lock" +} + +finish_active() { + local current + [ "$cleaned" -eq 0 ] || return 0 + cleaned=1 + with_lock + current="$(cat "$state/current")" + echo "$((current - 1))" >"$state/current" + unlock +} + +on_term() { + finish_active + exit 143 +} +trap finish_active EXIT +trap on_term TERM INT HUP + +with_lock +current="$(cat "$state/current")" +maximum="$(cat "$state/maximum")" +current=$((current + 1)) +echo "$current" >"$state/current" +[ "$current" -le "$maximum" ] || echo "$current" >"$state/maximum" +printf 'start:%s:%s\n' "$package" "$$" >>"$state/events" +unlock + +case "$package" in + timeout-pkg) + sleep "${FAKE_TIMEOUT_DELAY:-4}" & + child_pid=$! + echo "$child_pid" >"$state/orphan-pid" + wait "$child_pid" + ;; + *) sleep "${FAKE_CHROOT_DELAY:-0.15}" ;; +esac + +source_package="${package}.src.rpm" +binary_package="${package}-1.0-1.x86_64" +install=PASS +command=PASS +service=PASS +remove=PASS +overall=PASS +install_reason=- +command_reason=- +service_reason=- +remove_reason=- +preinstalled=0 + +case "$package" in + desktop) + command=SKIP + overall=SKIP + command_reason='desktop whitelist: command execution is not applicable' + ;; + special-service) + service=SKIP + overall=SKIP + service_reason='special-service list: configuration is required before startup' + ;; + install-fail) + install=FAIL + command=SKIP + service=SKIP + remove=SKIP + overall=FAIL + install_reason='nothing provides libmissing.so.1' + command_reason='installation failed' + service_reason='installation failed' + remove_reason='installation failed' + ;; + command-fail) + command=FAIL + overall=FAIL + command_reason='/usr/bin/badcmd: segmentation fault' + ;; + service-fail) + service=FAIL + overall=FAIL + service_reason='demo.service was not active after start' + ;; + remove-fail) + remove=FAIL + overall=FAIL + remove_reason='package remains installed after package-manager success' + ;; + preinstalled) + remove=SKIP + overall=SKIP + remove_reason='package belongs to the base rootfs; removal is skipped to preserve the test harness' + preinstalled=1 + ;; +esac + +tmp="$output/.result.fake.$$.tmp" +printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t10\t20\t30\t40\t100\t2\t1\t%s\n' \ + "$package" "$source_package" "$binary_package" \ + "$install" "$command" "$service" "$remove" "$overall" \ + "$install_reason" "$command_reason" "$service_reason" "$remove_reason" \ + "$preinstalled" >"$tmp" +mv -f "$tmp" "$output/result.tsv" +printf 'finish:%s:%s\n' "$package" "$$" >>"$state/events" +exit 0 +FAKE + +chmod 0755 "$g_fakebin/"* + +cat >"$g_worker_fakebin/dnf" <<'FAKE' +#!/bin/bash +set -u +action="${1:-}" +state="${WORKER_STATE:?}" +scenario="${WORKER_SCENARIO:?}" +case "$action" in + module) exit 0 ;; + install) + if [ "$scenario" = install-fail ]; then + echo 'Error: nothing provides libmissing.so.1' >&2 + exit 1 + fi + : >"$state/installed" + ;; + reinstall) + [ -f "$state/installed" ] || exit 1 + ;; + remove) + rm -f "$state/installed" + ;; +esac +exit 0 +FAKE + +cat >"$g_worker_fakebin/rpm" <<'FAKE' +#!/bin/bash +set -u +state="${WORKER_STATE:?}" +scenario="${WORKER_SCENARIO:?}" +case "${1:-}" in + -ql) + case "$scenario" in + special-service | require-systemd) echo '/usr/lib/systemd/system/demo.service' ;; + stdin-consumer) printf '/usr/bin/cat\n/usr/bin/true\n' ;; + *) echo '/usr/share/doc/demo/README' ;; + esac + exit 0 + ;; + -q) + if [ "${2:-}" = --qf ]; then + [ -f "$state/installed" ] || exit 1 + case "$3" in + *SOURCERPM*) echo 'demo-1.0-1.src.rpm' ;; + *NAME*VERSION*) echo 'demo-1.0-1.x86_64' ;; + *NAME*) echo demo ;; + *) exit 1 ;; + esac + exit 0 + fi + [ -f "$state/installed" ] + ;; + *) exit 1 ;; +esac +FAKE + +cat >"$g_worker_fakebin/file" <<'FAKE' +#!/bin/bash +if [ "${WORKER_SCENARIO:-}" = stdin-consumer ]; then + echo 'ELF 64-bit executable' +else + echo 'data' +fi +FAKE + +cat >"$g_worker_fakebin/timeout" <<'FAKE' +#!/bin/bash +set -u +while [ "$#" -gt 0 ] && [[ "$1" == --* ]]; do + shift +done +[ "$#" -gt 0 ] || exit 125 +shift +[ "$#" -gt 0 ] || exit 125 +# Reproduce a tested utility that reads the command-enumeration stream, but do +# not interfere with the worker's package-manager timeout calls. With the +# production stdin isolation in place, this read sees /dev/null instead. +if [ "${WORKER_SCENARIO:-}" = stdin-consumer ] && [[ "$1" == /usr/bin/* ]]; then + IFS= read -r -n 6 _discard || true +fi +exec "$@" +FAKE + +cat >"$g_worker_fakebin/strace" <<'FAKE' +#!/bin/bash +exit 0 +FAKE + +cat >"$g_worker_fakebin/systemctl" <<'FAKE' +#!/bin/bash +if [ "${1:-}" = is-system-running ]; then + echo offline + exit 1 +fi +exit 1 +FAKE + +chmod 0755 "$g_worker_fakebin/"* + +prepare_case() { + local case_dir="$1" + mkdir -p "$case_dir/base/bin" "$case_dir/base/usr/bin" \ + "$case_dir/base/etc/yum.repos.d" "$case_dir/repos" \ + "$case_dir/output" "$case_dir/work" "$case_dir/state" + : >"$case_dir/base/bin/bash" + : >"$case_dir/base/usr/bin/rpm" + : >"$case_dir/base/usr/bin/dnf" + chmod 0755 "$case_dir/base/bin/bash" "$case_dir/base/usr/bin/rpm" "$case_dir/base/usr/bin/dnf" + echo '[test]' >"$case_dir/repos/test.repo" + echo 0 >"$case_dir/state/current" + echo 0 >"$case_dir/state/maximum" + : >"$case_dir/state/events" + : >"$case_dir/mount.log" + : >"$case_dir/umount.log" + : >"$case_dir/empty.list" +} + +run_case() { + local case_dir="$1" + shift + PATH="$g_fakebin:$PATH" \ + YUM_CI_CHROOT_TEST_MODE=1 \ + YUM_CI_CHROOT_BIN="$g_fakebin/fake-chroot" \ + YUM_CI_MOUNT_BIN="$g_fakebin/fake-mount" \ + YUM_CI_UMOUNT_BIN="$g_fakebin/fake-umount" \ + YUM_CI_FINDMNT_BIN="$g_fakebin/fake-findmnt" \ + YUM_CI_UNSHARE_BIN="$g_fakebin/fake-unshare" \ + FAKE_STATE_DIR="$case_dir/state" \ + FAKE_MOUNT_LOG="$case_dir/mount.log" \ + FAKE_UMOUNT_LOG="$case_dir/umount.log" \ + bash "$g_script" --base-rootfs "$case_dir/base" \ + --repo-dir "$case_dir/repos" --root-mode copy --isolation namespace \ + --work-dir "$case_dir/work" "$@" +} + +test_cli_validation() { + bash -n "$g_script" || fail "controller syntax check failed" + bash -n "$g_repo_root/testcase/lib/chroot-package-worker.sh" || fail "worker syntax check failed" + bash "$g_script" --help >/dev/null || fail "--help failed" + if bash "$g_script" --jobs 0 a b c >/dev/null 2>&1; then + fail "--jobs 0 was accepted" + fi + if bash "$g_script" --service-mode invalid a b c >/dev/null 2>&1; then + fail "invalid service mode was accepted" + fi +} + +test_benchmark_destination_guard() { + local case_dir="$g_test_dir/benchmark-guard" + local benchmark="$g_repo_root/testcase/benchmark-command-execute-chroot.sh" + mkdir -p "$case_dir/base" "$case_dir/repos" "$case_dir/output/j1-r1" + : >"$case_dir/packages.list" + : >"$case_dir/desktop.list" + : >"$case_dir/service.list" + echo 'preserve-existing-table' >"$case_dir/output/benchmark.tsv" + echo 'old-run' >"$case_dir/output/j1-r1/marker" + + if JOBS_LIST=1 bash "$benchmark" "$case_dir/base" "$case_dir/repos" \ + "$case_dir/packages.list" "$case_dir/desktop.list" "$case_dir/service.list" \ + "$case_dir/output" >"$case_dir/run.log" 2>&1; then + fail "benchmark accepted a non-empty run directory" + fi + assert_contains "$case_dir/run.log" 'refusing to reuse non-empty benchmark run directory' + assert_contains "$case_dir/output/benchmark.tsv" 'preserve-existing-table' +} + +run_worker_case() { + local name="$1" + local scenario="$2" + local desktop_entry="$3" + local service_entry="$4" + local service_mode="$5" + local case_dir="$g_test_dir/worker-$name" + + mkdir -p "$case_dir/state" "$case_dir/output" + : >"$case_dir/desktop.list" + : >"$case_dir/service.list" + [ -z "$desktop_entry" ] || echo "$desktop_entry" >"$case_dir/desktop.list" + [ -z "$service_entry" ] || echo "$service_entry" >"$case_dir/service.list" + [ "$scenario" != preinstalled ] || : >"$case_dir/state/installed" + + PATH="$g_worker_fakebin:$PATH" \ + WORKER_STATE="$case_dir/state" WORKER_SCENARIO="$scenario" \ + bash "$g_repo_root/testcase/lib/chroot-package-worker.sh" demo \ + "$case_dir/desktop.list" "$case_dir/service.list" "$case_dir/output" \ + 5 1 1 "$service_mode" + echo "$case_dir/output/result.tsv" +} + +test_worker_matrix() { + local result + + result="$(run_worker_case pass pass '' '' auto)" + assert_contains "$result" $'demo\tdemo-1.0-1.src.rpm\tdemo-1.0-1.x86_64\tPASS\tPASS\tPASS\tPASS\tPASS' + [ "$(awk -F '\t' '{ print NF }' "$result")" -eq 20 ] || fail "worker result schema is not 20 fields" + + result="$(run_worker_case desktop pass demo '' auto)" + assert_contains "$result" $'\tPASS\tSKIP\tPASS\tPASS\tSKIP\t' + assert_contains "$result" 'desktop whitelist' + + result="$(run_worker_case special special-service '' demo auto)" + assert_contains "$result" $'\tPASS\tPASS\tSKIP\tPASS\tSKIP\t' + assert_contains "$result" 'special-service list' + + result="$(run_worker_case require require-systemd '' '' require)" + assert_contains "$result" $'\tPASS\tPASS\tFAIL\tPASS\tFAIL\t' + assert_contains "$result" 'systemd/PID1 is unavailable' + + result="$(run_worker_case preinstalled preinstalled '' '' auto)" + assert_contains "$result" $'\tPASS\tPASS\tPASS\tSKIP\tSKIP\t' + assert_contains "$result" 'base rootfs' + + result="$(run_worker_case install-fail install-fail '' '' auto)" + assert_contains "$result" $'\tFAIL\tSKIP\tSKIP\tSKIP\tFAIL\t' + assert_contains "$result" 'nothing provides libmissing.so.1' + + result="$(run_worker_case stdin-consumer stdin-consumer '' '' auto)" + assert_contains "$result" $'\tPASS\tPASS\tPASS\tPASS\tPASS\t' + [ "$(awk -F '\t' '{ print $18 }' "$result")" -eq 2 ] || \ + fail "stdin-consuming command corrupted command enumeration" +} + +test_mixed_matrix() { + local case_dir="$g_test_dir/mixed" + local report detail metrics audit maximum order + prepare_case "$case_dir" + cat >"$case_dir/packages.list" <<'EOF' +# comments and blank lines are ignored +pass +desktop +special-service +install-fail +command-fail +service-fail +remove-fail +preinstalled +EOF + echo desktop >"$case_dir/desktop.list" + echo special-service >"$case_dir/service.list" + + if run_case "$case_dir" -j 3 --output-dir "$case_dir/output" \ + "$case_dir/packages.list" "$case_dir/desktop.list" "$case_dir/service.list" \ + >"$case_dir/run.log" 2>&1; then + fail "mixed matrix with intentional failures returned success" + fi + + report="$(find "$case_dir/output" -maxdepth 1 -name 'package-command-result-chroot-*.txt' -print -quit)" + detail="$(find "$case_dir/output" -maxdepth 1 -name 'package-command-detail-chroot-*.tsv' -print -quit)" + metrics="$(find "$case_dir/output" -maxdepth 1 -name 'package-command-metrics-chroot-*.tsv' -print -quit)" + audit="$(find "$case_dir/output" -path '*/cleanup-audit.txt' -print -quit)" + [ -s "$report" ] || fail "legacy report missing" + [ -s "$detail" ] || fail "detailed report missing" + [ -s "$metrics" ] || fail "metrics report missing" + [ -s "$audit" ] || fail "cleanup audit missing" + + assert_contains "$report" $'# source-package\tbinary_package\tinstall\tcommand\tservice\tremove' + assert_contains "$report" '并发度: 3 | 待测包总数: 8 | 通过: 1 | 失败: 4 | 跳过: 3' + assert_contains "$report" '【安装失败】(1)' + assert_contains "$report" '【卸载失败】(1)' + assert_contains "$report" '【命令测试失败】(1)' + assert_contains "$report" '【服务测试失败】(1)' + assert_contains "$report" 'nothing provides libmissing.so.1' + assert_contains "$report" 'desktop whitelist' + assert_contains "$detail" $'preinstalled\tpreinstalled.src.rpm\tpreinstalled-1.0-1.x86_64\tPASS\tPASS\tPASS\tSKIP\tSKIP' + assert_contains "$audit" 'mounts_remaining=0' + assert_contains "$audit" 'work_root_removed=1' + maximum="$(cat "$case_dir/state/maximum")" + [ "$maximum" -gt 1 ] || fail "workers never overlapped" + [ "$maximum" -le 3 ] || fail "scheduler exceeded -j 3 (observed $maximum)" + [ "$(cat "$case_dir/state/current")" -eq 0 ] || fail "fake workers remain active" + [ -z "$(find "$case_dir/work" -mindepth 1 -print -quit)" ] || fail "work roots remain" + [ "$(grep -c -- '--rbind /dev ' "$case_dir/mount.log")" -eq 8 ] || fail "/dev was not mounted once per chroot" + [ "$(grep -c -- '-t proc proc ' "$case_dir/mount.log")" -eq 8 ] || fail "private /proc was not mounted once per chroot" + + order="$(awk -F '\t' 'NR > 1 { print $1 }' "$metrics" | paste -sd, -)" + [ "$order" = '1,2,3,4,5,6,7,8' ] || fail "report order is not deterministic: $order" +} + +test_timeout_synthesis() { + local case_dir="$g_test_dir/timeout" + local report started elapsed orphan_pid + prepare_case "$case_dir" + echo timeout-pkg >"$case_dir/packages.list" + + started="$(date +%s)" + if FAKE_TIMEOUT_DELAY=30 run_case "$case_dir" -j 1 --package-timeout 4 \ + --output-dir "$case_dir/output" "$case_dir/packages.list" \ + "$case_dir/empty.list" "$case_dir/empty.list" >"$case_dir/run.log" 2>&1; then + fail "timed-out package returned success" + fi + elapsed=$(( $(date +%s) - started )) + # The controller allows a five-second TERM -> KILL cleanup grace period. + [ "$elapsed" -lt 12 ] || fail "package timeout plus cleanup grace did not bound runtime (${elapsed}s)" + report="$(find "$case_dir/output" -maxdepth 1 -name 'package-command-result-chroot-*.txt' -print -quit)" + assert_contains "$report" 'package lifecycle timed out after 4s' + assert_contains "$report" '待测包总数: 1 | 通过: 0 | 失败: 1 | 跳过: 0' + [ -s "$case_dir/state/orphan-pid" ] || fail "fake descendant did not start before package timeout" + orphan_pid="$(cat "$case_dir/state/orphan-pid")" + if kill -0 "$orphan_pid" 2>/dev/null; then + fail "package timeout left descendant process $orphan_pid running" + fi + [ -z "$(find "$case_dir/work" -mindepth 1 -print -quit)" ] || fail "timeout left a work root" +} + +test_cleanup_failure_preserves_result() { + local case_dir="$g_test_dir/cleanup-preserve" + local detail + prepare_case "$case_dir" + echo command-fail >"$case_dir/packages.list" + + if YUM_CI_TEST_SESSION_CLEANUP_FAIL=1 run_case "$case_dir" -j 1 \ + --output-dir "$case_dir/output" "$case_dir/packages.list" \ + "$case_dir/empty.list" "$case_dir/empty.list" >"$case_dir/run.log" 2>&1; then + fail "forced cleanup failure returned success" + fi + detail="$(find "$case_dir/output" -maxdepth 1 -name 'package-command-detail-chroot-*.tsv' -print -quit)" + assert_contains "$detail" $'command-fail\tcommand-fail.src.rpm\tcommand-fail-1.0-1.x86_64\tPASS\tFAIL\tPASS\tFAIL\tFAIL' + assert_contains "$detail" '/usr/bin/badcmd: segmentation fault' + assert_contains "$detail" 'worker descendants remained after TERM/KILL escalation' +} + +test_unsafe_package_rejected() { + local case_dir="$g_test_dir/unsafe" + prepare_case "$case_dir" + echo '../escape' >"$case_dir/packages.list" + if run_case "$case_dir" --output-dir "$case_dir/output" "$case_dir/packages.list" \ + "$case_dir/empty.list" "$case_dir/empty.list" >"$case_dir/run.log" 2>&1; then + fail "unsafe package spec was accepted" + fi + assert_contains "$case_dir/run.log" 'unsafe or unsupported package spec' +} + +test_cli_validation +test_benchmark_destination_guard +test_worker_matrix +test_mixed_matrix +test_timeout_synthesis +test_cleanup_failure_preserves_result +test_unsafe_package_rejected + +echo "PASS: command-execute-chroot scheduler/report tests"