From 8e3aea9ff9b40e8a2f905563aaeb6c68bc1b416c Mon Sep 17 00:00:00 2001 From: silasyyyang Date: Thu, 30 Jul 2026 11:56:52 +0800 Subject: [PATCH 1/2] feat: add concurrent Docker package testing (IJV5J9) --- .gitattributes | 5 + .gitignore | 5 + README.md | 16 +- testcase/Makefile | 9 + testcase/README-command-execute-docker.md | 173 +++++ testcase/VALIDATION-command-execute-docker.md | 60 ++ testcase/benchmark-command-execute-docker.sh | 59 ++ testcase/command-execute-docker.sh | 721 ++++++++++++++++++ testcase/docker/Dockerfile | 28 + testcase/lib/docker-package-driver.sh | 188 +++++ testcase/lib/docker-package-worker.sh | 483 ++++++++++++ testcase/lib/docker-ts-common-runner.sh | 35 + testcase/tests/fake-docker-package-driver.sh | 87 +++ .../tests/test-command-execute-docker-e2e.sh | 144 ++++ testcase/tests/test-command-execute-docker.sh | 176 +++++ testcase/tests/test-docker-package-driver.sh | 73 ++ 16 files changed, 2256 insertions(+), 6 deletions(-) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 testcase/README-command-execute-docker.md create mode 100644 testcase/VALIDATION-command-execute-docker.md create mode 100755 testcase/benchmark-command-execute-docker.sh create mode 100755 testcase/command-execute-docker.sh create mode 100644 testcase/docker/Dockerfile create mode 100755 testcase/lib/docker-package-driver.sh create mode 100755 testcase/lib/docker-package-worker.sh create mode 100755 testcase/lib/docker-ts-common-runner.sh create mode 100755 testcase/tests/fake-docker-package-driver.sh create mode 100755 testcase/tests/test-command-execute-docker-e2e.sh create mode 100755 testcase/tests/test-command-execute-docker.sh create mode 100755 testcase/tests/test-docker-package-driver.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..50688ac --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +Dockerfile text eol=lf +Makefile text eol=lf +*.md text eol=lf +*.tsv text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c541aaa --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# Docker package-test runtime artifacts +package-command-result-docker-*.txt +package-command-detail-docker-*.tsv +yum-ci-docker-*/ +docker-benchmark-*/ diff --git a/README.md b/README.md index 8597cde..08756b5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ -# TencentOS发行版检查 -检查项目包括 -- KABI -- 软件包安装 -- 内核模块加载 -- systemd服务启动 \ No newline at end of file +# TencentOS发行版检查 +检查项目包括 +- KABI +- 软件包安装 +- 内核模块加载 +- systemd服务启动 + +## Docker 并发软件包测试 + +使用说明见 [testcase/README-command-execute-docker.md](testcase/README-command-execute-docker.md)。 diff --git a/testcase/Makefile b/testcase/Makefile index 1b4db5b..6839275 100644 --- a/testcase/Makefile +++ b/testcase/Makefile @@ -19,3 +19,12 @@ clean: rm -rfv $(ALL_OBJ) $(ALL_BIN) cleanall: clean + +.PHONY: docker-test docker-e2e + +docker-test: + bash tests/test-command-execute-docker.sh + bash tests/test-docker-package-driver.sh + +docker-e2e: + bash tests/test-command-execute-docker-e2e.sh diff --git a/testcase/README-command-execute-docker.md b/testcase/README-command-execute-docker.md new file mode 100644 index 0000000..3dab5a8 --- /dev/null +++ b/testcase/README-command-execute-docker.md @@ -0,0 +1,173 @@ +# Docker 并发软件包测试 + +`command-execute-docker.sh` 是独立新增用例，不修改原有 +`command-execute-whitelist.sh` 的 QEMU/串行路径。它为每个待测包创建独立 +Docker 容器，并发完成： + +`dnf install → ELF 命令验证 → systemd unit 验证 → dnf remove` + +安装与卸载通过镜像内适配器直接调用 `tst_lib/ts_common.sh` 的 +`test_single_rpm_install`、`test_single_rpm_remove` 判定。命令阶段保留公共 +逻辑中的缺库、致命信号和 `execve` 检查，同时增加逐命令超时，且不修改宿主 +全局 `kernel.core_pattern`。 + +## 环境要求 + +- Linux 宿主机，推荐 OpenCloudOS 9。 +- Docker 20.10.10 或更高版本；OpenCloudOS 文档说明旧版本可能因 clone + 系统调用导致容器内 `dnf install` 异常。 +- Docker 守护进程可用，执行用户有 Docker 权限。 +- 默认基础镜像为官方 `opencloudos/opencloudos9-init:latest`。 + +官方镜像说明： + +- +- + +## 基本用法 + +```bash +cd testcase + +./command-execute-docker.sh \ + -j 8 \ + --output-dir ./docker-results \ + pkg-to-test.list \ + desktop-white.pkg \ + service-white.pkg +``` + +三个位置参数均为必填文件： + +1. 待测软件包列表； +2. 桌面白名单，命中后命令阶段记为 `SKIP`； +3. 需特殊配置的服务包列表，命中后服务阶段记为 `SKIP`。 + +列表每行取第一个非注释字段，支持空行、`#` 注释和 CRLF。包 token +必须以字母或数字开头，非法输入会在创建容器前被拒绝。 + +## 常用参数 + +```text +-j, --jobs N 最大并发容器数 +--package-timeout SEC 单包完整生命周期超时 +--stage-timeout SEC 安装、卸载阶段超时 +--command-timeout SEC 每条命令探测超时 +--service-timeout SEC 每个 unit 的单次操作超时 +--startup-timeout SEC systemd 启动等待上限 +--cpus NUMBER 每容器 CPU 限制 +--memory LIMIT 每容器内存限制，例如 768m、2g +--pids-limit N 每容器进程数限制 +--service-mode MODE auto、systemd 或 skip +--base-image IMAGE 基础镜像 +--test-image IMAGE 派生测试镜像标签 +--repo-dir DIR 只读挂载指定 repo 配置 +--output-dir DIR 报告和逐包日志目录 +--keep-image 完成后保留派生测试镜像（默认删除） +--validate-only 只校验参数和清单，不访问 Docker +``` + +所有数值、枚举和路径参数都会先校验。CPU 必须大于 0；内存和 PID +限制不能为 0。 + +## 服务测试模式 + +默认 `--service-mode auto`： + +1. 只启动一次探测容器； +2. 确认 systemd 作为 PID 1 达到 `running` 或 `degraded`； +3. 探测成功后，所有包使用真实 systemd 容器； +4. 探测失败时，含 unit 的包服务阶段统一记为 `SKIP`，并写明环境原因。 + +`--service-mode systemd` 要求探测必须成功，否则整轮以基础设施错误退出。 +`--service-mode skip` 明确禁用服务启动测试。 + +systemd 模式使用官方 init 镜像和 `--privileged --cgroupns=private`，不会把 +宿主 `/sys/fs/cgroup` 读写挂载进容器。每个非模板 unit 都独立记录： + +- `systemctl start`； +- `systemctl is-active`； +- `systemctl stop`。 + +模板 unit（如 `foo@.service`）和 `RefuseManualStart=yes` 或 +`RefuseManualStop=yes` 的 unit 明确 +记为 `SKIP`，不会误报为通过或失败。 + +> 安全提示：RPM scriptlet、包内命令和服务会被实际执行。systemd 模式又需 +> 特权容器，因此应在一次性、隔离的 CI 节点上运行，不要在多租户生产宿主机 +> 上测试不可信仓库的软件包。 + +## Repo 与版本一致性 + +默认使用 OpenCloudOS 9 官方镜像内置 repo。若 CI 有快照源或内部源，使用： + +```bash +./command-execute-docker.sh \ + --repo-dir /etc/yum.repos.d \ + packages.list desktop.list services.list +``` + +指定目录会只读挂载到容器 `/etc/yum.repos.d`。调用方应确保 repo 与基础 +镜像的主版本和架构一致。运行清单会记录容器 OS、镜像 ID 和 repo 来源。 + +## 结果与退出码 + +输出包括： + +- `package-command-result-docker-*.txt`：兼容原结果格式的六列总表，并附分类汇总； +- `package-command-detail-docker-*.tsv`：逐阶段原因、耗时、命令数、unit 数； +- `yum-ci-docker-*/run-manifest.txt`：环境、镜像、并发和资源限制； +- `yum-ci-docker-*/packages/<序号>-<包名>/`：逐包安装、命令、服务、卸载日志。 + +每个包只计入一种整体状态： + +- 任一阶段 `FAIL`：整体失败； +- 无失败但任一阶段 `SKIP`：整体跳过； +- 四阶段均 `PASS`：整体通过。 + +总报告保证 `通过 + 失败 + 跳过 = 待测总数`。阶段失败会分别进入安装、 +命令、服务、卸载清单；同一包多个阶段失败时会出现在多个清单中。即使包 +整体失败，其中发生的阶段跳过也会保留在“跳过项”。 + +退出码： + +- `0`：没有失败包； +- `1`：至少一个包测试失败； +- `2`：参数、Docker、镜像或结果格式等基础设施错误。 + +## 隔离和清理 + +- 容器名包含 run ID 和包序号，两轮并行测试同一包不会重名。 +- 每轮容器带唯一 Label，信号和异常退出时只清理本轮资源。 +- 命令验证容器仅额外增加 `SYS_PTRACE` capability，供 `strace` 观察 + `execve`；systemd 模式按上文说明使用特权容器。 +- 每包独立 bind mount 日志目录，不创建 Docker volume。 +- 容器使用 `--rm`，并在退出 trap 中再次按精确名称和 Label 清理。 +- 不执行 `docker system prune`，不会删除其他任务资源。 +- 派生镜像标签包含基础镜像、Dockerfile 摘要和 run ID，默认在本轮完成后删除； + 调试或复用时可显式传入 `--keep-image`。 + +## 自动测试 + +宿主调度与报告回归测试不需要 Docker： + +```bash +bash testcase/tests/test-command-execute-docker.sh +``` + +真实 OpenCloudOS 9 集成验证需 Docker： + +```bash +bash testcase/tests/test-command-execute-docker-e2e.sh +``` + +并发基准脚本会先在计时区间外统一构建一次镜像，各档位复用同一镜像，结束后 +自动删除： + +```bash +bash testcase/benchmark-command-execute-docker.sh \ + packages.list desktop.list services.list +``` + +OpenCloudOS 9.6 实机验证和并发基准见 +[VALIDATION-command-execute-docker.md](VALIDATION-command-execute-docker.md)。 diff --git a/testcase/VALIDATION-command-execute-docker.md b/testcase/VALIDATION-command-execute-docker.md new file mode 100644 index 0000000..79cd282 --- /dev/null +++ b/testcase/VALIDATION-command-execute-docker.md @@ -0,0 +1,60 @@ +# IJV5J9 实机验证记录 + +验证日期：2026-07-30。 + +## 环境 + +| 项目 | 值 | +| --- | --- | +| 宿主系统 | OpenCloudOS 9.6，kernel 6.6.119-49.23.oc9.x86_64 | +| 容器系统 | OpenCloudOS 9.6，`opencloudos/opencloudos9-init:latest` | +| Docker | 28.0.1，overlay2，systemd cgroup v2 | +| 主机资源 | 8 vCPU，16 GiB | +| 单容器限制 | 1 CPU，1 GiB，512 PIDs | + +## 静态检查和宿主回归 + +- 所有新增 shell 脚本通过 `bash -n`。 +- 所有新增 shell 脚本通过 ShellCheck `--severity=warning`。 +- 调度、超时、分类、对账、基础设施错误码及并发隔离测试：24/24 通过。 +- Driver 原子结果发布、bind mount、repo 临时副本清理和 `SYS_PTRACE` 参数测试通过。 +- `git diff --check` 通过。 + +## 真实 Docker E2E + +执行： + +```bash +bash testcase/tests/test-command-execute-docker-e2e.sh +``` + +最终结果：14/14 通过。 + +- `tree`、`nano`、`cronie`、`openssh-server`、`strace` 均完成安装、命令、服务和卸载阶段。 +- 安装与卸载实际调用镜像内 `tst_lib/ts_common.sh` 公共判定函数。 +- 镜像内预装的 `strace` 也执行真实卸载，四阶段均为 `PASS`。 +- systemd 探测达到 `running`；`cronie` 分别记录 `start=0 is-active=0 stop=0`。 +- 不存在的软件包返回退出码 1，并准确归入“安装失败”。 +- 两轮同时测试同一软件包均返回 0，生成两个独立报告。 +- 测试完成后，带本用例 run Label 的容器数为 0，Docker volume 数为 0； + 派生镜像和带本题 Label 的 dangling image 数也均为 0。 + +## 并发基准 + +8 个真实软件包：`tree`、`nano`、`cronie`、`openssh-server`、`jq`、 +`wget`、`rsync`、`unzip`。镜像构建在计时区间外只执行一次，四轮退出码均为 0。 + +| 并发度 | 耗时 | 相对 j1 加速 | +| ---: | ---: | ---: | +| 1 | 69.078 s | 1.00x | +| 2 | 35.600 s | 1.94x | +| 4 | 20.550 s | 3.36x | +| 8 | 14.862 s | 4.65x | + +复现： + +```bash +JOBS_CSV=1,2,4,8 \ + bash testcase/benchmark-command-execute-docker.sh \ + packages.list desktop.list services.list +``` diff --git a/testcase/benchmark-command-execute-docker.sh b/testcase/benchmark-command-execute-docker.sh new file mode 100755 index 0000000..3094b45 --- /dev/null +++ b/testcase/benchmark-command-execute-docker.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Compare wall-clock time for several Docker concurrency levels. + +set -uo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +MAIN="$SCRIPT_DIR/command-execute-docker.sh" +JOBS_CSV=${JOBS_CSV:-1,2,4,8} +SERVICE_MODE=${BENCHMARK_SERVICE_MODE:-skip} +IMAGE=${BENCHMARK_IMAGE:-yum-ci-pkg-test:ijv5j9-benchmark-$$} +BASE_IMAGE=${BENCHMARK_BASE_IMAGE:-opencloudos/opencloudos9-init:latest} +OUTPUT_ROOT=${BENCHMARK_OUTPUT_DIR:-./docker-benchmark-$(date '+%Y%m%d-%H%M%S')} +IMAGE_BUILT=0 + +cleanup() { + [ "$IMAGE_BUILT" -eq 0 ] || docker image rm "$IMAGE" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +usage() { + echo "Usage: $0 PKG_LIST DESKTOP_LIST SERVICE_LIST [extra command options ...]" >&2 +} + +[ "$#" -ge 3 ] || { usage; exit 2; } +packages=$1 +desktop=$2 +services=$3 +shift 3 + +mkdir -p "$OUTPUT_ROOT" +summary="$OUTPUT_ROOT/benchmark.tsv" +printf 'jobs\telapsed_ms\texit_code\n' >"$summary" + +command -v docker >/dev/null 2>&1 || { echo 'Docker is required' >&2; exit 2; } +docker info >/dev/null 2>&1 || { echo 'Docker daemon is unavailable' >&2; exit 2; } +docker build --force-rm --build-arg "BASE_IMAGE=$BASE_IMAGE" --tag "$IMAGE" \ + --file "$SCRIPT_DIR/docker/Dockerfile" "$SCRIPT_DIR/.." \ + >"$OUTPUT_ROOT/image-build.log" 2>&1 || { + cat "$OUTPUT_ROOT/image-build.log" >&2 + exit 2 + } +IMAGE_BUILT=1 + +IFS=',' read -r -a levels <<<"$JOBS_CSV" +for jobs in "${levels[@]}"; do + [[ $jobs =~ ^[1-9][0-9]*$ ]] || { echo "invalid JOBS_CSV entry: $jobs" >&2; exit 2; } + run_output="$OUTPUT_ROOT/j$jobs" + mkdir -p "$run_output" + start=$(date +%s%3N) + bash "$MAIN" -j "$jobs" --service-mode "$SERVICE_MODE" \ + --output-dir "$run_output" --skip-build --test-image "$IMAGE" \ + --base-image "$BASE_IMAGE" "$@" \ + "$packages" "$desktop" "$services" >"$run_output/benchmark.log" 2>&1 + rc=$? + elapsed=$(( $(date +%s%3N) - start )) + printf '%s\t%s\t%s\n' "$jobs" "$elapsed" "$rc" | tee -a "$summary" +done + +echo "benchmark summary: $summary" diff --git a/testcase/command-execute-docker.sh b/testcase/command-execute-docker.sh new file mode 100755 index 0000000..f28b750 --- /dev/null +++ b/testcase/command-execute-docker.sh @@ -0,0 +1,721 @@ +#!/bin/bash +############################################################################### +# @用例ID: 20260729-000001-000000001 +# @用例名称: command-execute-docker +# @用例级别: 3 +# @用例标签: docker,parallel,package +# @用例类型: 功能测试 +# @自动化: 1 +# @用例描述: 在独立 Docker 容器中并发验证软件包安装、命令、服务和卸载 +############################################################################### + +set -uo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +TOPDIR=$(cd -- "$SCRIPT_DIR/.." && pwd -P) +DRIVER_DEFAULT="$SCRIPT_DIR/lib/docker-package-driver.sh" +DOCKERFILE_DEFAULT="$SCRIPT_DIR/docker/Dockerfile" + +JOBS="" +PACKAGE_TIMEOUT=1200 +STAGE_TIMEOUT=300 +COMMAND_TIMEOUT=5 +SERVICE_TIMEOUT=60 +STARTUP_TIMEOUT=60 +CPU_LIMIT=1 +MEMORY_LIMIT=1g +PIDS_LIMIT=512 +SERVICE_MODE=auto +EFFECTIVE_SERVICE_MODE=skip +SERVICE_MODE_REASON="not probed" +BASE_IMAGE="opencloudos/opencloudos9-init:latest" +TEST_IMAGE="yum-ci-pkg-test:ijv5j9" +TEST_IMAGE_EXPLICIT=0 +PULL_POLICY=missing +BUILD_IMAGE=1 +REMOVE_IMAGE=1 +VALIDATE_ONLY=0 +KEEP_WORK=0 +OUTPUT_DIR=. +WORK_PARENT="${TMPDIR:-/tmp}" +REPO_DIR="" +DRIVER="${YUM_CI_DOCKER_DRIVER:-$DRIVER_DEFAULT}" +DOCKERFILE="${YUM_CI_DOCKERFILE:-$DOCKERFILE_DEFAULT}" + +PKG_INPUT="" +DESKTOP_INPUT="" +SERVICE_INPUT="" +WORK_ROOT="" +RUN_DIR="" +RUN_ID="" +NORMALIZED_PACKAGES="" +NORMALIZED_DESKTOP="" +NORMALIZED_SERVICES="" +IMAGE_BUILT=0 +INTERRUPTED=0 +INFRA_FAILURE=0 +declare -a CHILD_PIDS=() + +usage() { + cat <<'EOF' +Usage: + command-execute-docker.sh [options] PKG_LIST DESKTOP_LIST SERVICE_LIST + +Required inputs: + PKG_LIST packages to test (first non-comment field per line) + DESKTOP_LIST packages whose command stage must be skipped + SERVICE_LIST packages requiring special service configuration + +Options: + -j, --jobs N concurrent containers (default: min(CPU, 8)) + --package-timeout SEC full lifecycle timeout per package (default: 1200) + --stage-timeout SEC install/remove timeout (default: 300) + --command-timeout SEC timeout for each command probe (default: 5) + --service-timeout SEC timeout for each service action (default: 60) + --startup-timeout SEC systemd container startup timeout (default: 60) + --cpus NUMBER CPUs available to each container (default: 1) + --memory LIMIT memory available to each container (default: 1g) + --pids-limit N process limit for each container (default: 512) + --service-mode MODE auto, systemd, or skip (default: auto) + --base-image IMAGE OpenCloudOS base image + --test-image IMAGE derived test image tag + --pull POLICY always, missing, or never (default: missing) + --skip-build use --test-image without building it + --keep-image retain the derived image after the run + --remove-image remove the derived image (default) + --repo-dir DIR mount repository definitions read-only + --work-dir DIR temporary work parent (default: /tmp) + --output-dir DIR reports and package logs (default: current dir) + --keep-work keep temporary scheduler files + --validate-only validate inputs and print the plan without Docker + -h, --help show this help + +Exit codes: 0 = PASS/SKIP only, 1 = package failure, 2 = usage/infrastructure. +EOF +} + +log() { + printf '[%s] %s\n' "$(date '+%F %T')" "$*" >&2 +} + +die() { + log "ERROR: $*" + exit 2 +} + +is_positive_integer() { + [[ ${1:-} =~ ^[1-9][0-9]*$ ]] +} + +is_positive_decimal() { + [[ ${1:-} =~ ^([0-9]+([.][0-9]+)?|[.][0-9]+)$ ]] && + awk -v value="$1" 'BEGIN { exit !(value > 0) }' +} + +is_memory_limit() { + [[ ${1:-} =~ ^[1-9][0-9]*([.][0-9]+)?[bBkKmMgGtTpP]?$ ]] +} + +sanitize_field() { + printf '%s' "${1:-}" | tr '\t\r\n' ' ' | + sed 's/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//' +} + +default_jobs() { + local cpu + cpu=$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '1') + is_positive_integer "$cpu" || cpu=1 + [ "$cpu" -le 8 ] || cpu=8 + printf '%s\n' "$cpu" +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "required command is missing: $1" +} + +normalize_list() { + local input=$1 output=$2 kind=$3 line value line_no=0 + : >"$output" + while IFS= read -r line || [ -n "$line" ]; do + line_no=$((line_no + 1)) + line=${line%$'\r'} + line=${line%%#*} + line=${line#"${line%%[![:space:]]*}"} + value=${line%%[[:space:]]*} + [ -n "$value" ] || continue + if [[ ! $value =~ ^[A-Za-z0-9][A-Za-z0-9_.+~:@%-]*$ ]]; then + die "$kind contains an unsafe package token at line $line_no: $value" + fi + printf '%s\n' "$value" >>"$output" + done <"$input" +} + +safe_remove_tree() { + local target=${1:-} + [ -n "$WORK_ROOT" ] || return 1 + case "$target" in + "$WORK_ROOT" | "$WORK_ROOT"/*) rm -rf -- "$target" ;; + *) log "refusing to remove path outside work root: $target"; return 1 ;; + esac +} + +cleanup_run_containers() { + local container + [ -n "$RUN_ID" ] || return 0 + command -v docker >/dev/null 2>&1 || return 0 + while IFS= read -r container || [ -n "$container" ]; do + [ -n "$container" ] || continue + docker rm -f "$container" >/dev/null 2>&1 || true + done < <(docker ps -aq --filter "label=io.opencloudos.yum-ci.run=$RUN_ID" 2>/dev/null || true) +} + +stop_children() { + local pid + for pid in "${CHILD_PIDS[@]:-}"; do + command -v pkill >/dev/null 2>&1 && pkill -TERM -P "$pid" 2>/dev/null || true + kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true + done + sleep 1 + for pid in "${CHILD_PIDS[@]:-}"; do + command -v pkill >/dev/null 2>&1 && pkill -KILL -P "$pid" 2>/dev/null || true + kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true + done +} + +on_signal() { + INTERRUPTED=1 + log "signal received; stopping package workers" + stop_children + cleanup_run_containers + exit 130 +} + +cleanup() { + local rc=$? + trap - EXIT INT TERM HUP + if [ "$INTERRUPTED" -eq 1 ]; then + stop_children + rc=2 + fi + cleanup_run_containers + if [ "$REMOVE_IMAGE" -eq 1 ] && [ "$IMAGE_BUILT" -eq 1 ]; then + docker image rm "$TEST_IMAGE" >/dev/null 2>&1 || true + fi + if [ -n "$WORK_ROOT" ] && [ -d "$WORK_ROOT" ] && [ "$KEEP_WORK" -eq 0 ]; then + safe_remove_tree "$WORK_ROOT" || rc=2 + elif [ -n "$WORK_ROOT" ] && [ -d "$WORK_ROOT" ]; then + log "temporary work retained at $WORK_ROOT" + fi + exit "$rc" +} + +pull_base_image() { + case "$PULL_POLICY" in + always) + docker pull "$BASE_IMAGE" || return 1 + ;; + missing) + docker image inspect "$BASE_IMAGE" >/dev/null 2>&1 || + docker pull "$BASE_IMAGE" || return 1 + ;; + never) + docker image inspect "$BASE_IMAGE" >/dev/null 2>&1 || return 1 + ;; + esac +} + +prepare_test_image() { + if [ "$BUILD_IMAGE" -eq 0 ]; then + docker image inspect "$TEST_IMAGE" >/dev/null 2>&1 || { + log "test image not found: $TEST_IMAGE" + return 1 + } + return 0 + fi + [ -f "$DOCKERFILE" ] || { log "Dockerfile not found: $DOCKERFILE"; return 1; } + pull_base_image || { log "cannot prepare base image: $BASE_IMAGE"; return 1; } + local -a build_args=( + build --force-rm + --build-arg "BASE_IMAGE=$BASE_IMAGE" + --label "io.opencloudos.yum-ci.issue=IJV5J9" + --tag "$TEST_IMAGE" + --file "$DOCKERFILE" + "$TOPDIR" + ) + [ "$PULL_POLICY" != always ] || build_args=(build --pull "${build_args[@]:1}") + log "building test image $TEST_IMAGE from $BASE_IMAGE" + docker "${build_args[@]}" || return 1 + IMAGE_BUILT=1 +} + +resolve_test_image_tag() { + local dockerfile_digest image_digest + + [ "$TEST_IMAGE_EXPLICIT" -eq 0 ] || return 0 + [ -f "$DOCKERFILE" ] || die "Dockerfile not found: $DOCKERFILE" + + if command -v sha256sum >/dev/null 2>&1; then + dockerfile_digest=$(sha256sum "$DOCKERFILE" | awk '{print $1}') + image_digest=$(printf '%s\n%s\n' "$BASE_IMAGE" "$dockerfile_digest" | + sha256sum | awk '{print substr($1, 1, 12)}') + else + dockerfile_digest=$(cksum "$DOCKERFILE" | awk '{print $1 ":" $2}') + image_digest=$(printf '%s\n%s\n' "$BASE_IMAGE" "$dockerfile_digest" | + cksum | awk '{printf "%08x%04x", $1, $2 % 65536}') + fi + TEST_IMAGE="yum-ci-pkg-test:ijv5j9-$image_digest-$RUN_ID" +} + +probe_systemd() { + local name="yumci-probe-${RUN_ID}" state="" attempt + if ! docker run -d --rm --privileged --cgroupns=private \ + --name "$name" \ + --label "io.opencloudos.yum-ci.run=$RUN_ID" \ + --entrypoint /sbin/init \ + "$TEST_IMAGE" >/dev/null 2>&1; then + SERVICE_MODE_REASON="privileged init container failed to start" + return 1 + fi + for ((attempt = 0; attempt < STARTUP_TIMEOUT; attempt++)); do + state=$(docker exec "$name" systemctl is-system-running 2>/dev/null || true) + case "$state" in + running | degraded) + docker rm -f "$name" >/dev/null 2>&1 || true + SERVICE_MODE_REASON="systemd probe reached $state" + return 0 + ;; + initializing | starting | "") sleep 1 ;; + *) break ;; + esac + done + docker rm -f "$name" >/dev/null 2>&1 || true + SERVICE_MODE_REASON="systemd probe did not become ready (state=${state:-unknown})" + return 1 +} + +resolve_service_mode() { + case "$SERVICE_MODE" in + skip) + EFFECTIVE_SERVICE_MODE=skip + SERVICE_MODE_REASON="service mode skip requested" + ;; + systemd) + probe_systemd || return 1 + EFFECTIVE_SERVICE_MODE=systemd + ;; + auto) + if probe_systemd; then + EFFECTIVE_SERVICE_MODE=systemd + else + EFFECTIVE_SERVICE_MODE=skip + log "systemd unavailable; service units will be marked SKIP: $SERVICE_MODE_REASON" + fi + ;; + esac +} + +write_manifest() { + local manifest="$RUN_DIR/run-manifest.txt" image_id os_info docker_info + image_id=$(docker image inspect --format '{{.Id}}' "$TEST_IMAGE" 2>/dev/null || printf unknown) + os_info=$(docker run --rm --entrypoint /bin/sh "$TEST_IMAGE" -c \ + '. /etc/os-release; printf "%s %s" "${NAME:-unknown}" "${VERSION_ID:-unknown}"' 2>/dev/null || printf unknown) + docker_info=$(docker version --format '{{.Server.Version}}' 2>/dev/null || printf unknown) + { + printf 'issue=IJV5J9\n' + printf 'started=%s\n' "$(date -Is)" + printf 'run_id=%s\n' "$RUN_ID" + printf 'host=%s\n' "$(uname -a)" + printf 'docker=%s\n' "$docker_info" + printf 'container_os=%s\n' "$os_info" + printf 'base_image=%s\n' "$BASE_IMAGE" + printf 'test_image=%s\n' "$TEST_IMAGE" + printf 'image_id=%s\n' "$image_id" + printf 'jobs=%s\n' "$JOBS" + printf 'service_mode_requested=%s\n' "$SERVICE_MODE" + printf 'service_mode_used=%s\n' "$EFFECTIVE_SERVICE_MODE" + printf 'service_mode_reason=%s\n' "$(sanitize_field "$SERVICE_MODE_REASON")" + printf 'package_timeout=%s\n' "$PACKAGE_TIMEOUT" + printf 'stage_timeout=%s\n' "$STAGE_TIMEOUT" + printf 'command_timeout=%s\n' "$COMMAND_TIMEOUT" + printf 'service_timeout=%s\n' "$SERVICE_TIMEOUT" + printf 'startup_timeout=%s\n' "$STARTUP_TIMEOUT" + printf 'cpus=%s\n' "$CPU_LIMIT" + printf 'memory=%s\n' "$MEMORY_LIMIT" + printf 'pids_limit=%s\n' "$PIDS_LIMIT" + printf 'repo_dir=%s\n' "${REPO_DIR:-image-default}" + printf 'package_count=%s\n' "$(wc -l <"$NORMALIZED_PACKAGES")" + command -v sha256sum >/dev/null 2>&1 && + printf 'package_list_sha256=%s\n' "$(sha256sum "$NORMALIZED_PACKAGES" | awk '{print $1}')" + } >"$manifest" +} + +synthesize_result() { + local index=$1 request=$2 reason=$3 result_file=$4 tmp + reason=$(sanitize_field "$reason") + tmp="$result_file.tmp.$$" + printf '%s\t%s\tunknown\t%s\tFAIL\t%s\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\t0\t0\t0\tnone\t1\n' \ + "$index" "$request" "$request" "$reason" >"$tmp" + mv -f "$tmp" "$result_file" +} + +publish_driver_checkpoint() { + local package_dir=$1 result_file=$2 checkpoint tmp + checkpoint="$package_dir/output/result.tsv" + [ -s "$checkpoint" ] || return 0 + tmp="$result_file.tmp.$$" + cp "$checkpoint" "$tmp" + mv -f "$tmp" "$result_file" +} + +finalize_timeout_result() { + local index=$1 request=$2 result_file=$3 tmp + local r_index r_request source binary install install_reason command command_reason + local service service_reason remove remove_reason duration command_count unit_count current complete + if [ ! -s "$result_file" ]; then + synthesize_result "$index" "$request" \ + "container lifecycle timed out after ${PACKAGE_TIMEOUT}s during install" "$result_file" + return 0 + fi + IFS=$'\t' read -r r_index r_request source binary install install_reason command command_reason \ + service service_reason remove remove_reason duration command_count unit_count current complete <"$result_file" + [ "$current" != complete ] || return 0 + case "$current" in + command) + command=FAIL + command_reason="container lifecycle timed out after ${PACKAGE_TIMEOUT}s during command test" + ;; + service) + service=FAIL + service_reason="container lifecycle timed out after ${PACKAGE_TIMEOUT}s during service test" + ;; + remove) + remove=FAIL + remove_reason="container lifecycle timed out after ${PACKAGE_TIMEOUT}s during remove" + ;; + *) + install=FAIL + install_reason="container lifecycle timed out after ${PACKAGE_TIMEOUT}s during install" + ;; + esac + complete=1 + tmp="$result_file.tmp.$$" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$r_index" "$r_request" "$source" "$binary" "$install" "$(sanitize_field "$install_reason")" \ + "$command" "$(sanitize_field "$command_reason")" "$service" "$(sanitize_field "$service_reason")" \ + "$remove" "$(sanitize_field "$remove_reason")" "$duration" "$command_count" "$unit_count" \ + "$current" "$complete" >"$tmp" + mv -f "$tmp" "$result_file" +} + +run_one() { + local index=$1 request=$2 package_dir result_file rc=0 safe_request timed_out=0 + safe_request=$(printf '%s' "$request" | tr -c 'A-Za-z0-9_.-' '_') + safe_request=${safe_request:0:80} + package_dir="$RUN_DIR/packages/$(printf '%05d' "$index")-$safe_request" + result_file="$RUN_DIR/results/$(printf '%05d' "$index").tsv" + mkdir -p "$package_dir" + + LC_ALL=C setsid timeout --verbose -k 20 "$PACKAGE_TIMEOUT" \ + bash "$DRIVER" "$index" "$request" "$package_dir" "$result_file" \ + "$NORMALIZED_DESKTOP" "$NORMALIZED_SERVICES" \ + 2>"$package_dir/lifecycle.log" || rc=$? + + [ -s "$result_file" ] || publish_driver_checkpoint "$package_dir" "$result_file" + if [ "$rc" -eq 124 ] || grep -q 'sending signal' "$package_dir/lifecycle.log" 2>/dev/null; then + timed_out=1 + fi + if [ "$timed_out" -eq 1 ]; then + finalize_timeout_result "$index" "$request" "$result_file" + elif [ ! -s "$result_file" ]; then + synthesize_result "$index" "$request" \ + "container driver exited with status $rc without a result" "$result_file" + fi + if [ "$timed_out" -eq 0 ]; then + case "$rc" in + 0 | 1 | 124 | 137) ;; + *) return 2 ;; + esac + fi + return 0 +} + +append_category() { + local file=$1 package=$2 reason=$3 + printf '%s\t%s\n' "$package" "$(sanitize_field "$reason")" >>"$file" +} + +print_category() { + local report=$1 title=$2 file=$3 count package reason + count=$(wc -l <"$file") + printf '\n【%s】(%s)\n' "$title" "$count" >>"$report" + if [ "$count" -eq 0 ]; then + printf ' (无)\n' >>"$report" + return + fi + while IFS=$'\t' read -r package reason || [ -n "$package" ]; do + printf ' - %s 原因: %s\n' "$package" "$reason" >>"$report" + done <"$file" +} + +valid_stage_status() { + case "$1" in PASS | FAIL | SKIP) return 0 ;; *) return 1 ;; esac +} + +aggregate_results() { + local run_suffix=$1 + local report="$OUTPUT_DIR/package-command-result-docker-$run_suffix.txt" + local detail="$OUTPUT_DIR/package-command-detail-docker-$run_suffix.tsv" + local install_fail="$RUN_DIR/install-fail.tsv" + local remove_fail="$RUN_DIR/remove-fail.tsv" + local command_fail="$RUN_DIR/command-fail.tsv" + local service_fail="$RUN_DIR/service-fail.tsv" + local skipped="$RUN_DIR/skipped.tsv" + local total=0 pass=0 fail=0 skip=0 malformed=0 + local index request source binary install install_reason command command_reason + local service service_reason remove remove_reason duration command_count unit_count current complete + local result_file has_fail has_skip safe_request package_dir service_results + local unit unit_status unit_reason + + : >"$install_fail"; : >"$remove_fail"; : >"$command_fail"; : >"$service_fail"; : >"$skipped" + printf '# source-package\tbinary_package\tinstall\tcommand\tservice\tremove\n' >"$report" + printf '# index\trequest\tsource-package\tbinary-package\tinstall\tinstall-reason\tcommand\tcommand-reason\tservice\tservice-reason\tremove\tremove-reason\tduration-ms\tcommand-count\tunit-count\tcurrent-stage\tcomplete\n' >"$detail" + + while IFS= read -r request || [ -n "$request" ]; do + total=$((total + 1)) + result_file="$RUN_DIR/results/$(printf '%05d' "$total").tsv" + [ -s "$result_file" ] || synthesize_result "$total" "$request" "missing container result" "$result_file" + IFS=$'\t' read -r index request source binary install install_reason command command_reason \ + service service_reason remove remove_reason duration command_count unit_count current complete <"$result_file" + + if ! valid_stage_status "${install:-}" || ! valid_stage_status "${command:-}" || + ! valid_stage_status "${service:-}" || ! valid_stage_status "${remove:-}"; then + synthesize_result "$total" "$request" "malformed container result" "$result_file" + IFS=$'\t' read -r index request source binary install install_reason command command_reason \ + service service_reason remove remove_reason duration command_count unit_count current complete <"$result_file" + malformed=$((malformed + 1)) + elif [ "${complete:-0}" != 1 ]; then + case "${current:-unknown}" in + install) install=FAIL; install_reason="container terminated during install" ;; + command) command=FAIL; command_reason="container terminated during command test" ;; + service) service=FAIL; service_reason="container terminated during service test" ;; + remove) remove=FAIL; remove_reason="container terminated during remove" ;; + *) install=FAIL; install_reason="container terminated at an unknown stage" ;; + esac + fi + + printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$source" "$binary" "$install" "$command" "$service" "$remove" >>"$report" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$index" "$request" "$source" "$binary" "$install" "$install_reason" \ + "$command" "$command_reason" "$service" "$service_reason" "$remove" "$remove_reason" \ + "$duration" "$command_count" "$unit_count" "$current" "$complete" >>"$detail" + + [ "$install" != FAIL ] || append_category "$install_fail" "$binary" "$install_reason" + [ "$command" != FAIL ] || append_category "$command_fail" "$binary" "$command_reason" + [ "$service" != FAIL ] || append_category "$service_fail" "$binary" "$service_reason" + [ "$remove" != FAIL ] || append_category "$remove_fail" "$binary" "$remove_reason" + + if [ "$service" = FAIL ]; then + safe_request=$(printf '%s' "$request" | tr -c 'A-Za-z0-9_.-' '_') + safe_request=${safe_request:0:80} + package_dir="$RUN_DIR/packages/$(printf '%05d' "$total")-$safe_request" + service_results="$package_dir/output/service-results.tsv" + if [ -s "$service_results" ]; then + while IFS=$'\t' read -r unit unit_status unit_reason || [ -n "$unit" ]; do + [ "$unit_status" != SKIP ] || append_category "$skipped" "$binary" \ + "$unit: $unit_reason" + done <"$service_results" + fi + fi + + has_fail=0; has_skip=0 + for status in "$install" "$command" "$service" "$remove"; do + [ "$status" != FAIL ] || has_fail=1 + [ "$status" != SKIP ] || has_skip=1 + done + if [ "$has_skip" -eq 1 ]; then + append_category "$skipped" "$binary" \ + "install=$install($install_reason); command=$command($command_reason); service=$service($service_reason); remove=$remove($remove_reason)" + fi + if [ "$has_fail" -eq 1 ]; then + fail=$((fail + 1)) + elif [ "$has_skip" -eq 1 ]; then + skip=$((skip + 1)) + else + pass=$((pass + 1)) + fi + done <"$NORMALIZED_PACKAGES" + + { + printf '\n========== 测试失败汇总 (%s) ==========\n' "$(date '+%F %T')" + printf '环境: docker | 并发度: %s | 待测包总数: %s | 通过: %s | 失败: %s | 跳过: %s\n' \ + "$JOBS" "$total" "$pass" "$fail" "$skip" + printf '镜像: %s | 服务模式: %s\n' "$TEST_IMAGE" "$EFFECTIVE_SERVICE_MODE" + } >>"$report" + print_category "$report" "安装失败" "$install_fail" + print_category "$report" "卸载失败" "$remove_fail" + print_category "$report" "命令测试失败" "$command_fail" + print_category "$report" "服务测试失败" "$service_fail" + print_category "$report" "跳过项" "$skipped" + printf '\n对账: %s + %s + %s = %s\n' "$pass" "$fail" "$skip" "$total" >>"$report" + printf '=======================================================\n' >>"$report" + + cat "$report" + log "main report: $report" + log "detail report: $detail" + [ "$malformed" -eq 0 ] || INFRA_FAILURE=1 + [ "$fail" -eq 0 ] || return 1 + return 0 +} + +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + -j | --jobs) [ "$#" -ge 2 ] || die "$1 requires a value"; JOBS=$2; shift 2 ;; + --package-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; PACKAGE_TIMEOUT=$2; shift 2 ;; + --stage-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; STAGE_TIMEOUT=$2; shift 2 ;; + --command-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; COMMAND_TIMEOUT=$2; shift 2 ;; + --service-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; SERVICE_TIMEOUT=$2; shift 2 ;; + --startup-timeout) [ "$#" -ge 2 ] || die "$1 requires a value"; STARTUP_TIMEOUT=$2; shift 2 ;; + --cpus) [ "$#" -ge 2 ] || die "$1 requires a value"; CPU_LIMIT=$2; shift 2 ;; + --memory) [ "$#" -ge 2 ] || die "$1 requires a value"; MEMORY_LIMIT=$2; shift 2 ;; + --pids-limit) [ "$#" -ge 2 ] || die "$1 requires a value"; PIDS_LIMIT=$2; shift 2 ;; + --service-mode) [ "$#" -ge 2 ] || die "$1 requires a value"; SERVICE_MODE=$2; shift 2 ;; + --base-image) [ "$#" -ge 2 ] || die "$1 requires a value"; BASE_IMAGE=$2; shift 2 ;; + --test-image) [ "$#" -ge 2 ] || die "$1 requires a value"; TEST_IMAGE=$2; TEST_IMAGE_EXPLICIT=1; shift 2 ;; + --pull) [ "$#" -ge 2 ] || die "$1 requires a value"; PULL_POLICY=$2; shift 2 ;; + --skip-build) BUILD_IMAGE=0; shift ;; + --keep-image) REMOVE_IMAGE=0; shift ;; + --remove-image) REMOVE_IMAGE=1; shift ;; + --repo-dir) [ "$#" -ge 2 ] || die "$1 requires a value"; REPO_DIR=$2; shift 2 ;; + --work-dir) [ "$#" -ge 2 ] || die "$1 requires a value"; WORK_PARENT=$2; shift 2 ;; + --output-dir) [ "$#" -ge 2 ] || die "$1 requires a value"; OUTPUT_DIR=$2; shift 2 ;; + --keep-work) KEEP_WORK=1; shift ;; + --validate-only) VALIDATE_ONLY=1; shift ;; + -h | --help) usage; exit 0 ;; + --) shift; break ;; + -*) die "unknown option: $1" ;; + *) break ;; + esac + done + [ "$#" -eq 3 ] || { usage >&2; die "exactly three input files are required"; } + PKG_INPUT=$1 + DESKTOP_INPUT=$2 + SERVICE_INPUT=$3 +} + +validate_options() { + [ -n "$JOBS" ] || JOBS=$(default_jobs) + local value + for value in "$JOBS" "$PACKAGE_TIMEOUT" "$STAGE_TIMEOUT" "$COMMAND_TIMEOUT" \ + "$SERVICE_TIMEOUT" "$STARTUP_TIMEOUT" "$PIDS_LIMIT"; do + is_positive_integer "$value" || die "integer options must be positive" + done + is_positive_decimal "$CPU_LIMIT" || die "--cpus must be a positive number" + is_memory_limit "$MEMORY_LIMIT" || die "--memory must be a positive Docker memory value" + case "$SERVICE_MODE" in auto | systemd | skip) ;; *) die "service mode must be auto, systemd, or skip" ;; esac + case "$PULL_POLICY" in always | missing | never) ;; *) die "pull policy must be always, missing, or never" ;; esac + [ -f "$PKG_INPUT" ] || die "package list not found: $PKG_INPUT" + [ -f "$DESKTOP_INPUT" ] || die "desktop list not found: $DESKTOP_INPUT" + [ -f "$SERVICE_INPUT" ] || die "service list not found: $SERVICE_INPUT" + [ -z "$REPO_DIR" ] || [ -d "$REPO_DIR" ] || die "repo directory not found: $REPO_DIR" +} + +main() { + local stamp token_fifo index=0 request pid rc=0 worker_rc + parse_args "$@" + validate_options + + WORK_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/yum-ci-docker-validate.XXXXXX") || + die "cannot create validation directory" + trap cleanup EXIT + trap on_signal INT TERM HUP + NORMALIZED_PACKAGES="$WORK_ROOT/packages.list" + NORMALIZED_DESKTOP="$WORK_ROOT/desktop.list" + NORMALIZED_SERVICES="$WORK_ROOT/services.list" + normalize_list "$PKG_INPUT" "$NORMALIZED_PACKAGES" "package list" + normalize_list "$DESKTOP_INPUT" "$NORMALIZED_DESKTOP" "desktop list" + normalize_list "$SERVICE_INPUT" "$NORMALIZED_SERVICES" "service list" + [ -s "$NORMALIZED_PACKAGES" ] || die "package list is empty after normalization" + + if [ "$VALIDATE_ONLY" -eq 1 ]; then + printf 'packages=%s jobs=%s service_mode=%s cpus=%s memory=%s pids_limit=%s\n' \ + "$(wc -l <"$NORMALIZED_PACKAGES")" "$JOBS" "$SERVICE_MODE" \ + "$CPU_LIMIT" "$MEMORY_LIMIT" "$PIDS_LIMIT" + cat "$NORMALIZED_PACKAGES" + return 0 + fi + + [ -f "$DRIVER" ] || die "container driver is not a file: $DRIVER" + for command_name in realpath timeout setsid awk sed grep tr mkfifo; do + require_command "$command_name" + done + mkdir -p "$WORK_PARENT" "$OUTPUT_DIR" + OUTPUT_DIR=$(realpath "$OUTPUT_DIR") + WORK_PARENT=$(realpath "$WORK_PARENT") + + safe_remove_tree "$WORK_ROOT" || true + WORK_ROOT=$(mktemp -d "$WORK_PARENT/yum-ci-docker.XXXXXX") || die "cannot create work directory" + NORMALIZED_PACKAGES="$WORK_ROOT/packages.list" + NORMALIZED_DESKTOP="$WORK_ROOT/desktop.list" + NORMALIZED_SERVICES="$WORK_ROOT/services.list" + normalize_list "$PKG_INPUT" "$NORMALIZED_PACKAGES" "package list" + normalize_list "$DESKTOP_INPUT" "$NORMALIZED_DESKTOP" "desktop list" + normalize_list "$SERVICE_INPUT" "$NORMALIZED_SERVICES" "service list" + + stamp=$(date '+%Y%m%d-%H%M%S') + RUN_ID="${stamp}-$$-${RANDOM:-0}" + RUN_DIR="$OUTPUT_DIR/yum-ci-docker-$RUN_ID" + mkdir -p "$RUN_DIR/packages" "$RUN_DIR/results" + resolve_test_image_tag + + if [ "${YUM_CI_DOCKER_SKIP_PREFLIGHT:-0}" = 1 ]; then + EFFECTIVE_SERVICE_MODE=$SERVICE_MODE + [ "$EFFECTIVE_SERVICE_MODE" != auto ] || EFFECTIVE_SERVICE_MODE=skip + SERVICE_MODE_REASON="Docker preflight skipped by test hook" + else + require_command docker + docker info >/dev/null 2>&1 || die "Docker daemon is unavailable" + prepare_test_image || die "test image preparation failed" + resolve_service_mode || die "systemd service mode requested but unavailable: $SERVICE_MODE_REASON" + fi + + export RUN_ID TEST_IMAGE EFFECTIVE_SERVICE_MODE SERVICE_MODE_REASON + export STAGE_TIMEOUT COMMAND_TIMEOUT SERVICE_TIMEOUT STARTUP_TIMEOUT + export CPU_LIMIT MEMORY_LIMIT PIDS_LIMIT REPO_DIR PACKAGE_TIMEOUT + [ "${YUM_CI_DOCKER_SKIP_PREFLIGHT:-0}" = 1 ] || write_manifest + + token_fifo="$WORK_ROOT/tokens" + mkfifo "$token_fifo" + exec 9<>"$token_fifo" + rm -f "$token_fifo" + for ((index = 0; index < JOBS; index++)); do printf '.' >&9; done + + index=0 + while IFS= read -r request || [ -n "$request" ]; do + index=$((index + 1)) + IFS= read -r -n 1 -u 9 + ( + worker_rc=0 + run_one "$index" "$request" || worker_rc=$? + printf '.' >&9 + exit "$worker_rc" + ) & + pid=$! + CHILD_PIDS+=("$pid") + done <"$NORMALIZED_PACKAGES" + + for pid in "${CHILD_PIDS[@]}"; do + wait "$pid" || INFRA_FAILURE=1 + done + exec 9>&- + + aggregate_results "$RUN_ID" || rc=1 + [ "$INFRA_FAILURE" -eq 0 ] || rc=2 + if [ -f "$RUN_DIR/run-manifest.txt" ]; then + printf 'finished=%s\nexit_code=%s\n' "$(date -Is)" "$rc" >>"$RUN_DIR/run-manifest.txt" + fi + return "$rc" +} + +main "$@" diff --git a/testcase/docker/Dockerfile b/testcase/docker/Dockerfile new file mode 100644 index 0000000..6e3b46e --- /dev/null +++ b/testcase/docker/Dockerfile @@ -0,0 +1,28 @@ +ARG BASE_IMAGE=opencloudos/opencloudos9-init:latest +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="yum-ci package test runtime" \ + org.opencontainers.image.description="OpenCloudOS package lifecycle test runtime for IJV5J9" \ + io.opencloudos.yum-ci.issue="IJV5J9" + +RUN dnf -y install \ + bash \ + coreutils \ + file \ + findutils \ + grep \ + procps-ng \ + rpm \ + sed \ + strace \ + systemd \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +COPY tst_lib/ts_common.sh tst_lib/other_common.sh /opt/yum-ci/tst_lib/ +COPY testcase/lib/docker-ts-common-runner.sh /usr/local/libexec/yum-ci/docker-ts-common-runner.sh +RUN sed -i 's/\r$//' /opt/yum-ci/tst_lib/*.sh \ + && chmod 0755 /usr/local/libexec/yum-ci/docker-ts-common-runner.sh + +STOPSIGNAL SIGRTMIN+3 +CMD ["/sbin/init"] diff --git a/testcase/lib/docker-package-driver.sh b/testcase/lib/docker-package-driver.sh new file mode 100755 index 0000000..dee89e0 --- /dev/null +++ b/testcase/lib/docker-package-driver.sh @@ -0,0 +1,188 @@ +#!/bin/bash +# Host-side lifecycle driver for one isolated package container. + +set -uo pipefail + +[ "$#" -eq 6 ] || { + echo "usage: docker-package-driver.sh INDEX REQUEST PACKAGE_DIR RESULT_FILE DESKTOP_LIST SERVICE_LIST" >&2 + exit 2 +} + +INDEX=$1 +REQUEST=$2 +PACKAGE_DIR=$3 +RESULT_FILE=$4 +DESKTOP_LIST=$5 +SERVICE_LIST=$6 + +RUN_ID=${RUN_ID:?RUN_ID is required} +TEST_IMAGE=${TEST_IMAGE:?TEST_IMAGE is required} +EFFECTIVE_SERVICE_MODE=${EFFECTIVE_SERVICE_MODE:-skip} +SERVICE_MODE_REASON=${SERVICE_MODE_REASON:-unspecified} +STARTUP_TIMEOUT=${STARTUP_TIMEOUT:-60} +STAGE_TIMEOUT=${STAGE_TIMEOUT:-300} +COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-5} +SERVICE_TIMEOUT=${SERVICE_TIMEOUT:-60} +CPU_LIMIT=${CPU_LIMIT:-1} +MEMORY_LIMIT=${MEMORY_LIMIT:-1g} +PIDS_LIMIT=${PIDS_LIMIT:-512} +REPO_DIR=${REPO_DIR:-} + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +WORKER=${YUM_CI_DOCKER_WORKER:-$SCRIPT_DIR/docker-package-worker.sh} +CONTAINER_NAME="yumci-${RUN_ID}-${INDEX}" +CONTAINER_STARTED=0 +CONTAINER_RESULT="$PACKAGE_DIR/output/result.tsv" +TERMINATION_REASON="" + +sanitize() { + printf '%s' "${1:-}" | tr '\t\r\n' ' ' | + sed 's/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//' +} + +write_driver_failure() { + local reason=$1 tmp + [ -s "$RESULT_FILE" ] && return 0 + tmp="$RESULT_FILE.tmp.$$" + printf '%s\t%s\tunknown\t%s\tFAIL\t%s\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\t0\t0\t0\tcontainer\t1\n' \ + "$INDEX" "$(sanitize "$REQUEST")" "$(sanitize "$REQUEST")" "$(sanitize "$reason")" >"$tmp" + mv -f "$tmp" "$RESULT_FILE" +} + +publish_worker_result() { + local tmp + [ -s "$CONTAINER_RESULT" ] || return 1 + tmp="$RESULT_FILE.tmp.$$" + cp "$CONTAINER_RESULT" "$tmp" + mv -f "$tmp" "$RESULT_FILE" +} + +remove_staged_repo() { + local staged="$PACKAGE_DIR/repos" + [ -d "$staged" ] || return 0 + case "$staged" in + "$PACKAGE_DIR"/repos) rm -rf -- "$staged" ;; + *) return 1 ;; + esac +} + +cleanup() { + local rc=$? + trap - EXIT INT TERM HUP + if [ "$CONTAINER_STARTED" -eq 1 ]; then + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + fi + [ -s "$RESULT_FILE" ] || publish_worker_result || true + if [ ! -s "$RESULT_FILE" ]; then + write_driver_failure "${TERMINATION_REASON:-container driver terminated unexpectedly with status $rc}" + fi + remove_staged_repo || rc=2 + exit "$rc" +} + +on_signal() { + TERMINATION_REASON="container lifecycle interrupted by signal" + exit 143 +} + +trap cleanup EXIT +trap on_signal INT TERM HUP + +mkdir -p "$PACKAGE_DIR/output" "$(dirname -- "$RESULT_FILE")" +: >"$PACKAGE_DIR/driver.log" + +[ -f "$WORKER" ] || { + write_driver_failure "container worker is not a file: $WORKER" + exit 2 +} + +if ! cp "$WORKER" "$PACKAGE_DIR/worker.sh" || + ! cp "$DESKTOP_LIST" "$PACKAGE_DIR/desktop.list" || + ! cp "$SERVICE_LIST" "$PACKAGE_DIR/service.list"; then + write_driver_failure "failed to stage worker and input lists" + exit 2 +fi +mount_args=(--volume "$PACKAGE_DIR:/yum-ci:Z") +if [ -n "$REPO_DIR" ]; then + mkdir -p "$PACKAGE_DIR/repos" + if ! cp -a "$REPO_DIR"/. "$PACKAGE_DIR/repos/"; then + write_driver_failure "failed to stage repository configuration" + exit 2 + fi + mount_args+=(--volume "$PACKAGE_DIR/repos:/etc/yum.repos.d:ro") +fi + +common_args=( + --detach + --rm + --name "$CONTAINER_NAME" + --hostname "yumci-$INDEX" + --label "io.opencloudos.yum-ci.run=$RUN_ID" + --label "io.opencloudos.yum-ci.index=$INDEX" + --label "io.opencloudos.yum-ci.package=$REQUEST" + --cpus "$CPU_LIMIT" + --memory "$MEMORY_LIMIT" + --pids-limit "$PIDS_LIMIT" + --cap-add SYS_PTRACE + --stop-timeout 10 + "${mount_args[@]}" +) + +if [ "$EFFECTIVE_SERVICE_MODE" = systemd ]; then + common_args+=(--privileged --cgroupns=private --entrypoint /sbin/init) +else + common_args+=(--entrypoint /usr/bin/sleep) +fi + +if [ "$EFFECTIVE_SERVICE_MODE" = systemd ]; then + docker run "${common_args[@]}" "$TEST_IMAGE" >>"$PACKAGE_DIR/driver.log" 2>&1 || { + write_driver_failure "failed to start privileged systemd container" + exit 2 + } +else + docker run "${common_args[@]}" "$TEST_IMAGE" infinity >>"$PACKAGE_DIR/driver.log" 2>&1 || { + write_driver_failure "failed to start package container" + exit 2 + } +fi +CONTAINER_STARTED=1 + +if [ "$EFFECTIVE_SERVICE_MODE" = systemd ]; then + state="" + for ((attempt = 0; attempt < STARTUP_TIMEOUT; attempt++)); do + state=$(docker exec "$CONTAINER_NAME" systemctl is-system-running 2>/dev/null || true) + case "$state" in + running | degraded) break ;; + initializing | starting | "") sleep 1 ;; + *) break ;; + esac + done + case "$state" in + running | degraded) ;; + *) + write_driver_failure "systemd container did not become ready (state=${state:-unknown})" + exit 2 + ;; + esac +fi + +docker exec \ + --env "INDEX=$INDEX" \ + --env "REQUEST=$REQUEST" \ + --env "OUTPUT_DIR=/yum-ci/output" \ + --env "RESULT_FILE=/yum-ci/output/result.tsv" \ + --env "DESKTOP_LIST=/yum-ci/desktop.list" \ + --env "SERVICE_LIST=/yum-ci/service.list" \ + --env "SERVICE_MODE=$EFFECTIVE_SERVICE_MODE" \ + --env "SERVICE_MODE_REASON=$SERVICE_MODE_REASON" \ + --env "STAGE_TIMEOUT=$STAGE_TIMEOUT" \ + --env "COMMAND_TIMEOUT=$COMMAND_TIMEOUT" \ + --env "SERVICE_TIMEOUT=$SERVICE_TIMEOUT" \ + "$CONTAINER_NAME" bash /yum-ci/worker.sh >>"$PACKAGE_DIR/driver.log" 2>&1 +rc=$? + +if [ ! -s "$RESULT_FILE" ]; then + publish_worker_result || + write_driver_failure "container worker exited with status $rc without a result" +fi +exit "$rc" diff --git a/testcase/lib/docker-package-worker.sh b/testcase/lib/docker-package-worker.sh new file mode 100755 index 0000000..9fe8cb7 --- /dev/null +++ b/testcase/lib/docker-package-worker.sh @@ -0,0 +1,483 @@ +#!/bin/bash +# Container-side package worker. One invocation owns one disposable container. + +set -uo pipefail + +INDEX=${INDEX:?INDEX is required} +REQUEST=${REQUEST:?REQUEST is required} +OUTPUT_DIR=${OUTPUT_DIR:-/yum-ci/output} +RESULT_FILE=${RESULT_FILE:-$OUTPUT_DIR/result.tsv} +DESKTOP_LIST=${DESKTOP_LIST:-/yum-ci/desktop.list} +SERVICE_LIST=${SERVICE_LIST:-/yum-ci/service.list} +SERVICE_MODE=${SERVICE_MODE:-skip} +SERVICE_MODE_REASON=${SERVICE_MODE_REASON:-environment does not provide systemd} +STAGE_TIMEOUT=${STAGE_TIMEOUT:-300} +COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-5} +SERVICE_TIMEOUT=${SERVICE_TIMEOUT:-60} +TS_COMMON_RUNNER=${TS_COMMON_RUNNER:-/usr/local/libexec/yum-ci/docker-ts-common-runner.sh} + +START_MS=$(date +%s%3N) +SOURCE_PACKAGE=unknown +BINARY_PACKAGE=$REQUEST +PACKAGE_NAME="" +PACKAGE_ARCH="" +PACKAGE_SPEC="" +PREINSTALLED=0 +INSTALL_STATUS=SKIP +INSTALL_REASON="not reached" +COMMAND_STATUS=SKIP +COMMAND_REASON="not reached" +SERVICE_STATUS=SKIP +SERVICE_REASON="not reached" +REMOVE_STATUS=SKIP +REMOVE_REASON="not reached" +COMMAND_COUNT=0 +UNIT_COUNT=0 +CURRENT_STAGE=bootstrap +COMPLETE=0 + +sanitize() { + printf '%s' "${1:-}" | tr '\t\r\n' ' ' | + sed 's/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//' +} + +elapsed_ms() { + local now + now=$(date +%s%3N) + printf '%s\n' "$((now - START_MS))" +} + +log() { + printf '[%s] %s\n' "$(date '+%F %T')" "$*" | tee -a "$OUTPUT_DIR/worker.log" >&2 +} + +checkpoint() { + local tmp="$RESULT_FILE.tmp.$$" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$INDEX" "$(sanitize "$REQUEST")" "$(sanitize "$SOURCE_PACKAGE")" \ + "$(sanitize "$BINARY_PACKAGE")" "$INSTALL_STATUS" "$(sanitize "$INSTALL_REASON")" \ + "$COMMAND_STATUS" "$(sanitize "$COMMAND_REASON")" \ + "$SERVICE_STATUS" "$(sanitize "$SERVICE_REASON")" \ + "$REMOVE_STATUS" "$(sanitize "$REMOVE_REASON")" \ + "$(elapsed_ms)" "$COMMAND_COUNT" "$UNIT_COUNT" "$CURRENT_STAGE" "$COMPLETE" >"$tmp" + mv -f "$tmp" "$RESULT_FILE" +} + +stage_begin() { + CURRENT_STAGE=$1 + checkpoint +} + +mark_current_stage_failed() { + local reason=$1 + case "$CURRENT_STAGE" in + install | bootstrap) + INSTALL_STATUS=FAIL; INSTALL_REASON=$reason + COMMAND_STATUS=SKIP; COMMAND_REASON="install did not complete" + SERVICE_STATUS=SKIP; SERVICE_REASON="install did not complete" + REMOVE_STATUS=SKIP; REMOVE_REASON="install did not complete" + ;; + command) COMMAND_STATUS=FAIL; COMMAND_REASON=$reason ;; + service) SERVICE_STATUS=FAIL; SERVICE_REASON=$reason ;; + remove) REMOVE_STATUS=FAIL; REMOVE_REASON=$reason ;; + esac +} + +cleanup() { + local rc=$? + trap - EXIT INT TERM HUP + if [ "$COMPLETE" -ne 1 ]; then + mark_current_stage_failed "worker terminated during $CURRENT_STAGE with status $rc" + COMPLETE=1 + checkpoint + fi + exit "$rc" +} + +on_signal() { + mark_current_stage_failed "worker interrupted during $CURRENT_STAGE" + COMPLETE=1 + checkpoint + exit 143 +} + +trap cleanup EXIT +trap on_signal INT TERM HUP + +first_error() { + local file=$1 fallback=$2 line + line=$(grep -Eim1 'error:|failed|failure|conflict|nothing to do|no match|timed out|cannot|not found|requires' \ + "$file" 2>/dev/null || true) + [ -n "$line" ] && sanitize "$line" || printf '%s' "$fallback" +} + +run_with_timeout() { + local seconds=$1 log_file=$2 + shift 2 + timeout -k 10 "$seconds" "$@" >"$log_file" 2>&1 +} + +list_matches() { + local file=$1 item candidate + shift + [ -s "$file" ] || return 1 + while IFS= read -r item || [ -n "$item" ]; do + item=${item%$'\r'} + item=${item%%#*} + item=${item%%[[:space:]]*} + [ -n "$item" ] || continue + for candidate in "$@"; do + [ -n "$candidate" ] && [ "$item" = "$candidate" ] && return 0 + done + done <"$file" + return 1 +} + +query_installed_package() { + local query + query=$(rpm -q --qf '%{SOURCERPM}\t%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}\t%{NAME}\t%{ARCH}\n' \ + "$REQUEST" 2>/dev/null | head -n 1) || return 1 + [ -n "$query" ] || return 1 + IFS=$'\t' read -r SOURCE_PACKAGE BINARY_PACKAGE PACKAGE_NAME PACKAGE_ARCH <<<"$query" + SOURCE_PACKAGE=${SOURCE_PACKAGE%.src.rpm} + PACKAGE_SPEC="$PACKAGE_NAME.$PACKAGE_ARCH" +} + +install_package() { + local rc=0 + stage_begin install + if rpm -q "$REQUEST" >/dev/null 2>&1; then + PREINSTALLED=1 + fi + if [ ! -x "$TS_COMMON_RUNNER" ]; then + INSTALL_STATUS=FAIL + INSTALL_REASON="ts_common adapter is missing from the test image" + return 1 + fi + log "install stage: tst_lib/ts_common.sh test_single_rpm_install $REQUEST" + run_with_timeout "$STAGE_TIMEOUT" "$OUTPUT_DIR/install.log" \ + "$TS_COMMON_RUNNER" install "$REQUEST" "$OUTPUT_DIR/common-install" || rc=$? + if [ "$rc" -ne 0 ]; then + INSTALL_STATUS=FAIL + if [ "$rc" -eq 124 ]; then + INSTALL_REASON="installation timed out after ${STAGE_TIMEOUT}s" + elif [ "$rc" -eq 137 ]; then + INSTALL_REASON="package installation was killed (possible OOM or external SIGKILL)" + else + INSTALL_REASON=$(first_error "$OUTPUT_DIR/install.log" "ts_common install predicate exited with status $rc") + fi + return 1 + fi + if grep -Eqi 'scriptlet failed|cpio:.*failed|No such file or directory' "$OUTPUT_DIR/install.log"; then + INSTALL_STATUS=FAIL + INSTALL_REASON=$(first_error "$OUTPUT_DIR/install.log" "install log contains a scriptlet/file error") + return 1 + fi + if ! query_installed_package; then + INSTALL_STATUS=FAIL + INSTALL_REASON="dnf succeeded but rpm did not find $REQUEST" + return 1 + fi + if ! rpm -ql "$PACKAGE_SPEC" >"$OUTPUT_DIR/rpm-files.list" 2>"$OUTPUT_DIR/rpm-files.log"; then + INSTALL_STATUS=FAIL + INSTALL_REASON="rpm -ql failed for $PACKAGE_SPEC" + return 1 + fi + INSTALL_STATUS=PASS + if [ "$PREINSTALLED" -eq 1 ]; then + INSTALL_REASON="ts_common verified preinstalled package as $BINARY_PACKAGE" + else + INSTALL_REASON="installed as $BINARY_PACKAGE" + fi + checkpoint +} + +probe_command() { + local command_path=$1 trace_file=$2 argument=$3 rc=0 + local attempt_log="$trace_file.log" + : >"$trace_file" + if [ "$argument" = __NO_ARG__ ]; then + timeout -k 2 "$COMMAND_TIMEOUT" strace -f -qq -e trace=execve -o "$trace_file" \ + "$command_path" "$attempt_log" 2>&1 || rc=$? + else + timeout -k 2 "$COMMAND_TIMEOUT" strace -f -qq -e trace=execve -o "$trace_file" \ + "$command_path" "$argument" "$attempt_log" 2>&1 || rc=$? + fi + cat "$attempt_log" >>"$OUTPUT_DIR/command.log" + if [ "$rc" -eq 124 ]; then + return 2 + fi + if grep -Eqi 'error while loading shared libraries|cannot open shared object file' "$attempt_log"; then + return 4 + fi + if [ "$rc" -eq 132 ] || [ "$rc" -eq 134 ] || [ "$rc" -eq 135 ] || + [ "$rc" -eq 137 ] || [ "$rc" -eq 139 ] || + grep -Eq 'killed by (SIGSEGV|SIGABRT|SIGBUS|SIGILL)' "$trace_file"; then + return 3 + fi + grep -Eq 'execve\(.*\)[[:space:]]*=[[:space:]]*0' "$trace_file" || return 1 + return 0 +} + +test_commands() { + local path type argument trace rc failed=0 timeouts=0 crashes=0 load_errors=0 reason + stage_begin command + : >"$OUTPUT_DIR/commands.list" + : >"$OUTPUT_DIR/command-results.tsv" + : >"$OUTPUT_DIR/command.log" + mkdir -p "$OUTPUT_DIR/traces" + while IFS= read -r path || [ -n "$path" ]; do + case "$path" in /usr/bin/* | /usr/sbin/*) ;; *) continue ;; esac + [ -x "$path" ] || continue + type=$(file -Lb "$path" 2>/dev/null || true) + [[ $type == *ELF* ]] || continue + printf '%s\n' "$path" >>"$OUTPUT_DIR/commands.list" + done <"$OUTPUT_DIR/rpm-files.list" + + COMMAND_COUNT=$(wc -l <"$OUTPUT_DIR/commands.list") + if [ "$COMMAND_COUNT" -eq 0 ]; then + COMMAND_STATUS=PASS + COMMAND_REASON="package contains no executable ELF command under /usr/bin or /usr/sbin" + checkpoint + return 0 + fi + + command_index=0 + while IFS= read -r path || [ -n "$path" ]; do + command_index=$((command_index + 1)) + printf '\n===== %s =====\n' "$path" >>"$OUTPUT_DIR/command.log" + rc=1 + reason="execve did not succeed" + for argument in --help --version __NO_ARG__; do + trace="$OUTPUT_DIR/traces/$(printf '%05d' "$command_index").trace" + if probe_command "$path" "$trace" "$argument"; then + rc=0 + reason="execve succeeded" + break + else + rc=$? + fi + case "$rc" in + 2) reason="command timed out after ${COMMAND_TIMEOUT}s"; timeouts=$((timeouts + 1)) ;; + 3) reason="command terminated by a fatal signal"; crashes=$((crashes + 1)); break ;; + 4) reason="command has a missing shared-library dependency"; load_errors=$((load_errors + 1)); break ;; + *) reason="execve was not observed" ;; + esac + done + if [ "$rc" -eq 0 ]; then + printf '%s\tPASS\t%s\n' "$path" "$reason" >>"$OUTPUT_DIR/command-results.tsv" + else + failed=$((failed + 1)) + printf '%s\tFAIL\t%s\n' "$path" "$reason" >>"$OUTPUT_DIR/command-results.tsv" + fi + done <"$OUTPUT_DIR/commands.list" + + if [ "$failed" -gt 0 ]; then + COMMAND_STATUS=FAIL + COMMAND_REASON="$failed/$COMMAND_COUNT commands failed executable smoke tests (timeouts=$timeouts crashes=$crashes missing-libs=$load_errors)" + checkpoint + return 1 + fi + COMMAND_STATUS=PASS + COMMAND_REASON="all $COMMAND_COUNT executable ELF commands reached execve successfully" + checkpoint +} + +discover_units() { + local path + : >"$OUTPUT_DIR/units.list" + while IFS= read -r path || [ -n "$path" ]; do + case "$path" in + /usr/lib/systemd/system/*.service | /usr/lib/systemd/system/*.socket | /usr/lib/systemd/system/*.target | \ + /lib/systemd/system/*.service | /lib/systemd/system/*.socket | /lib/systemd/system/*.target | \ + /etc/systemd/system/*.service | /etc/systemd/system/*.socket | /etc/systemd/system/*.target) + basename "$path" + ;; + esac + done <"$OUTPUT_DIR/rpm-files.list" | sort -u >"$OUTPUT_DIR/units.list" + UNIT_COUNT=$(wc -l <"$OUTPUT_DIR/units.list") +} + +systemd_ready() { + local state + [ -d /run/systemd/system ] || return 1 + state=$(systemctl is-system-running 2>/dev/null || true) + case "$state" in running | degraded) return 0 ;; *) return 1 ;; esac +} + +test_services() { + local unit start_rc active_rc stop_rc daemon_reload_rc=0 failed=0 tested=0 skipped=0 + local refuse_start refuse_stop + stage_begin service + : >"$OUTPUT_DIR/service-results.tsv" + : >"$OUTPUT_DIR/service.log" + discover_units + if [ "$UNIT_COUNT" -eq 0 ]; then + SERVICE_STATUS=PASS + SERVICE_REASON="package provides no systemd service/socket/target units" + checkpoint + return 0 + fi + if list_matches "$SERVICE_LIST" "$REQUEST" "$PACKAGE_NAME" "$PACKAGE_NAME.$PACKAGE_ARCH" "$BINARY_PACKAGE"; then + SERVICE_STATUS=SKIP + SERVICE_REASON="package is in the special-configuration service list" + awk '{printf "%s\tSKIP\trequires special configuration\n", $0}' \ + "$OUTPUT_DIR/units.list" >"$OUTPUT_DIR/service-results.tsv" + checkpoint + return 0 + fi + if [ "$SERVICE_MODE" != systemd ]; then + SERVICE_STATUS=SKIP + SERVICE_REASON="service environment unavailable: $SERVICE_MODE_REASON" + awk -v reason="$(sanitize "$SERVICE_MODE_REASON")" \ + '{printf "%s\tSKIP\t%s\n", $0, reason}' "$OUTPUT_DIR/units.list" \ + >"$OUTPUT_DIR/service-results.tsv" + checkpoint + return 0 + fi + if ! systemd_ready; then + SERVICE_STATUS=FAIL + SERVICE_REASON="systemd was requested but is not running in the container" + checkpoint + return 1 + fi + run_with_timeout "$SERVICE_TIMEOUT" "$OUTPUT_DIR/service.log" systemctl daemon-reload || daemon_reload_rc=$? + if [ "$daemon_reload_rc" -ne 0 ]; then + SERVICE_STATUS=FAIL + if [ "$daemon_reload_rc" -eq 124 ]; then + SERVICE_REASON="systemctl daemon-reload timed out after ${SERVICE_TIMEOUT}s" + elif [ "$daemon_reload_rc" -eq 137 ]; then + SERVICE_REASON="systemctl daemon-reload was killed (possible OOM or external SIGKILL)" + else + SERVICE_REASON="systemctl daemon-reload failed with status $daemon_reload_rc" + fi + checkpoint + return 1 + fi + + while IFS= read -r unit || [ -n "$unit" ]; do + [ -n "$unit" ] || continue + if [[ $unit == *@.* ]]; then + skipped=$((skipped + 1)) + printf '%s\tSKIP\ttemplate unit requires an instance name\n' "$unit" \ + >>"$OUTPUT_DIR/service-results.tsv" + continue + fi + refuse_start=$(systemctl show "$unit" -p RefuseManualStart --value 2>/dev/null || true) + refuse_stop=$(systemctl show "$unit" -p RefuseManualStop --value 2>/dev/null || true) + if [ "$refuse_start" = yes ] || [ "$refuse_stop" = yes ]; then + skipped=$((skipped + 1)) + printf '%s\tSKIP\tRefuseManualStart=%s RefuseManualStop=%s\n' \ + "$unit" "${refuse_start:-no}" "${refuse_stop:-no}" \ + >>"$OUTPUT_DIR/service-results.tsv" + continue + fi + tested=$((tested + 1)) + start_rc=0; active_rc=125; stop_rc=0 + printf '\n===== %s =====\n' "$unit" >>"$OUTPUT_DIR/service.log" + timeout -k 5 "$SERVICE_TIMEOUT" systemctl --no-pager start "$unit" \ + >>"$OUTPUT_DIR/service.log" 2>&1 || start_rc=$? + if [ "$start_rc" -eq 0 ]; then + active_rc=0 + timeout -k 5 "$SERVICE_TIMEOUT" systemctl --no-pager is-active "$unit" \ + >>"$OUTPUT_DIR/service.log" 2>&1 || active_rc=$? + fi + timeout -k 5 "$SERVICE_TIMEOUT" systemctl --no-pager stop "$unit" \ + >>"$OUTPUT_DIR/service.log" 2>&1 || stop_rc=$? + if [ "$start_rc" -eq 0 ] && [ "$active_rc" -eq 0 ] && [ "$stop_rc" -eq 0 ]; then + printf '%s\tPASS\tstart=0 is-active=0 stop=0\n' "$unit" \ + >>"$OUTPUT_DIR/service-results.tsv" + else + failed=$((failed + 1)) + printf '%s\tFAIL\tstart=%s is-active=%s stop=%s\n' \ + "$unit" "$start_rc" "$active_rc" "$stop_rc" >>"$OUTPUT_DIR/service-results.tsv" + systemctl --no-pager status "$unit" >>"$OUTPUT_DIR/service.log" 2>&1 || true + fi + done <"$OUTPUT_DIR/units.list" + + if [ "$failed" -gt 0 ]; then + SERVICE_STATUS=FAIL + SERVICE_REASON="$failed/$tested testable units failed start/is-active/stop; $skipped skipped" + checkpoint + return 1 + fi + if [ "$skipped" -gt 0 ]; then + SERVICE_STATUS=SKIP + SERVICE_REASON="$tested testable units passed; $skipped units skipped with explicit reasons" + checkpoint + return 0 + fi + SERVICE_STATUS=PASS + SERVICE_REASON="all $tested units passed start/is-active/stop" + checkpoint +} + +remove_package() { + local rc=0 + stage_begin remove + log "remove stage: tst_lib/ts_common.sh test_single_rpm_remove $BINARY_PACKAGE" + run_with_timeout "$STAGE_TIMEOUT" "$OUTPUT_DIR/remove.log" \ + "$TS_COMMON_RUNNER" remove "$PACKAGE_SPEC" "$BINARY_PACKAGE" \ + "$OUTPUT_DIR/common-remove" || rc=$? + if [ "$rc" -ne 0 ]; then + REMOVE_STATUS=FAIL + if [ "$rc" -eq 124 ]; then + REMOVE_REASON="removal timed out after ${STAGE_TIMEOUT}s" + elif [ "$rc" -eq 137 ]; then + REMOVE_REASON="dnf remove was killed (possible OOM or external SIGKILL)" + else + REMOVE_REASON=$(first_error "$OUTPUT_DIR/remove.log" "ts_common remove predicate exited with status $rc") + fi + checkpoint + return 1 + fi + if grep -Eqi 'scriptlet failed|cpio:.*failed' "$OUTPUT_DIR/remove.log"; then + REMOVE_STATUS=FAIL + REMOVE_REASON=$(first_error "$OUTPUT_DIR/remove.log" "remove log contains a scriptlet error") + checkpoint + return 1 + fi + if rpm -q "$PACKAGE_SPEC" >/dev/null 2>&1; then + REMOVE_STATUS=FAIL + REMOVE_REASON="remove predicate succeeded but rpm still reports $PACKAGE_SPEC installed" + checkpoint + return 1 + fi + REMOVE_STATUS=PASS + REMOVE_REASON="package removed and rpm confirms absence" + checkpoint +} + +main() { + local rc=0 + mkdir -p "$OUTPUT_DIR" + : >"$OUTPUT_DIR/worker.log" + checkpoint + log "testing $REQUEST in isolated Docker container" + + if ! install_package; then + COMMAND_STATUS=SKIP; COMMAND_REASON="install failed" + SERVICE_STATUS=SKIP; SERVICE_REASON="install failed" + REMOVE_STATUS=SKIP; REMOVE_REASON="install failed" + COMPLETE=1 + checkpoint + return 1 + fi + + if list_matches "$DESKTOP_LIST" "$REQUEST" "$PACKAGE_NAME" "$PACKAGE_NAME.$PACKAGE_ARCH" "$BINARY_PACKAGE"; then + CURRENT_STAGE="command" + COMMAND_STATUS=SKIP + COMMAND_REASON="package is in the desktop whitelist" + checkpoint + else + test_commands || rc=1 + fi + test_services || rc=1 + remove_package || rc=1 + CURRENT_STAGE=complete + COMPLETE=1 + checkpoint + return "$rc" +} + +main diff --git a/testcase/lib/docker-ts-common-runner.sh b/testcase/lib/docker-ts-common-runner.sh new file mode 100755 index 0000000..0d8c94d --- /dev/null +++ b/testcase/lib/docker-ts-common-runner.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Timeout-friendly adapter around the existing yum-ci package predicates. + +set -uo pipefail + +ACTION=${1:-} +export TST_TS_TOPDIR=/opt/yum-ci + +msg() { + printf '[ts_common] %s\n' "$*" >&2 +} + +err() { + printf '[ts_common] ERROR: %s\n' "$*" >&2 +} + +# shellcheck source=/dev/null +source "$TST_TS_TOPDIR/tst_lib/ts_common.sh" || exit 2 + +case "$ACTION" in + install) + [ "$#" -eq 3 ] || exit 2 + test_single_rpm_install "--setopt=install_weak_deps=False" "$2" "$3" + ;; + remove) + [ "$#" -eq 4 ] || exit 2 + # Every package lives in a disposable container, so force the common + # predicate down its real remove path even when the base image had it. + test_single_rpm_remove "$2" "$3" "$4" /dev/null + ;; + *) + echo "usage: $0 install PACKAGE LOG_BASE | remove SHORT LONG LOG_BASE" >&2 + exit 2 + ;; +esac diff --git a/testcase/tests/fake-docker-package-driver.sh b/testcase/tests/fake-docker-package-driver.sh new file mode 100755 index 0000000..1ec85da --- /dev/null +++ b/testcase/tests/fake-docker-package-driver.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Deterministic host-only driver used by scheduler/report regression tests. + +set -uo pipefail + +[ "$#" -eq 6 ] || exit 2 +index=$1 +request=$2 +package_dir=$3 +result_file=$4 +desktop_list=$5 +service_list=$6 + +mkdir -p "$package_dir/output" "$(dirname -- "$result_file")" + +list_has() { + local file=$1 value=$2 + grep -Fqx "$value" "$file" 2>/dev/null +} + +install=PASS; install_reason="installed" +command=PASS; command_reason="command passed" +service=PASS; service_reason="service passed" +remove=PASS; remove_reason="removed" +infra=0 + +case "$request" in + sleep-*) sleep "${FAKE_SLEEP_SECONDS:-1}" ;; + timeout-command-*) + printf '%s\t%s\t%s-src\t%s\tPASS\tinstalled\tSKIP\tcommand running\tSKIP\tnot reached\tSKIP\tnot reached\t100\t1\t0\tcommand\t0\n' \ + "$index" "$request" "$request" "$request" >"$result_file" + sleep "${FAKE_TIMEOUT_SECONDS:-5}" + exit 124 + ;; + timeout-*) sleep "${FAKE_TIMEOUT_SECONDS:-5}"; exit 124 ;; + infra-*) + infra=1 + install=FAIL; install_reason="injected Docker infrastructure failure" + command=SKIP; command_reason="infrastructure failure" + service=SKIP; service_reason="infrastructure failure" + remove=SKIP; remove_reason="infrastructure failure" + ;; + crash-*) exit 23 ;; + install-fail-*) + install=FAIL; install_reason="injected install failure" + command=SKIP; command_reason="install failed" + service=SKIP; service_reason="install failed" + remove=SKIP; remove_reason="install failed" + ;; + command-fail-*) command=FAIL; command_reason="injected command failure" ;; + service-fail-*) service=FAIL; service_reason="injected service failure" ;; + service-mixed-*) + service=FAIL; service_reason="injected normal unit failure" + printf 'demo@.service\tSKIP\ttemplate unit requires an instance name\n' \ + >"$package_dir/output/service-results.tsv" + printf 'demo.service\tFAIL\tstart=1 is-active=125 stop=0\n' \ + >>"$package_dir/output/service-results.tsv" + ;; + remove-fail-*) remove=FAIL; remove_reason="injected remove failure" ;; + multi-fail-*) + command=FAIL; command_reason="injected command failure" + remove=FAIL; remove_reason="injected remove failure" + ;; + template-unit-*) service=SKIP; service_reason="template unit requires an instance name" ;; +esac + +if list_has "$desktop_list" "$request"; then + command=SKIP + command_reason="package is in the desktop whitelist" +fi +if list_has "$service_list" "$request"; then + service=SKIP + service_reason="package is in the special-configuration service list" +fi + +tmp="$result_file.tmp.$$" +printf '%s\t%s\t%s-src\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t100\t1\t1\tcomplete\t1\n' \ + "$index" "$request" "$request" "$request" "$install" "$install_reason" \ + "$command" "$command_reason" "$service" "$service_reason" "$remove" "$remove_reason" >"$tmp" +mv -f "$tmp" "$result_file" + +[ "$infra" -eq 0 ] || exit 2 + +case "$install:$command:$service:$remove" in + *FAIL*) exit 1 ;; + *) exit 0 ;; +esac diff --git a/testcase/tests/test-command-execute-docker-e2e.sh b/testcase/tests/test-command-execute-docker-e2e.sh new file mode 100755 index 0000000..26aa499 --- /dev/null +++ b/testcase/tests/test-command-execute-docker-e2e.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Real Docker integration test. Intended for an isolated OpenCloudOS 9 CI host. + +set -uo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +CASE_DIR=$(cd -- "$SCRIPT_DIR/.." && pwd -P) +MAIN="$CASE_DIR/command-execute-docker.sh" +OUTPUT_ROOT=${E2E_OUTPUT_DIR:-"$PWD/docker-e2e-$(date '+%Y%m%d-%H%M%S')"} +IMAGE=${E2E_IMAGE:-"yum-ci-pkg-test:ijv5j9-e2e-$$"} +AUTO_IMAGE=${E2E_AUTO_IMAGE:-"yum-ci-pkg-test:ijv5j9-auto-remove-$$"} +TMP=$(mktemp -d) +PASS=0 +FAIL=0 + +cleanup() { + [ -z "${IMAGE:-}" ] || docker image rm "$IMAGE" >/dev/null 2>&1 || true + [ -z "${AUTO_IMAGE:-}" ] || docker image rm "$AUTO_IMAGE" >/dev/null 2>&1 || true + rm -rf -- "$TMP" +} +trap cleanup EXIT + +ok() { PASS=$((PASS + 1)); printf 'PASS: %s\n' "$1"; } +not_ok() { FAIL=$((FAIL + 1)); printf 'FAIL: %s\n' "$1" >&2; } + +assert_contains() { + local file=$1 pattern=$2 name=$3 + if grep -Fq "$pattern" "$file"; then ok "$name"; else not_ok "$name"; fi +} + +assert_matches() { + local file=$1 pattern=$2 name=$3 + if grep -Eq "$pattern" "$file"; then ok "$name"; else not_ok "$name"; fi +} + +command -v docker >/dev/null 2>&1 || { echo 'Docker is required' >&2; exit 2; } +docker info >/dev/null 2>&1 || { echo 'Docker daemon is unavailable' >&2; exit 2; } +mkdir -p "$OUTPUT_ROOT/success" "$OUTPUT_ROOT/failure" "$OUTPUT_ROOT/concurrent" + +cat >"$TMP/packages.list" <<'EOF' +tree +nano +cronie +openssh-server +strace +EOF +printf 'nano\n' >"$TMP/desktop.list" +printf 'openssh-server\n' >"$TMP/service.list" + +if bash "$MAIN" -j 2 --service-mode systemd --test-image "$IMAGE" --keep-image \ + --output-dir "$OUTPUT_ROOT/success" "$TMP/packages.list" \ + "$TMP/desktop.list" "$TMP/service.list" >"$OUTPUT_ROOT/success/run.log" 2>&1; then + ok 'OpenCloudOS packages complete install/command/service/remove lifecycle' +else + not_ok 'OpenCloudOS packages complete install/command/service/remove lifecycle' +fi + +success_report=$(find "$OUTPUT_ROOT/success" -maxdepth 1 \ + -name 'package-command-result-docker-*' | head -n 1) +if [ -n "$success_report" ]; then + assert_contains "$success_report" '待测包总数: 5' 'five real packages are accounted' + assert_contains "$success_report" '对账:' 'real package report contains reconciliation' + assert_matches "$success_report" $'^strace.*\tstrace.*\tPASS\tPASS\tPASS\tPASS$' \ + 'preinstalled image package completes the lifecycle' +else + not_ok 'real package report exists' +fi + +service_detail=$(find "$OUTPUT_ROOT/success" -path '*/packages/*cronie*/output/service-results.tsv' | head -n 1) +if [ -n "$service_detail" ]; then + assert_contains "$service_detail" $'PASS\tstart=0 is-active=0 stop=0' \ + 'cronie service records start/is-active/stop separately' +else + not_ok 'cronie service detail exists' +fi + +printf 'yum-ci-package-that-does-not-exist-ijv5j9\n' >"$TMP/missing.list" +: >"$TMP/empty.list" +if bash "$MAIN" -j 1 --skip-build --test-image "$IMAGE" --service-mode skip \ + --output-dir "$OUTPUT_ROOT/failure" "$TMP/missing.list" \ + "$TMP/empty.list" "$TMP/empty.list" >"$OUTPUT_ROOT/failure/run.log" 2>&1; then + not_ok 'missing package returns failure' +else + rc=$? + if [ "$rc" -eq 1 ]; then ok 'missing package returns exit code 1'; else not_ok "missing package exit code is 1 (got $rc)"; fi +fi +failure_report=$(find "$OUTPUT_ROOT/failure" -maxdepth 1 \ + -name 'package-command-result-docker-*' | head -n 1) +assert_contains "$failure_report" '【安装失败】(1)' 'real install failure is classified' + +printf 'tree\n' >"$TMP/one.list" +bash "$MAIN" -j 1 --skip-build --test-image "$IMAGE" --service-mode skip \ + --output-dir "$OUTPUT_ROOT/concurrent" "$TMP/one.list" \ + "$TMP/empty.list" "$TMP/empty.list" >"$OUTPUT_ROOT/concurrent/a.log" 2>&1 & +pid_a=$! +bash "$MAIN" -j 1 --skip-build --test-image "$IMAGE" --service-mode skip \ + --output-dir "$OUTPUT_ROOT/concurrent" "$TMP/one.list" \ + "$TMP/empty.list" "$TMP/empty.list" >"$OUTPUT_ROOT/concurrent/b.log" 2>&1 & +pid_b=$! +wait "$pid_a"; rc_a=$? +wait "$pid_b"; rc_b=$? +if [ "$rc_a:$rc_b" = 0:0 ]; then ok 'two real concurrent runs both complete'; else not_ok "two real concurrent runs complete ($rc_a:$rc_b)"; fi +report_count=$(find "$OUTPUT_ROOT/concurrent" -maxdepth 1 \ + -name 'package-command-result-docker-*' | wc -l) +if [ "$report_count" -eq 2 ]; then ok 'concurrent reports are isolated'; else not_ok "concurrent reports are isolated (got $report_count)"; fi + +residual=0 +while IFS= read -r manifest; do + run_id=$(awk -F= '$1 == "run_id" { print $2 }' "$manifest") + [ -n "$run_id" ] || continue + count=$(docker ps -aq --filter "label=io.opencloudos.yum-ci.run=$run_id" | wc -l) + residual=$((residual + count)) +done < <(find "$OUTPUT_ROOT" -name run-manifest.txt) +if [ "$residual" -eq 0 ]; then ok 'no test containers remain'; else not_ok "no test containers remain (got $residual)"; fi + +if docker image rm "$IMAGE" >/dev/null 2>&1 && + ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + ok 'derived test image is removed' +else + not_ok 'derived test image is removed' +fi +IMAGE="" + +mkdir -p "$OUTPUT_ROOT/auto-cleanup" +if bash "$MAIN" -j 1 --service-mode skip --test-image "$AUTO_IMAGE" \ + --output-dir "$OUTPUT_ROOT/auto-cleanup" "$TMP/one.list" \ + "$TMP/empty.list" "$TMP/empty.list" >"$OUTPUT_ROOT/auto-cleanup/run.log" 2>&1; then + ok 'default image-cleanup run completes' +else + not_ok 'default image-cleanup run completes' +fi +if ! docker image inspect "$AUTO_IMAGE" >/dev/null 2>&1; then + ok 'default run removes its derived image' +else + not_ok 'default run removes its derived image' +fi +AUTO_IMAGE="" + +dangling=$(docker image ls -q --filter dangling=true \ + --filter label=io.opencloudos.yum-ci.issue=IJV5J9 | wc -l) +if [ "$dangling" -eq 0 ]; then ok 'no labeled dangling image remains'; else not_ok "no labeled dangling image remains (got $dangling)"; fi + +printf '\nE2E tests: %s passed, %s failed\nArtifacts: %s\n' "$PASS" "$FAIL" "$OUTPUT_ROOT" +[ "$FAIL" -eq 0 ] diff --git a/testcase/tests/test-command-execute-docker.sh b/testcase/tests/test-command-execute-docker.sh new file mode 100755 index 0000000..86570a9 --- /dev/null +++ b/testcase/tests/test-command-execute-docker.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# Host-only regression tests for Docker scheduler, accounting, and reports. + +set -uo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +CASE_DIR=$(cd -- "$SCRIPT_DIR/.." && pwd -P) +MAIN="$CASE_DIR/command-execute-docker.sh" +FAKE="$SCRIPT_DIR/fake-docker-package-driver.sh" +TMP=$(mktemp -d) +PASS=0 +FAIL=0 + +cleanup() { + rm -rf -- "$TMP" +} +trap cleanup EXIT + +ok() { + PASS=$((PASS + 1)) + printf 'PASS: %s\n' "$1" +} + +not_ok() { + FAIL=$((FAIL + 1)) + printf 'FAIL: %s\n' "$1" >&2 +} + +assert_contains() { + local file=$1 pattern=$2 name=$3 + if grep -Fq "$pattern" "$file"; then ok "$name"; else not_ok "$name"; fi +} + +assert_eq() { + local actual=$1 expected=$2 name=$3 + if [ "$actual" = "$expected" ]; then + ok "$name" + else + not_ok "$name (expected=$expected actual=$actual)" + fi +} + +run_main() { + YUM_CI_DOCKER_DRIVER="$FAKE" YUM_CI_DOCKER_SKIP_PREFLIGHT=1 \ + bash "$MAIN" "$@" +} + +mkdir -p "$TMP/inputs" "$TMP/out" +: >"$TMP/inputs/empty.list" +printf '# comment\nalpha extra-column\r\n\nbeta\n gamma trailing-column\n' >"$TMP/inputs/normalize.list" + +validate_output=$(bash "$MAIN" --validate-only -j 3 --cpus 0.5 --memory 256m \ + "$TMP/inputs/normalize.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list") +assert_contains <(printf '%s\n' "$validate_output") 'packages=3 jobs=3' 'three positional lists parse and normalize' +assert_contains <(printf '%s\n' "$validate_output") 'alpha' 'first field is used as package token' +assert_contains <(printf '%s\n' "$validate_output") 'gamma' 'leading whitespace is accepted' + +if bash "$MAIN" --validate-only -j 0 "$TMP/inputs/normalize.list" \ + "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1; then + not_ok 'invalid zero concurrency is rejected' +else + assert_eq "$?" 2 'invalid zero concurrency returns infrastructure/usage code' +fi + +mkdir -p "$TMP/out-infra" +printf 'infra-one\n' >"$TMP/inputs/infra.list" +if run_main -j 1 --service-mode skip --output-dir "$TMP/out-infra" \ + "$TMP/inputs/infra.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" \ + >/dev/null 2>&1; then + not_ok 'Docker infrastructure failure is rejected' +else + assert_eq "$?" 2 'Docker infrastructure failure returns exit code 2' +fi + +cat >"$TMP/inputs/mixed.list" <<'EOF' +ok-one +desktop-one +special-one +multi-fail-one +service-fail-one +service-mixed-one +template-unit-one +EOF +printf 'desktop-one\n' >"$TMP/inputs/desktop.list" +printf 'special-one\n' >"$TMP/inputs/service.list" + +if run_main -j 3 --service-mode skip --output-dir "$TMP/out" \ + "$TMP/inputs/mixed.list" "$TMP/inputs/desktop.list" "$TMP/inputs/service.list" \ + >"$TMP/mixed.stdout" 2>"$TMP/mixed.stderr"; then + not_ok 'mixed failures make the command return nonzero' +else + assert_eq "$?" 1 'package failures return exit code 1' +fi + +mixed_report=$(find "$TMP/out" -maxdepth 1 -name 'package-command-result-docker-*' | head -n 1) +assert_contains "$mixed_report" '待测包总数: 7 | 通过: 1 | 失败: 3 | 跳过: 3' 'pass/fail/skip accounting balances' +assert_contains "$mixed_report" 'multi-fail-one 原因: injected command failure' 'command failure is classified' +assert_contains "$mixed_report" 'multi-fail-one 原因: injected remove failure' 'same package appears in every failed stage' +assert_contains "$mixed_report" 'service-fail-one 原因: injected service failure' 'service failure is classified' +assert_contains "$mixed_report" 'service-mixed-one 原因: demo@.service: template unit requires an instance name' \ + 'unit skip remains visible when another unit fails' +assert_contains "$mixed_report" 'template-unit-one' 'template unit is explicitly skipped' +assert_contains "$mixed_report" '对账: 1 + 3 + 3 = 7' 'summary reconciliation is exact' + +rm -rf "$TMP/out-timeout" +mkdir -p "$TMP/out-timeout" +printf 'timeout-one\n' >"$TMP/inputs/timeout.list" +if FAKE_TIMEOUT_SECONDS=3 run_main -j 1 --package-timeout 1 --service-mode skip \ + --output-dir "$TMP/out-timeout" "$TMP/inputs/timeout.list" \ + "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1; then + not_ok 'lifecycle timeout is a failure' +else + assert_eq "$?" 1 'lifecycle timeout returns package failure code' +fi +timeout_report=$(find "$TMP/out-timeout" -maxdepth 1 -name 'package-command-result-docker-*' | head -n 1) +assert_contains "$timeout_report" 'container lifecycle timed out after 1s' 'timeout reason is retained' +assert_contains "$timeout_report" '【安装失败】(1)' 'missing pre-stage result is classified as install failure' + +mkdir -p "$TMP/out-command-timeout" +printf 'timeout-command-one\n' >"$TMP/inputs/command-timeout.list" +if FAKE_TIMEOUT_SECONDS=3 run_main -j 1 --package-timeout 1 --service-mode skip \ + --output-dir "$TMP/out-command-timeout" "$TMP/inputs/command-timeout.list" \ + "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1; then + not_ok 'command-stage lifecycle timeout is a failure' +else + assert_eq "$?" 1 'command-stage timeout returns package failure code' +fi +command_timeout_report=$(find "$TMP/out-command-timeout" -maxdepth 1 \ + -name 'package-command-result-docker-*' | head -n 1) +assert_contains "$command_timeout_report" '【安装失败】(0)' 'command timeout is not misclassified as install failure' +assert_contains "$command_timeout_report" '【命令测试失败】(1)' 'command timeout is classified by current stage' +assert_contains "$command_timeout_report" 'during command test' 'command timeout keeps stage-specific reason' + +printf 'ok-only\n' >"$TMP/inputs/pass.list" +if run_main -j 1 --service-mode skip --output-dir "$TMP/out-pass" \ + "$TMP/inputs/pass.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1; then + ok 'all-pass run returns zero' +else + not_ok 'all-pass run returns zero' +fi + +cat >"$TMP/inputs/sleep.list" <<'EOF' +sleep-a +sleep-b +sleep-c +sleep-d +EOF +start=$(date +%s%3N) +FAKE_SLEEP_SECONDS=0.4 run_main -j 1 --service-mode skip --output-dir "$TMP/out-j1" \ + "$TMP/inputs/sleep.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1 +j1_ms=$(( $(date +%s%3N) - start )) +start=$(date +%s%3N) +FAKE_SLEEP_SECONDS=0.4 run_main -j 4 --service-mode skip --output-dir "$TMP/out-j4" \ + "$TMP/inputs/sleep.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1 +j4_ms=$(( $(date +%s%3N) - start )) +if [ "$j4_ms" -lt "$j1_ms" ]; then + ok "parallel scheduler improves elapsed time (j1=${j1_ms}ms j4=${j4_ms}ms)" +else + not_ok "parallel scheduler improves elapsed time (j1=${j1_ms}ms j4=${j4_ms}ms)" +fi + +mkdir -p "$TMP/out-concurrent" +FAKE_SLEEP_SECONDS=0.4 run_main -j 1 --service-mode skip --output-dir "$TMP/out-concurrent" \ + "$TMP/inputs/sleep.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1 & +pid_a=$! +FAKE_SLEEP_SECONDS=0.4 run_main -j 1 --service-mode skip --output-dir "$TMP/out-concurrent" \ + "$TMP/inputs/sleep.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1 & +pid_b=$! +wait "$pid_a"; rc_a=$? +wait "$pid_b"; rc_b=$? +report_count=$(find "$TMP/out-concurrent" -maxdepth 1 -name 'package-command-result-docker-*' | wc -l) +assert_eq "$rc_a:$rc_b" '0:0' 'two concurrent runs both complete' +assert_eq "$report_count" 2 'concurrent runs keep distinct reports' + +printf '\nTests: %s passed, %s failed\n' "$PASS" "$FAIL" +[ "$FAIL" -eq 0 ] diff --git a/testcase/tests/test-docker-package-driver.sh b/testcase/tests/test-docker-package-driver.sh new file mode 100755 index 0000000..a25d65d --- /dev/null +++ b/testcase/tests/test-docker-package-driver.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Host-only regression test for driver mount layout and result publication. + +set -uo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +CASE_DIR=$(cd -- "$SCRIPT_DIR/.." && pwd -P) +DRIVER="$CASE_DIR/lib/docker-package-driver.sh" +WORKER="$CASE_DIR/lib/docker-package-worker.sh" +TMP=$(mktemp -d) +trap 'rm -rf -- "$TMP"' EXIT + +mkdir -p "$TMP/bin" "$TMP/package" "$TMP/results" +: >"$TMP/desktop.list" +: >"$TMP/service.list" + +cat >"$TMP/bin/docker" <<'EOF' +#!/bin/bash +printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG" +case "$1" in + run | rm) exit 0 ;; + exec) + mkdir -p "$FAKE_PACKAGE_DIR/output" + printf '1\talpha\talpha-src\talpha-1.0-1.x86_64\tPASS\tinstalled\tPASS\tcommand passed\tPASS\tno units\tPASS\tremoved\t25\t1\t0\tcomplete\t1\n' \ + >"$FAKE_PACKAGE_DIR/output/result.tsv" + exit 0 + ;; + *) exit 0 ;; +esac +EOF +chmod +x "$TMP/bin/docker" + +export PATH="$TMP/bin:$PATH" +export FAKE_DOCKER_LOG="$TMP/docker.log" +export FAKE_PACKAGE_DIR="$TMP/package" +export YUM_CI_DOCKER_WORKER="$WORKER" +export RUN_ID=test-run +export TEST_IMAGE=test-image +export EFFECTIVE_SERVICE_MODE=skip +export SERVICE_MODE_REASON='test mode' +export CPU_LIMIT=1 +export MEMORY_LIMIT=256m +export PIDS_LIMIT=64 +export REPO_DIR= + +bash "$DRIVER" 1 alpha "$TMP/package" "$TMP/results/00001.tsv" \ + "$TMP/desktop.list" "$TMP/service.list" + +cmp "$TMP/package/output/result.tsv" "$TMP/results/00001.tsv" +grep -Fq -- "--volume $TMP/package:/yum-ci:Z" "$TMP/docker.log" +[ "$(grep -o -- '--cap-add SYS_PTRACE' "$TMP/docker.log" | wc -l)" -eq 1 ] +[ "$(grep -o -- '--volume' "$TMP/docker.log" | wc -l)" -eq 1 ] +[ -f "$TMP/package/worker.sh" ] +[ -f "$TMP/package/desktop.list" ] +[ -f "$TMP/package/service.list" ] + +mkdir -p "$TMP/package-copy-fail" +if bash "$DRIVER" 2 beta "$TMP/package-copy-fail" "$TMP/results/00002.tsv" \ + "$TMP/missing-desktop.list" "$TMP/service.list" >/dev/null 2>&1; then + exit 1 +else + [ "$?" -eq 2 ] +fi +grep -Fq 'failed to stage worker and input lists' "$TMP/results/00002.tsv" + +mkdir -p "$TMP/package-repo" "$TMP/repo" +printf '[private-test]\nname=private-test\n' >"$TMP/repo/private.repo" +FAKE_PACKAGE_DIR="$TMP/package-repo" REPO_DIR="$TMP/repo" \ + bash "$DRIVER" 3 gamma "$TMP/package-repo" "$TMP/results/00003.tsv" \ + "$TMP/desktop.list" "$TMP/service.list" +[ ! -e "$TMP/package-repo/repos" ] + +echo 'PASS: driver publishes atomic worker result through one SELinux-safe bind mount' -- Gitee From 299d099ec1673034e095f1942bfe4423c0ee7d0b Mon Sep 17 00:00:00 2001 From: silasyyyang Date: Fri, 31 Jul 2026 14:41:12 +0800 Subject: [PATCH 2/2] fix: harden Docker package test lifecycle (IJV5J9) --- .dockerignore | 9 + testcase/Makefile | 2 + testcase/README-command-execute-docker.md | 15 +- testcase/VALIDATION-command-execute-docker.md | 10 +- testcase/benchmark-command-execute-docker.sh | 19 +- testcase/command-execute-docker.sh | 202 +++++++++++++----- testcase/lib/docker-package-driver.sh | 49 ++++- testcase/lib/docker-package-worker.sh | 74 ++++++- testcase/tests/fake-docker-package-driver.sh | 40 ++++ .../test-benchmark-command-execute-docker.sh | 47 ++++ testcase/tests/test-command-execute-docker.sh | 86 ++++++++ testcase/tests/test-docker-package-driver.sh | 43 +++- testcase/tests/test-docker-package-worker.sh | 70 ++++++ 13 files changed, 582 insertions(+), 84 deletions(-) create mode 100644 .dockerignore create mode 100755 testcase/tests/test-benchmark-command-execute-docker.sh create mode 100755 testcase/tests/test-docker-package-worker.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ef713a3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +** +!tst_lib/ +!tst_lib/ts_common.sh +!tst_lib/other_common.sh +!testcase/ +!testcase/docker/ +!testcase/docker/Dockerfile +!testcase/lib/ +!testcase/lib/docker-ts-common-runner.sh diff --git a/testcase/Makefile b/testcase/Makefile index 6839275..7007680 100644 --- a/testcase/Makefile +++ b/testcase/Makefile @@ -25,6 +25,8 @@ cleanall: clean docker-test: bash tests/test-command-execute-docker.sh bash tests/test-docker-package-driver.sh + bash tests/test-docker-package-worker.sh + bash tests/test-benchmark-command-execute-docker.sh docker-e2e: bash tests/test-command-execute-docker-e2e.sh diff --git a/testcase/README-command-execute-docker.md b/testcase/README-command-execute-docker.md index 3dab5a8..1f222e8 100644 --- a/testcase/README-command-execute-docker.md +++ b/testcase/README-command-execute-docker.md @@ -49,7 +49,7 @@ cd testcase ## 常用参数 ```text --j, --jobs N 最大并发容器数 +-j, --jobs N 最大并发容器数（上限 64，且不超过待测包数） --package-timeout SEC 单包完整生命周期超时 --stage-timeout SEC 安装、卸载阶段超时 --command-timeout SEC 每条命令探测超时 @@ -83,7 +83,7 @@ cd testcase `--service-mode skip` 明确禁用服务启动测试。 systemd 模式使用官方 init 镜像和 `--privileged --cgroupns=private`，不会把 -宿主 `/sys/fs/cgroup` 读写挂载进容器。每个非模板 unit 都独立记录： +宿主 `/sys/fs/cgroup` 读写挂载进容器。每个可安全手动操作的 system unit 都独立记录： - `systemctl start`； - `systemctl is-active`； @@ -92,6 +92,11 @@ systemd 模式使用官方 init 镜像和 `--privileged --cgroupns=private`， 模板 unit（如 `foo@.service`）和 `RefuseManualStart=yes` 或 `RefuseManualStop=yes` 的 unit 明确 记为 `SKIP`，不会误报为通过或失败。 +user unit 会被发现并以“缺少 per-user manager/session”为由跳过； +`poweroff.target`、`reboot.target` 等关机/重启 unit 永远不会启动。 + +若待测包提供 worker 完成结果写入所必需的 `bash`、`rpm`、`sed`、coreutils +命令等，卸载阶段会明确记为 `SKIP`，避免破坏 worker 后生成不可信结果。 > 安全提示：RPM scriptlet、包内命令和服务会被实际执行。systemd 模式又需 > 特权容器，因此应在一次性、隔离的 CI 节点上运行，不要在多租户生产宿主机 @@ -128,6 +133,8 @@ systemd 模式使用官方 init 镜像和 `--privileged --cgroupns=private`， 总报告保证 `通过 + 失败 + 跳过 = 待测总数`。阶段失败会分别进入安装、 命令、服务、卸载清单；同一包多个阶段失败时会出现在多个清单中。即使包 整体失败，其中发生的阶段跳过也会保留在“跳过项”。 +Worker 结果必须严格满足 17 列 TSV、包序号/请求身份、阶段状态和完成标记校验； +缺失、截断或身份不符的结果会改记为基础设施失败（退出码 `2`），不会计为通过。 退出码： @@ -146,6 +153,7 @@ systemd 模式使用官方 init 镜像和 `--privileged --cgroupns=private`， - 不执行 `docker system prune`，不会删除其他任务资源。 - 派生镜像标签包含基础镜像、Dockerfile 摘要和 run ID，默认在本轮完成后删除； 调试或复用时可显式传入 `--keep-image`。 +- 删除派生镜像不等于清除 Docker/BuildKit 的全局构建缓存；本用例不会操作该全局缓存。 ## 自动测试 @@ -153,6 +161,9 @@ systemd 模式使用官方 init 镜像和 `--privileged --cgroupns=private`， ```bash bash testcase/tests/test-command-execute-docker.sh +bash testcase/tests/test-docker-package-driver.sh +bash testcase/tests/test-docker-package-worker.sh +bash testcase/tests/test-benchmark-command-execute-docker.sh ``` 真实 OpenCloudOS 9 集成验证需 Docker： diff --git a/testcase/VALIDATION-command-execute-docker.md b/testcase/VALIDATION-command-execute-docker.md index 79cd282..7262f71 100644 --- a/testcase/VALIDATION-command-execute-docker.md +++ b/testcase/VALIDATION-command-execute-docker.md @@ -1,6 +1,6 @@ # IJV5J9 实机验证记录 -验证日期：2026-07-30。 +实机验证日期：2026-07-30；审查修复回归日期：2026-07-31。 ## 环境 @@ -16,8 +16,9 @@ - 所有新增 shell 脚本通过 `bash -n`。 - 所有新增 shell 脚本通过 ShellCheck `--severity=warning`。 -- 调度、超时、分类、对账、基础设施错误码及并发隔离测试：24/24 通过。 -- Driver 原子结果发布、bind mount、repo 临时副本清理和 `SYS_PTRACE` 参数测试通过。 +- 调度、超时、严格结果校验、分类、对账、清理错误码及并发隔离测试：32/32 通过。 +- Driver 的原子结果发布、Docker 控制命令硬超时、bind mount、repo 临时副本清理和 + `SYS_PTRACE` 参数测试通过；Worker unit 安全策略与 benchmark 错误码回归测试通过。 - `git diff --check` 通过。 ## 真实 Docker E2E @@ -37,7 +38,8 @@ bash testcase/tests/test-command-execute-docker-e2e.sh - 不存在的软件包返回退出码 1，并准确归入“安装失败”。 - 两轮同时测试同一软件包均返回 0，生成两个独立报告。 - 测试完成后，带本用例 run Label 的容器数为 0，Docker volume 数为 0； - 派生镜像和带本题 Label 的 dangling image 数也均为 0。 + 派生镜像和带本题 Label 的 dangling image 数也均为 0。该检查不代表、也不操作 + Docker/BuildKit 的全局构建缓存。 ## 并发基准 diff --git a/testcase/benchmark-command-execute-docker.sh b/testcase/benchmark-command-execute-docker.sh index 3094b45..1f64de4 100755 --- a/testcase/benchmark-command-execute-docker.sh +++ b/testcase/benchmark-command-execute-docker.sh @@ -4,16 +4,24 @@ set -uo pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) -MAIN="$SCRIPT_DIR/command-execute-docker.sh" +MAIN=${YUM_CI_DOCKER_MAIN:-$SCRIPT_DIR/command-execute-docker.sh} JOBS_CSV=${JOBS_CSV:-1,2,4,8} SERVICE_MODE=${BENCHMARK_SERVICE_MODE:-skip} IMAGE=${BENCHMARK_IMAGE:-yum-ci-pkg-test:ijv5j9-benchmark-$$} BASE_IMAGE=${BENCHMARK_BASE_IMAGE:-opencloudos/opencloudos9-init:latest} OUTPUT_ROOT=${BENCHMARK_OUTPUT_DIR:-./docker-benchmark-$(date '+%Y%m%d-%H%M%S')} IMAGE_BUILT=0 +OVERALL_RC=0 cleanup() { - [ "$IMAGE_BUILT" -eq 0 ] || docker image rm "$IMAGE" >/dev/null 2>&1 || true + local rc=$? + trap - EXIT + if [ "$IMAGE_BUILT" -ne 0 ] && + ! timeout -k 5 60 docker image rm "$IMAGE" >/dev/null 2>&1; then + echo "failed to remove benchmark image: $IMAGE" >&2 + rc=2 + fi + exit "$rc" } trap cleanup EXIT @@ -54,6 +62,13 @@ for jobs in "${levels[@]}"; do rc=$? elapsed=$(( $(date +%s%3N) - start )) printf '%s\t%s\t%s\n' "$jobs" "$elapsed" "$rc" | tee -a "$summary" + case "$rc" in + 0) ;; + 1) [ "$OVERALL_RC" -ne 0 ] || OVERALL_RC=1 ;; + 2) OVERALL_RC=2 ;; + *) OVERALL_RC=2 ;; + esac done echo "benchmark summary: $summary" +exit "$OVERALL_RC" diff --git a/testcase/command-execute-docker.sh b/testcase/command-execute-docker.sh index f28b750..226baae 100755 --- a/testcase/command-execute-docker.sh +++ b/testcase/command-execute-docker.sh @@ -17,6 +17,7 @@ DRIVER_DEFAULT="$SCRIPT_DIR/lib/docker-package-driver.sh" DOCKERFILE_DEFAULT="$SCRIPT_DIR/docker/Dockerfile" JOBS="" +MAX_JOBS=64 PACKAGE_TIMEOUT=1200 STAGE_TIMEOUT=300 COMMAND_TIMEOUT=5 @@ -67,7 +68,7 @@ Required inputs: SERVICE_LIST packages requiring special service configuration Options: - -j, --jobs N concurrent containers (default: min(CPU, 8)) + -j, --jobs N concurrent containers (default: min(CPU, 8), max: 64) --package-timeout SEC full lifecycle timeout per package (default: 1200) --stage-timeout SEC install/remove timeout (default: 300) --command-timeout SEC timeout for each command probe (default: 5) @@ -133,6 +134,20 @@ require_command() { command -v "$1" >/dev/null 2>&1 || die "required command is missing: $1" } +docker_with_timeout() { + local seconds=$1 + shift + timeout -k 5 "$seconds" docker "$@" +} + +docker_before_deadline() { + local deadline=$1 remaining + shift + remaining=$((deadline - SECONDS)) + [ "$remaining" -gt 0 ] || return 124 + docker_with_timeout "$remaining" "$@" +} + normalize_list() { local input=$1 output=$2 kind=$3 line value line_no=0 : >"$output" @@ -160,13 +175,24 @@ safe_remove_tree() { } cleanup_run_containers() { - local container + local container containers rc=0 deadline [ -n "$RUN_ID" ] || return 0 + [ "${YUM_CI_DOCKER_SKIP_PREFLIGHT:-0}" != 1 ] || return 0 command -v docker >/dev/null 2>&1 || return 0 + deadline=$((SECONDS + STARTUP_TIMEOUT)) + containers=$(docker_before_deadline "$deadline" ps -aq \ + --filter "label=io.opencloudos.yum-ci.run=$RUN_ID" 2>/dev/null) || { + log "failed to list containers for run $RUN_ID during cleanup" + return 1 + } while IFS= read -r container || [ -n "$container" ]; do [ -n "$container" ] || continue - docker rm -f "$container" >/dev/null 2>&1 || true - done < <(docker ps -aq --filter "label=io.opencloudos.yum-ci.run=$RUN_ID" 2>/dev/null || true) + docker_before_deadline "$deadline" rm -f "$container" >/dev/null 2>&1 || { + log "failed to remove container $container during cleanup" + rc=1 + } + done <<<"$containers" + return "$rc" } stop_children() { @@ -197,9 +223,12 @@ cleanup() { stop_children rc=2 fi - cleanup_run_containers + cleanup_run_containers || rc=2 if [ "$REMOVE_IMAGE" -eq 1 ] && [ "$IMAGE_BUILT" -eq 1 ]; then - docker image rm "$TEST_IMAGE" >/dev/null 2>&1 || true + docker_with_timeout "$STARTUP_TIMEOUT" image rm "$TEST_IMAGE" >/dev/null 2>&1 || { + log "failed to remove derived image $TEST_IMAGE" + rc=2 + } fi if [ -n "$WORK_ROOT" ] && [ -d "$WORK_ROOT" ] && [ "$KEEP_WORK" -eq 0 ]; then safe_remove_tree "$WORK_ROOT" || rc=2 @@ -267,20 +296,35 @@ resolve_test_image_tag() { } probe_systemd() { - local name="yumci-probe-${RUN_ID}" state="" attempt - if ! docker run -d --rm --privileged --cgroupns=private \ + local name="yumci-probe-${RUN_ID}" state="" attempt rc=0 + local deadline=$((SECONDS + STARTUP_TIMEOUT)) + docker_before_deadline "$deadline" run -d --rm --privileged --cgroupns=private \ --name "$name" \ --label "io.opencloudos.yum-ci.run=$RUN_ID" \ --entrypoint /sbin/init \ - "$TEST_IMAGE" >/dev/null 2>&1; then + "$TEST_IMAGE" >/dev/null 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + if [ "$rc" -eq 124 ]; then + SERVICE_MODE_REASON="systemd probe container start timed out after ${STARTUP_TIMEOUT}s" + return 2 + fi SERVICE_MODE_REASON="privileged init container failed to start" return 1 fi for ((attempt = 0; attempt < STARTUP_TIMEOUT; attempt++)); do - state=$(docker exec "$name" systemctl is-system-running 2>/dev/null || true) + state=$(docker_before_deadline "$deadline" exec "$name" \ + systemctl is-system-running 2>/dev/null) || rc=$? + if [ "$rc" -eq 124 ]; then + state=timeout + break + fi + rc=0 case "$state" in running | degraded) - docker rm -f "$name" >/dev/null 2>&1 || true + if ! docker_before_deadline "$deadline" rm -f "$name" >/dev/null 2>&1; then + SERVICE_MODE_REASON="systemd probe passed but its container cleanup failed" + return 2 + fi SERVICE_MODE_REASON="systemd probe reached $state" return 0 ;; @@ -288,7 +332,10 @@ probe_systemd() { *) break ;; esac done - docker rm -f "$name" >/dev/null 2>&1 || true + if ! docker_before_deadline "$deadline" rm -f "$name" >/dev/null 2>&1; then + SERVICE_MODE_REASON="systemd probe failed and its container cleanup also failed" + return 2 + fi SERVICE_MODE_REASON="systemd probe did not become ready (state=${state:-unknown})" return 1 } @@ -300,16 +347,19 @@ resolve_service_mode() { SERVICE_MODE_REASON="service mode skip requested" ;; systemd) - probe_systemd || return 1 + probe_systemd || return $? EFFECTIVE_SERVICE_MODE=systemd ;; auto) - if probe_systemd; then - EFFECTIVE_SERVICE_MODE=systemd - else - EFFECTIVE_SERVICE_MODE=skip - log "systemd unavailable; service units will be marked SKIP: $SERVICE_MODE_REASON" - fi + probe_systemd + case $? in + 0) EFFECTIVE_SERVICE_MODE=systemd ;; + 1) + EFFECTIVE_SERVICE_MODE=skip + log "systemd unavailable; service units will be marked SKIP: $SERVICE_MODE_REASON" + ;; + *) return 2 ;; + esac ;; esac } @@ -317,8 +367,11 @@ resolve_service_mode() { write_manifest() { local manifest="$RUN_DIR/run-manifest.txt" image_id os_info docker_info image_id=$(docker image inspect --format '{{.Id}}' "$TEST_IMAGE" 2>/dev/null || printf unknown) - os_info=$(docker run --rm --entrypoint /bin/sh "$TEST_IMAGE" -c \ - '. /etc/os-release; printf "%s %s" "${NAME:-unknown}" "${VERSION_ID:-unknown}"' 2>/dev/null || printf unknown) + os_info=$(docker_with_timeout "$STARTUP_TIMEOUT" run --rm --entrypoint /bin/sh "$TEST_IMAGE" -c \ + '. /etc/os-release; printf "%s %s" "${NAME:-unknown}" "${VERSION_ID:-unknown}"' 2>/dev/null) || { + log "cannot inspect container OS before the startup deadline" + return 1 + } docker_info=$(docker version --format '{{.Server.Version}}' 2>/dev/null || printf unknown) { printf 'issue=IJV5J9\n' @@ -353,7 +406,7 @@ synthesize_result() { local index=$1 request=$2 reason=$3 result_file=$4 tmp reason=$(sanitize_field "$reason") tmp="$result_file.tmp.$$" - printf '%s\t%s\tunknown\t%s\tFAIL\t%s\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\t0\t0\t0\tnone\t1\n' \ + printf '%s\t%s\tunknown\t%s\tFAIL\t%s\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\t0\t0\t0\tcomplete\t1\n' \ "$index" "$request" "$request" "$reason" >"$tmp" mv -f "$tmp" "$result_file" } @@ -371,9 +424,9 @@ finalize_timeout_result() { local index=$1 request=$2 result_file=$3 tmp local r_index r_request source binary install install_reason command command_reason local service service_reason remove remove_reason duration command_count unit_count current complete - if [ ! -s "$result_file" ]; then + if ! valid_result_checkpoint "$result_file" "$index" "$request"; then synthesize_result "$index" "$request" \ - "container lifecycle timed out after ${PACKAGE_TIMEOUT}s during install" "$result_file" + "container lifecycle timed out after ${PACKAGE_TIMEOUT}s without a valid checkpoint" "$result_file" return 0 fi IFS=$'\t' read -r r_index r_request source binary install install_reason command command_reason \ @@ -398,6 +451,7 @@ finalize_timeout_result() { ;; esac complete=1 + current=complete tmp="$result_file.tmp.$$" printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ "$r_index" "$r_request" "$source" "$binary" "$install" "$(sanitize_field "$install_reason")" \ @@ -408,7 +462,7 @@ finalize_timeout_result() { } run_one() { - local index=$1 request=$2 package_dir result_file rc=0 safe_request timed_out=0 + local index=$1 request=$2 package_dir result_file rc=0 safe_request timed_out=0 infra=0 safe_request=$(printf '%s' "$request" | tr -c 'A-Za-z0-9_.-' '_') safe_request=${safe_request:0:80} package_dir="$RUN_DIR/packages/$(printf '%05d' "$index")-$safe_request" @@ -429,10 +483,18 @@ run_one() { elif [ ! -s "$result_file" ]; then synthesize_result "$index" "$request" \ "container driver exited with status $rc without a result" "$result_file" + infra=1 fi + if ! valid_complete_result "$result_file" "$index" "$request"; then + [ ! -s "$result_file" ] || cp "$result_file" "$result_file.invalid" 2>/dev/null || true + synthesize_result "$index" "$request" \ + "invalid container result schema, identity, or completion marker" "$result_file" + infra=1 + fi + [ "$infra" -eq 0 ] || return 2 if [ "$timed_out" -eq 0 ]; then case "$rc" in - 0 | 1 | 124 | 137) ;; + 0 | 1) ;; *) return 2 ;; esac fi @@ -457,8 +519,30 @@ print_category() { done <"$file" } -valid_stage_status() { - case "$1" in PASS | FAIL | SKIP) return 0 ;; *) return 1 ;; esac +valid_result_checkpoint() { + local file=$1 expected_index=$2 expected_request=$3 + [ -s "$file" ] || return 1 + awk -F '\t' -v expected_index="$expected_index" -v expected_request="$expected_request" ' + NR != 1 { bad = 1 } + NR == 1 { + if (NF != 17 || $1 != expected_index || $2 != expected_request || + $3 == "" || $4 == "" || $6 == "" || $8 == "" || $10 == "" || $12 == "" || + $5 !~ /^(PASS|FAIL|SKIP)$/ || $7 !~ /^(PASS|FAIL|SKIP)$/ || + $9 !~ /^(PASS|FAIL|SKIP)$/ || $11 !~ /^(PASS|FAIL|SKIP)$/ || + $13 !~ /^[0-9]+$/ || $14 !~ /^[0-9]+$/ || $15 !~ /^[0-9]+$/ || + $16 !~ /^(bootstrap|install|command|service|remove|complete)$/ || + $17 !~ /^(0|1)$/ || ($17 == "1" && $16 != "complete") || + ($5 == "PASS" && ($3 == "unknown" || $4 == "unknown"))) bad = 1 + } + END { exit !(NR == 1 && !bad) } + ' "$file" +} + +valid_complete_result() { + local file=$1 expected_index=$2 expected_request=$3 + valid_result_checkpoint "$file" "$expected_index" "$expected_request" || return 1 + awk -F '\t' 'NR == 1 && NF == 17 && $16 == "complete" && $17 == "1" { ok = 1 } + END { exit !ok }' "$file" } aggregate_results() { @@ -473,35 +557,24 @@ aggregate_results() { local total=0 pass=0 fail=0 skip=0 malformed=0 local index request source binary install install_reason command command_reason local service service_reason remove remove_reason duration command_count unit_count current complete - local result_file has_fail has_skip safe_request package_dir service_results + local result_file has_fail has_skip safe_request package_dir service_results expected_request local unit unit_status unit_reason : >"$install_fail"; : >"$remove_fail"; : >"$command_fail"; : >"$service_fail"; : >"$skipped" printf '# source-package\tbinary_package\tinstall\tcommand\tservice\tremove\n' >"$report" printf '# index\trequest\tsource-package\tbinary-package\tinstall\tinstall-reason\tcommand\tcommand-reason\tservice\tservice-reason\tremove\tremove-reason\tduration-ms\tcommand-count\tunit-count\tcurrent-stage\tcomplete\n' >"$detail" - while IFS= read -r request || [ -n "$request" ]; do + while IFS= read -r expected_request || [ -n "$expected_request" ]; do total=$((total + 1)) result_file="$RUN_DIR/results/$(printf '%05d' "$total").tsv" - [ -s "$result_file" ] || synthesize_result "$total" "$request" "missing container result" "$result_file" - IFS=$'\t' read -r index request source binary install install_reason command command_reason \ - service service_reason remove remove_reason duration command_count unit_count current complete <"$result_file" - - if ! valid_stage_status "${install:-}" || ! valid_stage_status "${command:-}" || - ! valid_stage_status "${service:-}" || ! valid_stage_status "${remove:-}"; then - synthesize_result "$total" "$request" "malformed container result" "$result_file" - IFS=$'\t' read -r index request source binary install install_reason command command_reason \ - service service_reason remove remove_reason duration command_count unit_count current complete <"$result_file" + if ! valid_complete_result "$result_file" "$total" "$expected_request"; then + [ ! -s "$result_file" ] || cp "$result_file" "$result_file.invalid" 2>/dev/null || true + synthesize_result "$total" "$expected_request" \ + "missing or invalid container result" "$result_file" malformed=$((malformed + 1)) - elif [ "${complete:-0}" != 1 ]; then - case "${current:-unknown}" in - install) install=FAIL; install_reason="container terminated during install" ;; - command) command=FAIL; command_reason="container terminated during command test" ;; - service) service=FAIL; service_reason="container terminated during service test" ;; - remove) remove=FAIL; remove_reason="container terminated during remove" ;; - *) install=FAIL; install_reason="container terminated at an unknown stage" ;; - esac fi + IFS=$'\t' read -r index request source binary install install_reason command command_reason \ + service service_reason remove remove_reason duration command_count unit_count current complete <"$result_file" printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ "$source" "$binary" "$install" "$command" "$service" "$remove" >>"$report" @@ -515,17 +588,15 @@ aggregate_results() { [ "$service" != FAIL ] || append_category "$service_fail" "$binary" "$service_reason" [ "$remove" != FAIL ] || append_category "$remove_fail" "$binary" "$remove_reason" - if [ "$service" = FAIL ]; then - safe_request=$(printf '%s' "$request" | tr -c 'A-Za-z0-9_.-' '_') - safe_request=${safe_request:0:80} - package_dir="$RUN_DIR/packages/$(printf '%05d' "$total")-$safe_request" - service_results="$package_dir/output/service-results.tsv" - if [ -s "$service_results" ]; then - while IFS=$'\t' read -r unit unit_status unit_reason || [ -n "$unit" ]; do - [ "$unit_status" != SKIP ] || append_category "$skipped" "$binary" \ - "$unit: $unit_reason" - done <"$service_results" - fi + safe_request=$(printf '%s' "$request" | tr -c 'A-Za-z0-9_.-' '_') + safe_request=${safe_request:0:80} + package_dir="$RUN_DIR/packages/$(printf '%05d' "$total")-$safe_request" + service_results="$package_dir/output/service-results.tsv" + if [ -s "$service_results" ]; then + while IFS=$'\t' read -r unit unit_status unit_reason || [ -n "$unit" ]; do + [ "$unit_status" != SKIP ] || append_category "$skipped" "$binary" \ + "$unit: $unit_reason" + done <"$service_results" fi has_fail=0; has_skip=0 @@ -611,6 +682,7 @@ validate_options() { "$SERVICE_TIMEOUT" "$STARTUP_TIMEOUT" "$PIDS_LIMIT"; do is_positive_integer "$value" || die "integer options must be positive" done + [ "$JOBS" -le "$MAX_JOBS" ] || die "--jobs must not exceed $MAX_JOBS" is_positive_decimal "$CPU_LIMIT" || die "--cpus must be a positive number" is_memory_limit "$MEMORY_LIMIT" || die "--memory must be a positive Docker memory value" case "$SERVICE_MODE" in auto | systemd | skip) ;; *) die "service mode must be auto, systemd, or skip" ;; esac @@ -621,6 +693,15 @@ validate_options() { [ -z "$REPO_DIR" ] || [ -d "$REPO_DIR" ] || die "repo directory not found: $REPO_DIR" } +adjust_jobs_to_package_count() { + local package_count + package_count=$(wc -l <"$NORMALIZED_PACKAGES") + if [ "$JOBS" -gt "$package_count" ]; then + log "reducing concurrency from $JOBS to package count $package_count" + JOBS=$package_count + fi +} + main() { local stamp token_fifo index=0 request pid rc=0 worker_rc parse_args "$@" @@ -637,6 +718,7 @@ main() { normalize_list "$DESKTOP_INPUT" "$NORMALIZED_DESKTOP" "desktop list" normalize_list "$SERVICE_INPUT" "$NORMALIZED_SERVICES" "service list" [ -s "$NORMALIZED_PACKAGES" ] || die "package list is empty after normalization" + adjust_jobs_to_package_count if [ "$VALIDATE_ONLY" -eq 1 ]; then printf 'packages=%s jobs=%s service_mode=%s cpus=%s memory=%s pids_limit=%s\n' \ @@ -662,6 +744,7 @@ main() { normalize_list "$PKG_INPUT" "$NORMALIZED_PACKAGES" "package list" normalize_list "$DESKTOP_INPUT" "$NORMALIZED_DESKTOP" "desktop list" normalize_list "$SERVICE_INPUT" "$NORMALIZED_SERVICES" "service list" + adjust_jobs_to_package_count stamp=$(date '+%Y%m%d-%H%M%S') RUN_ID="${stamp}-$$-${RANDOM:-0}" @@ -683,7 +766,8 @@ main() { export RUN_ID TEST_IMAGE EFFECTIVE_SERVICE_MODE SERVICE_MODE_REASON export STAGE_TIMEOUT COMMAND_TIMEOUT SERVICE_TIMEOUT STARTUP_TIMEOUT export CPU_LIMIT MEMORY_LIMIT PIDS_LIMIT REPO_DIR PACKAGE_TIMEOUT - [ "${YUM_CI_DOCKER_SKIP_PREFLIGHT:-0}" = 1 ] || write_manifest + [ "${YUM_CI_DOCKER_SKIP_PREFLIGHT:-0}" = 1 ] || + write_manifest || die "failed to write a complete run manifest" token_fifo="$WORK_ROOT/tokens" mkfifo "$token_fifo" diff --git a/testcase/lib/docker-package-driver.sh b/testcase/lib/docker-package-driver.sh index dee89e0..a523c77 100755 --- a/testcase/lib/docker-package-driver.sh +++ b/testcase/lib/docker-package-driver.sh @@ -20,6 +20,7 @@ TEST_IMAGE=${TEST_IMAGE:?TEST_IMAGE is required} EFFECTIVE_SERVICE_MODE=${EFFECTIVE_SERVICE_MODE:-skip} SERVICE_MODE_REASON=${SERVICE_MODE_REASON:-unspecified} STARTUP_TIMEOUT=${STARTUP_TIMEOUT:-60} +PACKAGE_TIMEOUT=${PACKAGE_TIMEOUT:-1200} STAGE_TIMEOUT=${STAGE_TIMEOUT:-300} COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-5} SERVICE_TIMEOUT=${SERVICE_TIMEOUT:-60} @@ -34,17 +35,38 @@ CONTAINER_NAME="yumci-${RUN_ID}-${INDEX}" CONTAINER_STARTED=0 CONTAINER_RESULT="$PACKAGE_DIR/output/result.tsv" TERMINATION_REASON="" +LIFECYCLE_DEADLINE=$((SECONDS + PACKAGE_TIMEOUT)) sanitize() { printf '%s' "${1:-}" | tr '\t\r\n' ' ' | sed 's/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//' } +docker_with_timeout() { + local seconds=$1 + shift + timeout -k 5 "$seconds" docker "$@" +} + +docker_before_deadline() { + local deadline=$1 remaining + shift + remaining=$((deadline - SECONDS)) + [ "$remaining" -gt 0 ] || return 124 + docker_with_timeout "$remaining" "$@" +} + +startup_deadline() { + local deadline=$((SECONDS + STARTUP_TIMEOUT)) + [ "$deadline" -le "$LIFECYCLE_DEADLINE" ] || deadline=$LIFECYCLE_DEADLINE + printf '%s\n' "$deadline" +} + write_driver_failure() { local reason=$1 tmp [ -s "$RESULT_FILE" ] && return 0 tmp="$RESULT_FILE.tmp.$$" - printf '%s\t%s\tunknown\t%s\tFAIL\t%s\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\t0\t0\t0\tcontainer\t1\n' \ + printf '%s\t%s\tunknown\t%s\tFAIL\t%s\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\tSKIP\tinstall did not complete\t0\t0\t0\tcomplete\t1\n' \ "$INDEX" "$(sanitize "$REQUEST")" "$(sanitize "$REQUEST")" "$(sanitize "$reason")" >"$tmp" mv -f "$tmp" "$RESULT_FILE" } @@ -70,7 +92,11 @@ cleanup() { local rc=$? trap - EXIT INT TERM HUP if [ "$CONTAINER_STARTED" -eq 1 ]; then - docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + docker_with_timeout "$STARTUP_TIMEOUT" rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || { + printf 'failed to remove container %s during cleanup\n' "$CONTAINER_NAME" \ + >>"$PACKAGE_DIR/driver.log" + rc=2 + } fi [ -s "$RESULT_FILE" ] || publish_worker_result || true if [ ! -s "$RESULT_FILE" ]; then @@ -134,13 +160,16 @@ else common_args+=(--entrypoint /usr/bin/sleep) fi +STARTUP_DEADLINE=$(startup_deadline) if [ "$EFFECTIVE_SERVICE_MODE" = systemd ]; then - docker run "${common_args[@]}" "$TEST_IMAGE" >>"$PACKAGE_DIR/driver.log" 2>&1 || { + docker_before_deadline "$STARTUP_DEADLINE" run "${common_args[@]}" "$TEST_IMAGE" \ + >>"$PACKAGE_DIR/driver.log" 2>&1 || { write_driver_failure "failed to start privileged systemd container" exit 2 } else - docker run "${common_args[@]}" "$TEST_IMAGE" infinity >>"$PACKAGE_DIR/driver.log" 2>&1 || { + docker_before_deadline "$STARTUP_DEADLINE" run "${common_args[@]}" "$TEST_IMAGE" infinity \ + >>"$PACKAGE_DIR/driver.log" 2>&1 || { write_driver_failure "failed to start package container" exit 2 } @@ -149,8 +178,14 @@ CONTAINER_STARTED=1 if [ "$EFFECTIVE_SERVICE_MODE" = systemd ]; then state="" - for ((attempt = 0; attempt < STARTUP_TIMEOUT; attempt++)); do - state=$(docker exec "$CONTAINER_NAME" systemctl is-system-running 2>/dev/null || true) + while [ "$SECONDS" -lt "$STARTUP_DEADLINE" ]; do + state=$(docker_before_deadline "$STARTUP_DEADLINE" exec "$CONTAINER_NAME" \ + systemctl is-system-running 2>/dev/null) || exec_rc=$? + if [ "${exec_rc:-0}" -eq 124 ]; then + state=timeout + break + fi + exec_rc=0 case "$state" in running | degraded) break ;; initializing | starting | "") sleep 1 ;; @@ -166,7 +201,7 @@ if [ "$EFFECTIVE_SERVICE_MODE" = systemd ]; then esac fi -docker exec \ +docker_before_deadline "$LIFECYCLE_DEADLINE" exec \ --env "INDEX=$INDEX" \ --env "REQUEST=$REQUEST" \ --env "OUTPUT_DIR=/yum-ci/output" \ diff --git a/testcase/lib/docker-package-worker.sh b/testcase/lib/docker-package-worker.sh index 9fe8cb7..ba4a6fd 100755 --- a/testcase/lib/docker-package-worker.sh +++ b/testcase/lib/docker-package-worker.sh @@ -101,9 +101,6 @@ on_signal() { exit 143 } -trap cleanup EXIT -trap on_signal INT TERM HUP - first_error() { local file=$1 fallback=$2 line line=$(grep -Eim1 'error:|failed|failure|conflict|nothing to do|no match|timed out|cannot|not found|requires' \ @@ -293,11 +290,33 @@ discover_units() { /etc/systemd/system/*.service | /etc/systemd/system/*.socket | /etc/systemd/system/*.target) basename "$path" ;; + /usr/lib/systemd/user/*.service | /usr/lib/systemd/user/*.socket | /usr/lib/systemd/user/*.target | \ + /lib/systemd/user/*.service | /lib/systemd/user/*.socket | /lib/systemd/user/*.target | \ + /etc/systemd/user/*.service | /etc/systemd/user/*.socket | /etc/systemd/user/*.target) + printf 'user:%s\n' "$(basename "$path")" + ;; esac done <"$OUTPUT_DIR/rpm-files.list" | sort -u >"$OUTPUT_DIR/units.list" UNIT_COUNT=$(wc -l <"$OUTPUT_DIR/units.list") } +is_dangerous_unit() { + local unit=${1#user:} + case "$unit" in + poweroff.target | reboot.target | halt.target | kexec.target | soft-reboot.target | \ + ctrl-alt-del.target | shutdown.target | exit.target | final.target | \ + runlevel0.target | runlevel6.target | sleep.target | suspend.target | \ + hibernate.target | hybrid-sleep.target | suspend-then-hibernate.target | \ + systemd-poweroff.service | systemd-reboot.service | systemd-halt.service | \ + systemd-kexec.service | systemd-soft-reboot.service | systemd-suspend.service | \ + systemd-hibernate.service | systemd-hybrid-sleep.service | \ + systemd-suspend-then-hibernate.service) + return 0 + ;; + *) return 1 ;; + esac +} + systemd_ready() { local state [ -d /run/systemd/system ] || return 1 @@ -357,14 +376,28 @@ test_services() { while IFS= read -r unit || [ -n "$unit" ]; do [ -n "$unit" ] || continue + if [[ $unit == user:* ]]; then + skipped=$((skipped + 1)) + printf '%s\tSKIP\tuser unit requires a per-user systemd manager/session\n' "$unit" \ + >>"$OUTPUT_DIR/service-results.tsv" + continue + fi + if is_dangerous_unit "$unit"; then + skipped=$((skipped + 1)) + printf '%s\tSKIP\tdangerous shutdown/reboot unit is never started by this test\n' "$unit" \ + >>"$OUTPUT_DIR/service-results.tsv" + continue + fi if [[ $unit == *@.* ]]; then skipped=$((skipped + 1)) printf '%s\tSKIP\ttemplate unit requires an instance name\n' "$unit" \ >>"$OUTPUT_DIR/service-results.tsv" continue fi - refuse_start=$(systemctl show "$unit" -p RefuseManualStart --value 2>/dev/null || true) - refuse_stop=$(systemctl show "$unit" -p RefuseManualStop --value 2>/dev/null || true) + refuse_start=$(timeout -k 5 "$SERVICE_TIMEOUT" systemctl show "$unit" \ + -p RefuseManualStart --value 2>/dev/null || true) + refuse_stop=$(timeout -k 5 "$SERVICE_TIMEOUT" systemctl show "$unit" \ + -p RefuseManualStop --value 2>/dev/null || true) if [ "$refuse_start" = yes ] || [ "$refuse_stop" = yes ]; then skipped=$((skipped + 1)) printf '%s\tSKIP\tRefuseManualStart=%s RefuseManualStop=%s\n' \ @@ -391,7 +424,8 @@ test_services() { failed=$((failed + 1)) printf '%s\tFAIL\tstart=%s is-active=%s stop=%s\n' \ "$unit" "$start_rc" "$active_rc" "$stop_rc" >>"$OUTPUT_DIR/service-results.tsv" - systemctl --no-pager status "$unit" >>"$OUTPUT_DIR/service.log" 2>&1 || true + timeout -k 5 "$SERVICE_TIMEOUT" systemctl --no-pager status "$unit" \ + >>"$OUTPUT_DIR/service.log" 2>&1 || true fi done <"$OUTPUT_DIR/units.list" @@ -412,9 +446,29 @@ test_services() { checkpoint } +runtime_critical_path() { + local command_name path owner + for command_name in bash date grep mv rpm sed tr; do + path=$(command -v "$command_name" 2>/dev/null || true) + [ -n "$path" ] || continue + owner=$(rpm -qf --qf '%{NAME}\n' "$path" 2>/dev/null | head -n 1 || true) + if [ -n "$owner" ] && [ "$owner" = "$PACKAGE_NAME" ]; then + printf '%s\n' "$path" + return 0 + fi + done + return 1 +} + remove_package() { - local rc=0 + local rc=0 critical_path stage_begin remove + if critical_path=$(runtime_critical_path); then + REMOVE_STATUS=SKIP + REMOVE_REASON="destructive removal skipped: package owns worker runtime command $critical_path" + checkpoint + return 0 + fi log "remove stage: tst_lib/ts_common.sh test_single_rpm_remove $BINARY_PACKAGE" run_with_timeout "$STAGE_TIMEOUT" "$OUTPUT_DIR/remove.log" \ "$TS_COMMON_RUNNER" remove "$PACKAGE_SPEC" "$BINARY_PACKAGE" \ @@ -450,6 +504,8 @@ remove_package() { main() { local rc=0 + trap cleanup EXIT + trap on_signal INT TERM HUP mkdir -p "$OUTPUT_DIR" : >"$OUTPUT_DIR/worker.log" checkpoint @@ -480,4 +536,6 @@ main() { return "$rc" } -main +if [ "${YUM_CI_DOCKER_WORKER_LIB_ONLY:-0}" != 1 ]; then + main +fi diff --git a/testcase/tests/fake-docker-package-driver.sh b/testcase/tests/fake-docker-package-driver.sh index 1ec85da..7b683de 100755 --- a/testcase/tests/fake-docker-package-driver.sh +++ b/testcase/tests/fake-docker-package-driver.sh @@ -13,6 +13,39 @@ service_list=$6 mkdir -p "$package_dir/output" "$(dirname -- "$result_file")" +case "$request" in + silent-*) exit 0 ;; + truncated-*) + printf '%s\t%s\tbroken\n' "$index" "$request" >"$result_file" + exit 0 + ;; + wrong-identity-*) + printf '%s\tother-request\tother-src\tother-bin\tPASS\tok\tPASS\tok\tPASS\tok\tPASS\tok\t1\t0\t0\tcomplete\t1\n' \ + "$((index + 1))" >"$result_file" + exit 0 + ;; + empty-binary-*) + printf '%s\t%s\t%s-src\t\tPASS\tok\tPASS\tok\tPASS\tok\tPASS\tok\t1\t0\t0\tcomplete\t1\n' \ + "$index" "$request" "$request" >"$result_file" + exit 0 + ;; + unknown-source-*) + printf '%s\t%s\tunknown\t%s\tPASS\tok\tPASS\tok\tPASS\tok\tPASS\tok\t1\t0\t0\tcomplete\t1\n' \ + "$index" "$request" "$request" >"$result_file" + exit 0 + ;; + bad-status-*) + printf '%s\t%s\t%s-src\t%s\tOK\tok\tPASS\tok\tPASS\tok\tPASS\tok\t1\t0\t0\tcomplete\t1\n' \ + "$index" "$request" "$request" "$request" >"$result_file" + exit 0 + ;; + incomplete-*) + printf '%s\t%s\t%s-src\t%s\tPASS\tok\tPASS\tok\tPASS\tok\tPASS\tok\t1\t0\t0\tremove\t0\n' \ + "$index" "$request" "$request" "$request" >"$result_file" + exit 0 + ;; +esac + list_has() { local file=$1 value=$2 grep -Fqx "$value" "$file" 2>/dev/null @@ -56,6 +89,13 @@ case "$request" in printf 'demo.service\tFAIL\tstart=1 is-active=125 stop=0\n' \ >>"$package_dir/output/service-results.tsv" ;; + service-skips-*) + service=SKIP; service_reason="units skipped with explicit reasons" + printf 'demo@.service\tSKIP\ttemplate unit requires an instance name\n' \ + >"$package_dir/output/service-results.tsv" + printf 'manual-blocked.service\tSKIP\tRefuseManualStart=yes RefuseManualStop=no\n' \ + >>"$package_dir/output/service-results.tsv" + ;; remove-fail-*) remove=FAIL; remove_reason="injected remove failure" ;; multi-fail-*) command=FAIL; command_reason="injected command failure" diff --git a/testcase/tests/test-benchmark-command-execute-docker.sh b/testcase/tests/test-benchmark-command-execute-docker.sh new file mode 100755 index 0000000..a329b32 --- /dev/null +++ b/testcase/tests/test-benchmark-command-execute-docker.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Host-only regression test for benchmark exit-code propagation. + +set -uo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +CASE_DIR=$(cd -- "$SCRIPT_DIR/.." && pwd -P) +BENCHMARK="$CASE_DIR/benchmark-command-execute-docker.sh" +TMP=$(mktemp -d) +trap 'rm -rf -- "$TMP"' EXIT + +mkdir -p "$TMP/bin" +: >"$TMP/packages.list" +: >"$TMP/desktop.list" +: >"$TMP/services.list" + +cat >"$TMP/bin/docker" <<'EOF' +#!/bin/bash +exit 0 +EOF +cat >"$TMP/fake-main" <<'EOF' +#!/bin/bash +exit "${FAKE_MAIN_RC:-0}" +EOF +chmod +x "$TMP/bin/docker" "$TMP/fake-main" + +if PATH="$TMP/bin:$PATH" JOBS_CSV=1 FAKE_MAIN_RC=2 \ + YUM_CI_DOCKER_MAIN="$TMP/fake-main" BENCHMARK_OUTPUT_DIR="$TMP/out-2" \ + bash "$BENCHMARK" "$TMP/packages.list" "$TMP/desktop.list" "$TMP/services.list" \ + >/dev/null 2>&1; then + exit 1 +else + [ "$?" -eq 2 ] +fi +grep -Fq $'1\t' "$TMP/out-2/benchmark.tsv" +grep -Fq $'\t2' "$TMP/out-2/benchmark.tsv" + +if PATH="$TMP/bin:$PATH" JOBS_CSV=1 FAKE_MAIN_RC=1 \ + YUM_CI_DOCKER_MAIN="$TMP/fake-main" BENCHMARK_OUTPUT_DIR="$TMP/out-1" \ + bash "$BENCHMARK" "$TMP/packages.list" "$TMP/desktop.list" "$TMP/services.list" \ + >/dev/null 2>&1; then + exit 1 +else + [ "$?" -eq 1 ] +fi + +echo 'PASS: benchmark propagates package and infrastructure failures' diff --git a/testcase/tests/test-command-execute-docker.sh b/testcase/tests/test-command-execute-docker.sh index 86570a9..064c726 100755 --- a/testcase/tests/test-command-execute-docker.sh +++ b/testcase/tests/test-command-execute-docker.sh @@ -62,6 +62,18 @@ else assert_eq "$?" 2 'invalid zero concurrency returns infrastructure/usage code' fi +if bash "$MAIN" --validate-only -j 65 "$TMP/inputs/normalize.list" \ + "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1; then + not_ok 'excessive concurrency is rejected' +else + assert_eq "$?" 2 'concurrency above the hard limit returns usage code' +fi + +trimmed_output=$(bash "$MAIN" --validate-only -j 8 "$TMP/inputs/normalize.list" \ + "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" 2>/dev/null) +assert_contains <(printf '%s\n' "$trimmed_output") 'packages=3 jobs=3' \ + 'concurrency is reduced to the normalized package count' + mkdir -p "$TMP/out-infra" printf 'infra-one\n' >"$TMP/inputs/infra.list" if run_main -j 1 --service-mode skip --output-dir "$TMP/out-infra" \ @@ -72,6 +84,27 @@ else assert_eq "$?" 2 'Docker infrastructure failure returns exit code 2' fi +mkdir -p "$TMP/out-invalid" +cat >"$TMP/inputs/invalid.list" <<'EOF' +silent-one +truncated-one +wrong-identity-one +empty-binary-one +unknown-source-one +bad-status-one +incomplete-one +EOF +if run_main -j 7 --service-mode skip --output-dir "$TMP/out-invalid" \ + "$TMP/inputs/invalid.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" \ + >/dev/null 2>&1; then + not_ok 'missing and malformed worker results are rejected' +else + assert_eq "$?" 2 'missing/schema/identity/completion failures return infrastructure code' +fi +invalid_detail=$(find "$TMP/out-invalid" -maxdepth 1 -name 'package-command-detail-docker-*' | head -n 1) +assert_eq "$(grep -c 'invalid container result\|without a result' "$invalid_detail")" 7 \ + 'every invalid all-pass result is replaced with an explicit failure' + cat >"$TMP/inputs/mixed.list" <<'EOF' ok-one desktop-one @@ -102,6 +135,59 @@ assert_contains "$mixed_report" 'service-mixed-one 原因: demo@.service: templ assert_contains "$mixed_report" 'template-unit-one' 'template unit is explicitly skipped' assert_contains "$mixed_report" '对账: 1 + 3 + 3 = 7' 'summary reconciliation is exact' +mkdir -p "$TMP/out-service-skips" +printf 'service-skips-one\n' >"$TMP/inputs/service-skips.list" +run_main -j 1 --service-mode skip --output-dir "$TMP/out-service-skips" \ + "$TMP/inputs/service-skips.list" "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" \ + >/dev/null 2>&1 +service_skip_report=$(find "$TMP/out-service-skips" -maxdepth 1 \ + -name 'package-command-result-docker-*' | head -n 1) +assert_contains "$service_skip_report" 'demo@.service: template unit requires an instance name' \ + 'template-unit skip reason is retained when the service stage is SKIP' +assert_contains "$service_skip_report" 'manual-blocked.service: RefuseManualStart=yes RefuseManualStop=no' \ + 'RefuseManualStart skip reason is retained in the summary' + +mkdir -p "$TMP/bin-cleanup" "$TMP/out-cleanup" +cat >"$TMP/bin-cleanup/docker" <<'EOF' +#!/bin/bash +case "$1" in + info | run | version) exit 0 ;; + image) exit 0 ;; + ps) printf 'leftover-container\n'; exit 0 ;; + rm) exit 1 ;; + *) exit 0 ;; +esac +EOF +chmod +x "$TMP/bin-cleanup/docker" +printf 'ok-cleanup\n' >"$TMP/inputs/cleanup.list" +if PATH="$TMP/bin-cleanup:$PATH" YUM_CI_DOCKER_DRIVER="$FAKE" \ + bash "$MAIN" -j 1 --skip-build --test-image fake-image --service-mode skip \ + --startup-timeout 1 --output-dir "$TMP/out-cleanup" "$TMP/inputs/cleanup.list" \ + "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1; then + not_ok 'container cleanup failure is rejected' +else + assert_eq "$?" 2 'container cleanup failure returns infrastructure code' +fi + +mkdir -p "$TMP/bin-startup" "$TMP/out-startup" +cat >"$TMP/bin-startup/docker" <<'EOF' +#!/bin/bash +case "$1" in + info | version | image | ps | rm) exit 0 ;; + run) sleep 2; exit 0 ;; + *) exit 0 ;; +esac +EOF +chmod +x "$TMP/bin-startup/docker" +if PATH="$TMP/bin-startup:$PATH" YUM_CI_DOCKER_DRIVER="$FAKE" \ + bash "$MAIN" -j 1 --skip-build --test-image fake-image --service-mode auto \ + --startup-timeout 1 --output-dir "$TMP/out-startup" "$TMP/inputs/cleanup.list" \ + "$TMP/inputs/empty.list" "$TMP/inputs/empty.list" >/dev/null 2>&1; then + not_ok 'systemd probe startup timeout is rejected' +else + assert_eq "$?" 2 'systemd probe startup deadline returns infrastructure code' +fi + rm -rf "$TMP/out-timeout" mkdir -p "$TMP/out-timeout" printf 'timeout-one\n' >"$TMP/inputs/timeout.list" diff --git a/testcase/tests/test-docker-package-driver.sh b/testcase/tests/test-docker-package-driver.sh index a25d65d..b2004fe 100755 --- a/testcase/tests/test-docker-package-driver.sh +++ b/testcase/tests/test-docker-package-driver.sh @@ -18,8 +18,16 @@ cat >"$TMP/bin/docker" <<'EOF' #!/bin/bash printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG" case "$1" in - run | rm) exit 0 ;; + run) + [ -z "${FAKE_DOCKER_RUN_SLEEP:-}" ] || sleep "$FAKE_DOCKER_RUN_SLEEP" + exit 0 + ;; + rm) + [ -z "${FAKE_DOCKER_RM_SLEEP:-}" ] || sleep "$FAKE_DOCKER_RM_SLEEP" + exit 0 + ;; exec) + [ -z "${FAKE_DOCKER_EXEC_SLEEP:-}" ] || sleep "$FAKE_DOCKER_EXEC_SLEEP" mkdir -p "$FAKE_PACKAGE_DIR/output" printf '1\talpha\talpha-src\talpha-1.0-1.x86_64\tPASS\tinstalled\tPASS\tcommand passed\tPASS\tno units\tPASS\tremoved\t25\t1\t0\tcomplete\t1\n' \ >"$FAKE_PACKAGE_DIR/output/result.tsv" @@ -70,4 +78,35 @@ FAKE_PACKAGE_DIR="$TMP/package-repo" REPO_DIR="$TMP/repo" \ "$TMP/desktop.list" "$TMP/service.list" [ ! -e "$TMP/package-repo/repos" ] -echo 'PASS: driver publishes atomic worker result through one SELinux-safe bind mount' +mkdir -p "$TMP/package-run-timeout" +if STARTUP_TIMEOUT=1 FAKE_DOCKER_RUN_SLEEP=2 FAKE_PACKAGE_DIR="$TMP/package-run-timeout" \ + bash "$DRIVER" 4 delta "$TMP/package-run-timeout" "$TMP/results/00004.tsv" \ + "$TMP/desktop.list" "$TMP/service.list" >/dev/null 2>&1; then + exit 1 +else + [ "$?" -eq 2 ] +fi +grep -Fq 'failed to start package container' "$TMP/results/00004.tsv" + +mkdir -p "$TMP/package-exec-timeout" +if PACKAGE_TIMEOUT=1 STARTUP_TIMEOUT=5 FAKE_DOCKER_EXEC_SLEEP=2 \ + FAKE_PACKAGE_DIR="$TMP/package-exec-timeout" \ + bash "$DRIVER" 5 epsilon "$TMP/package-exec-timeout" "$TMP/results/00005.tsv" \ + "$TMP/desktop.list" "$TMP/service.list" >/dev/null 2>&1; then + exit 1 +else + [ "$?" -eq 124 ] +fi +grep -Fq 'container driver terminated unexpectedly with status 124' "$TMP/results/00005.tsv" + +mkdir -p "$TMP/package-rm-timeout" +if STARTUP_TIMEOUT=1 FAKE_DOCKER_RM_SLEEP=2 FAKE_PACKAGE_DIR="$TMP/package-rm-timeout" \ + bash "$DRIVER" 6 zeta "$TMP/package-rm-timeout" "$TMP/results/00006.tsv" \ + "$TMP/desktop.list" "$TMP/service.list" >/dev/null 2>&1; then + exit 1 +else + [ "$?" -eq 2 ] +fi +grep -Fq 'failed to remove container' "$TMP/package-rm-timeout/driver.log" + +echo 'PASS: driver publishes results and bounds Docker run/exec/rm with cleanup errors as infrastructure failures' diff --git a/testcase/tests/test-docker-package-worker.sh b/testcase/tests/test-docker-package-worker.sh new file mode 100755 index 0000000..561a5c0 --- /dev/null +++ b/testcase/tests/test-docker-package-worker.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Host-only regression tests for unit discovery and destructive-removal guards. +# Globals below are consumed by sourced worker functions. +# shellcheck disable=SC2034 + +set -uo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +CASE_DIR=$(cd -- "$SCRIPT_DIR/.." && pwd -P) +WORKER="$CASE_DIR/lib/docker-package-worker.sh" +TMP=$(mktemp -d) +trap 'rm -rf -- "$TMP"' EXIT + +mkdir -p "$TMP/output" "$TMP/bin" +: >"$TMP/desktop.list" +: >"$TMP/service.list" + +export INDEX=1 REQUEST=runtime-owner OUTPUT_DIR="$TMP/output" +export RESULT_FILE="$TMP/output/result.tsv" +export DESKTOP_LIST="$TMP/desktop.list" SERVICE_LIST="$TMP/service.list" +export YUM_CI_DOCKER_WORKER_LIB_ONLY=1 + +# shellcheck source=/dev/null +source "$WORKER" + +cat >"$OUTPUT_DIR/rpm-files.list" <<'EOF' +/usr/lib/systemd/system/demo.service +/usr/lib/systemd/system/poweroff.target +/usr/lib/systemd/user/user-demo.service +/etc/systemd/user/user-demo.socket +EOF +discover_units +grep -Fxq 'demo.service' "$OUTPUT_DIR/units.list" +grep -Fxq 'poweroff.target' "$OUTPUT_DIR/units.list" +grep -Fxq 'user:user-demo.service' "$OUTPUT_DIR/units.list" +grep -Fxq 'user:user-demo.socket' "$OUTPUT_DIR/units.list" +[ "$UNIT_COUNT" -eq 4 ] +is_dangerous_unit poweroff.target +is_dangerous_unit reboot.target +! is_dangerous_unit demo.service + +cat >"$TMP/bin/rpm" <<'EOF' +#!/bin/bash +if [ "$1" = -qf ]; then + printf 'runtime-owner\n' + exit 0 +fi +exit 1 +EOF +chmod +x "$TMP/bin/rpm" +PATH="$TMP/bin:$PATH" +PACKAGE_NAME=runtime-owner +BINARY_PACKAGE=runtime-owner-1.0-1.x86_64 +PACKAGE_SPEC=runtime-owner.x86_64 +SOURCE_PACKAGE=runtime-owner +INSTALL_STATUS=PASS +INSTALL_REASON=installed +COMMAND_STATUS=PASS +COMMAND_REASON=passed +SERVICE_STATUS=PASS +SERVICE_REASON=passed +REMOVE_STATUS=SKIP +REMOVE_REASON='not reached' + +remove_package +[ "$REMOVE_STATUS" = SKIP ] +[[ $REMOVE_REASON == destructive\ removal\ skipped:* ]] +grep -Fq $'\tSKIP\tdestructive removal skipped:' "$RESULT_FILE" + +echo 'PASS: worker discovers user units, blocks shutdown targets, and protects its runtime tools' -- Gitee