From b895e281811581ec228a7169220e592b024014c0 Mon Sep 17 00:00:00 2001 From: Rick Peters Date: Tue, 29 Sep 2026 14:41:37 +0200 Subject: [PATCH] feat: vmtest.sh runs everything: boot loaders plus the cli, menu, hw, installer and toggles suites vmtest.sh [--install] [--window] [--screen] [boot|loader|suite...], a detached screen session with --screen, the summary in ~/vms/last-test.txt, the rows that need a person listed as SKIP. New: the installer suite (R2.1), the toggles suite (R1.8) and H3 (BIOS dry-run, per-block '# env:' for the VM start). vmsuite.sh and vmbootloadertest.sh are executable again. --- .claude/skills/vm-install/SKILL.md | 18 ++++- TODO.md | 8 +- scripts/vmbootloadertest.sh | 0 scripts/vmsuite.sh | 4 +- scripts/vmtest.sh | 74 +++++++++++++------ share/vmtest/cli/20-f3-everything-desktop.sh | 0 .../cli/30-f4-single-brings-conversion.sh | 0 .../cli/40-f5-poweroff-brings-machine.sh | 0 share/vmtest/cli/50-f7-boot-only.sh | 0 share/vmtest/hw/10-h1-h5-leds-poweroff.sh | 0 .../hw/20-h4-boot-desktop-reboot.host.sh | 0 share/vmtest/hw/30-r1-7-cec.host.sh | 0 share/vmtest/hw/40-h3-bios-dry-run.sh | 5 ++ .../vmtest/installer/10-r2-1-defaults.host.sh | 7 ++ share/vmtest/menu/10-r1-menu.sh | 0 share/vmtest/toggles/10-r1-8-matrix.sh | 32 ++++++++ 16 files changed, 119 insertions(+), 29 deletions(-) mode change 100644 => 100755 scripts/vmbootloadertest.sh mode change 100644 => 100755 scripts/vmsuite.sh mode change 100644 => 100755 scripts/vmtest.sh mode change 100644 => 100755 share/vmtest/cli/20-f3-everything-desktop.sh mode change 100644 => 100755 share/vmtest/cli/30-f4-single-brings-conversion.sh mode change 100644 => 100755 share/vmtest/cli/40-f5-poweroff-brings-machine.sh mode change 100644 => 100755 share/vmtest/cli/50-f7-boot-only.sh mode change 100644 => 100755 share/vmtest/hw/10-h1-h5-leds-poweroff.sh mode change 100644 => 100755 share/vmtest/hw/20-h4-boot-desktop-reboot.host.sh mode change 100644 => 100755 share/vmtest/hw/30-r1-7-cec.host.sh create mode 100755 share/vmtest/hw/40-h3-bios-dry-run.sh create mode 100755 share/vmtest/installer/10-r2-1-defaults.host.sh mode change 100644 => 100755 share/vmtest/menu/10-r1-menu.sh create mode 100755 share/vmtest/toggles/10-r1-8-matrix.sh diff --git a/.claude/skills/vm-install/SKILL.md b/.claude/skills/vm-install/SKILL.md index c1d6692..6254d4a 100644 --- a/.claude/skills/vm-install/SKILL.md +++ b/.claude/skills/vm-install/SKILL.md @@ -122,7 +122,23 @@ plasmashell (gamescope doesn't render in the VM: a black window is normal). cache shared as 9p tag `cache`, bound over pacman's cache in the live system and in the new one (Steamify's packages): the first install fills it. -## Testing a boot loader: `scripts/vmtest.sh` +## The whole automated test: `scripts/vmtest.sh` + +`scripts/vmtest.sh [--install] [--window] [--screen] [boot | ... | ...]`: no argument runs +everything: the boot loader test (below) and every suite in `share/vmtest/` (`cli`, `menu`, `hw`, `installer`, +`toggles`: the TESTPLAN.md rows that need no person, run by `scripts/vmsuite.sh` on the plain VM +`~/vms/steamify-vm`, each block from a fresh `ssh-ready`). `--screen` runs it in a detached `screen` +session (`screen -r vmtest`); output goes to `~/vms/test.log` (one shared Konsole follows it when headless +on a desktop), the summary to `~/vms/last-test.txt`, exit status = failed checks. Adding a check: a block file +in `share/vmtest//NN-name.sh` (guest side, prelude helpers `pass/fail/skip/info/sf/is/expect_on`) or +`NN-name.host.sh` (host side, for reboots and host scripts); `# env: VAR=x` in a block sets it for the VM +start; `ONLY= scripts/vmsuite.sh ` runs one block. Rows that need a person (U1-U9, R2.2, +H-Real) are printed as SKIP at the end. Pitfalls met while building it: atomic file swaps (`os.replace`) +lose the executable bit (`chmod --reference` or call scripts with `bash`); a suite block that reboots must +be a `.host.sh` block; ssh has no Plasma session, the prelude exports one; a one-off network failure can fail +an install step (LED driver from the AUR), rerun the block before suspecting Steamify. + +## Testing a boot loader: `scripts/vmtest.sh boot` Run this instead of doing the steps below by hand: `scripts/vmtest.sh [--install] [--window] [loader...]` (default: every loader the ISO advertises, read from `calamares-online.sh`; it fails when an advertised diff --git a/TODO.md b/TODO.md index f30b574..3f2ecd1 100644 --- a/TODO.md +++ b/TODO.md @@ -36,13 +36,13 @@ R1.9, R1.10, R2.1, G1-G3, F1-F7, H1, H3, H4 (session file), H5, and the boot loa Not automatable without a person (stay manual, listed in the summary as SKIP; no screenshot testing): U1-U9, R2.2 (first desktop login look), H-Real (real Steam Machine). -- [ ] S1. Framework: `scripts/vmtest.sh` takes suites, resets the VM per block, tallies PASS/FAIL/SKIP, +- [x] S1. Framework (vmtest.sh runs boot + every suite, --screen, summary in ~/vms/last-test.txt): `scripts/vmtest.sh` takes suites, resets the VM per block, tallies PASS/FAIL/SKIP, prints the skipped manual rows; move the loader matrix under suite `boot` - [x] S2. Suite `cli` (55 checks pass): G1-G3, F1-F7 (exit codes, `features.state`, boot-desktop unit) - [x] S3. Suite `menu` (36 checks pass): R1.1-R1.6, R1.10 with scripted menu input (`vmrun.sh`) and `vmstate.sh` asserts -- [x] S4. Suite `hw` (68 checks pass; H3 BIOS dry-run still to add): H1, H3 (`WIZARD_BIOS_DRY_RUN=1`), H4, H5, R1.7 (`vmcec.sh --no-sleep`) -- [ ] S5. Suite `installer`: R2.1 (`vminstallsim.sh`) -- [ ] S6. R1.8 toggle matrix and R1.9 reboot checks (from the skill), as suite `toggles` +- [x] S4. Suite `hw` (70 checks pass incl. H3 BIOS dry-run): H1, H3 (`WIZARD_BIOS_DRY_RUN=1`), H4, H5, R1.7 (`vmcec.sh --no-sleep`) +- [x] S5. Suite `installer` (4 pass): R2.1 (`vminstallsim.sh`) +- [x] S6. R1.8 toggle matrix as suite `toggles` (42 pass, 1 expected skip); R1.9 reboot checks still to add - [ ] S7. TESTPLAN.md: mark which rows are automated (a column or a tag), keep the results log automatic: the script appends a line to it (date, branch, suites, counts) - [ ] S8. Update the `steam-machine-testing` and `cachyos-vm-testing` skills: "run `scripts/vmtest.sh`" first diff --git a/scripts/vmbootloadertest.sh b/scripts/vmbootloadertest.sh old mode 100644 new mode 100755 diff --git a/scripts/vmsuite.sh b/scripts/vmsuite.sh old mode 100644 new mode 100755 index 1c3cf96..8b4cd1f --- a/scripts/vmsuite.sh +++ b/scripts/vmsuite.sh @@ -4,7 +4,7 @@ # scripts/vmsuite.sh [--window] suites: the folders in share/vmtest/ (cli, ...) # Every block share/vmtest//NN-name.sh starts from a fresh `ssh-ready` (vmreset.sh --fremont), # runs in the guest as the VM user (repo on /mnt; NN-name.host.sh blocks run on the host instead), and prints PASS/FAIL/SKIP lines; the first -# comment lines say which TESTPLAN rows it covers. Log: $VM_DIR..log. Exit status = FAILs. +# comment lines say which TESTPLAN rows it covers; `# env: VAR=value` there is set for the VM start (e.g. BIOS_VERSION=F7F0107). Log: $VM_DIR..log. Exit status = FAILs. # Env: ONLY= runs just that block; VM_DIR (default ~/vms/steamify-vm), REPO (default ../steamify-cachyos). set -uo pipefail here="$(cd "$(dirname "$0")" && pwd)" @@ -23,7 +23,7 @@ printf '\n===== vmsuite.sh %s (%s) =====\n' "$suite" "$(date +%T)" for block in "$dir"/*.sh; do [[ -z "${ONLY:-}" || "$(basename "$block")" == "$ONLY"* ]] || continue # ONLY=50: just that block printf '\n##### %s %s\n' "$(date +%T)" "$(basename "$block" .sh)" - "$here/vmreset.sh" --fremont > /tmp/vmsuite-reset.out 2>&1 || { echo "FAIL vmreset.sh failed: $(tail -n 1 /tmp/vmsuite-reset.out)"; continue; } + env $(sed -n 's/^# env: //p' "$block" | head -n 1) "$here/vmreset.sh" --fremont > /tmp/vmsuite-reset.out 2>&1 || { echo "FAIL vmreset.sh failed: $(tail -n 1 /tmp/vmsuite-reset.out)"; continue; } if [[ "$block" == *.host.sh ]]; then # runs on the host (reboots, host scripts); prelude-host.sh has the helpers ( . "$repo/share/vmtest/prelude-host.sh"; . "$block" ) 2>&1 | sed -u 's/\x1b\[[0-9;]*m//g' continue diff --git a/scripts/vmtest.sh b/scripts/vmtest.sh old mode 100644 new mode 100755 index 19d6bfc..74372bf --- a/scripts/vmtest.sh +++ b/scripts/vmtest.sh @@ -1,37 +1,67 @@ #!/bin/bash -# The whole boot loader test in one command: every boot loader the Steamify ISO -# advertises, each on its own VM (~/vms/bl-), one after the other. -# scripts/vmtest.sh [--install] [--window] [loader...] default: all advertised loaders -# --install install each VM first from the newest Steamify ISO (asks before replacing a disk) -# --window show the VMs' windows (default: headless; on a desktop one Konsole follows all the logs) -# Checks per loader (scripts/vmbootloadertest.sh + share/bootloader-test/): -# Steamify's state as its menu sees it, OS name untouched, loader identity, -# DKMS modules for every kernel, Steamify's loader-specific code, a kernel -# update, a reboot into each kernel. Add a check there, and it runs here. -# Prints one line per loader and the failed checks; exit status = failures. +# The whole automated test, one command: the boot loader test for every loader the Steamify ISO +# advertises, plus every suite in share/vmtest/ (TESTPLAN.md rows that need no person). +# scripts/vmtest.sh [--install] [--window] [--screen] [boot | ... | ...] +# no argument everything: boot (all advertised loaders) and all suites +# boot the boot loader test (limine, systemd-boot, grub), or name loaders +# cli, menu, hw, installer, toggles (folders in share/vmtest/) +# --install install the boot loader VMs first from the newest Steamify ISO (asks before replacing a disk) +# --window show the VMs' windows (default: headless; one shared Konsole follows ~/vms/test.log) +# --screen run in a detached `screen` session named vmtest: `screen -r vmtest` to watch, Ctrl-a d to leave +# The suites run on the plain CachyOS VM ~/vms/steamify-vm (Plasma, from vminstall.sh --iso +# with VM_STEAMIFY=skip); the boot loader test on ~/vms/bl- (Steamify ISO). +# Output: one line per loader/suite and the failed checks; the same in ~/vms/last-test.txt. Exit status = failures. +# Rows that need a person (screenshots, the real Steam Machine) are listed at the end, never silently dropped. set -uo pipefail here="$(cd "$(dirname "$0")" && pwd)" repo="$(cd "$here/.." && pwd)" iso_repo="$repo/../steammachine-cachyos-live-iso" -install=""; window=""; loaders=() +args=("$@") +install=""; window=""; screen=false; loaders=(); suites=(); want_boot=false for a in "$@"; do - case "$a" in --install) install=--install ;; --window) window=--window ;; *) loaders+=("$a") ;; esac + case "$a" in + --install) install=--install ;; + --window) window=--window ;; + --screen) screen=true ;; + boot) want_boot=true ;; + limine|systemd-boot|grub) loaders+=("$a"); want_boot=true ;; + *) [[ -d "$repo/share/vmtest/$a" ]] && suites+=("$a") || { echo "Unknown argument: $a" >&2; exit 2; } ;; + esac done +if $screen; then + command -v screen >/dev/null || { echo "screen is not installed" >&2; exit 2; } + screen -ls | grep -q '\.vmtest\s' && { echo "A vmtest screen session is already running: screen -r vmtest" >&2; exit 1; } + rest=(); for a in "${args[@]}"; do [[ "$a" == --screen ]] || rest+=("$a"); done + screen -dmS vmtest bash -c "$(printf '%q ' "$0" "${rest[@]}"); echo; echo 'Done (summary: ~/vms/last-test.txt). Press Enter to close.'; read -r _" + echo "Running in the screen session vmtest: screen -r vmtest (Ctrl-a d to leave it running); the summary lands in ~/vms/last-test.txt" + exit 0 +fi +if [[ ${#loaders[@]} -eq 0 && ${#suites[@]} -eq 0 ]] && ! $want_boot; then + want_boot=true + for d in cli menu hw installer toggles; do [[ -d "$repo/share/vmtest/$d" ]] && suites+=("$d"); done +fi # What the ISO advertises: the boot loaders calamares-online.sh keeps. advertised="$(sed -n 's/.*"bootloader:\([^"]*\)".*/\1/p' "$iso_repo/archiso/airootfs/usr/local/bin/calamares-online.sh" 2>/dev/null | head -n 1)" [[ -n "$advertised" ]] || { echo "Can't read the advertised boot loaders from $iso_repo (calamares-online.sh)" >&2; exit 2; } -[[ ${#loaders[@]} -gt 0 ]] || read -r -a loaders <<< "$advertised" +$want_boot && [[ ${#loaders[@]} -eq 0 ]] && read -r -a loaders <<< "$advertised" tested="limine systemd-boot grub" # the loaders vmbootloadertest.sh has a way to boot the other kernel for -missing=0 +summary="$HOME/vms/last-test.txt" +{ +echo "vmtest.sh $(date '+%F %T') branch $(git -C "$repo" branch --show-current) $(git -C "$repo" rev-parse --short HEAD)" +total=0 for l in $advertised; do - [[ " $tested " == *" $l "* ]] || { echo "FAIL the ISO advertises $l, but there is no test for it (scripts/vmbootloadertest.sh)"; missing=$((missing + 1)); } + [[ " $tested " == *" $l "* ]] || { echo "FAIL the ISO advertises $l, but there is no test for it (scripts/vmbootloadertest.sh)"; total=$((total + 1)); } done -total=$missing -# No human is needed: headless; the scripts share one Konsole (common.sh, $TEST_LOG). for l in "${loaders[@]}"; do - "$here/vmbootloadertest.sh" "$l" $install $window > "/tmp/vmtest-$l.out" 2>&1; rc=$? - sed -n '/^== /,$p' "/tmp/vmtest-$l.out" - total=$((total + rc)) + bash "$here/vmbootloadertest.sh" "$l" $install $window > "/tmp/vmtest-$l.out" 2>&1; rc=$? + sed -n '/^== /,$p' "/tmp/vmtest-$l.out"; total=$((total + rc)) done -echo; echo "== total: $total failed check(s) for: ${loaders[*]} (advertised: $advertised)" -exit "$total" +for s in "${suites[@]}"; do + bash "$here/vmsuite.sh" "$s" $window > "/tmp/vmtest-$s.out" 2>&1; rc=$? + sed -n '/^== /,$p' "/tmp/vmtest-$s.out"; total=$((total + rc)) +done +echo +echo "SKIP (needs a person, TESTPLAN.md): U1-U9 the app's screens, R2.2 the first desktop login, H-Real the real Steam Machine" +echo "== total: $total failed check(s); boot loaders: ${loaders[*]:-none}; suites: ${suites[*]:-none}" +} 2>&1 | tee "$summary" +exit "$(grep -c '^FAIL' "$summary")" diff --git a/share/vmtest/cli/20-f3-everything-desktop.sh b/share/vmtest/cli/20-f3-everything-desktop.sh old mode 100644 new mode 100755 diff --git a/share/vmtest/cli/30-f4-single-brings-conversion.sh b/share/vmtest/cli/30-f4-single-brings-conversion.sh old mode 100644 new mode 100755 diff --git a/share/vmtest/cli/40-f5-poweroff-brings-machine.sh b/share/vmtest/cli/40-f5-poweroff-brings-machine.sh old mode 100644 new mode 100755 diff --git a/share/vmtest/cli/50-f7-boot-only.sh b/share/vmtest/cli/50-f7-boot-only.sh old mode 100644 new mode 100755 diff --git a/share/vmtest/hw/10-h1-h5-leds-poweroff.sh b/share/vmtest/hw/10-h1-h5-leds-poweroff.sh old mode 100644 new mode 100755 diff --git a/share/vmtest/hw/20-h4-boot-desktop-reboot.host.sh b/share/vmtest/hw/20-h4-boot-desktop-reboot.host.sh old mode 100644 new mode 100755 diff --git a/share/vmtest/hw/30-r1-7-cec.host.sh b/share/vmtest/hw/30-r1-7-cec.host.sh old mode 100644 new mode 100755 diff --git a/share/vmtest/hw/40-h3-bios-dry-run.sh b/share/vmtest/hw/40-h3-bios-dry-run.sh new file mode 100755 index 0000000..08bdbd2 --- /dev/null +++ b/share/vmtest/hw/40-h3-bios-dry-run.sh @@ -0,0 +1,5 @@ +# H3: BIOS update, the whole flow with WIZARD_BIOS_DRY_RUN=1 (never flashes). The VM reports an older BIOS. +# env: BIOS_VERSION=F7F0107 +out=$(printf '11\n\ny\ny\nUPDATE\nm\nq\nn\n' | WIZARD_BIOS_DRY_RUN=1 bash /mnt/steamify.sh 2>&1 | sed 's/\x1b\[[0-9;]*m//g') +grep -qi "dry run" <<< "$out" && pass "H3 the dry run went through" || fail "H3 no dry-run message: $(tail -n 3 <<< "$out" | tr '\n' ' ' | cut -c1-160)" +grep -q '^\[ERROR\]' <<< "$out" && fail "H3 errors: $(grep -B4 -m1 '^\[ERROR\]' <<< "$out" | tr '\n' '|' | cut -c1-400)" || pass "H3 no errors" diff --git a/share/vmtest/installer/10-r2-1-defaults.host.sh b/share/vmtest/installer/10-r2-1-defaults.host.sh new file mode 100755 index 0000000..fb1f8d9 --- /dev/null +++ b/share/vmtest/installer/10-r2-1-defaults.host.sh @@ -0,0 +1,7 @@ +# R2.1: the ISO's installer step: a new user who never logged in gets plain `--defaults` (scripts/vminstallsim.sh). +info "installer simulation: a new user, --defaults (a few minutes)..." +out=$("$repo/scripts/vminstallsim.sh" --fresh 2>&1 | sed 's/\x1b\[[0-9;]*m//g') +grep -q '^exit: 0' <<< "$out" && pass "R2.1 exit 0" || fail "R2.1 $(grep -m1 '^exit' <<< "$out")" +grep -q '^\[ERROR\]' <<< "$out" && fail "R2.1 errors: $(grep -m1 '^\[ERROR\]' <<< "$out")" || pass "R2.1 no errors" +[[ $(grep -c '^Turning on' <<< "$out") -ge 5 ]] && pass "R2.1 every default item turned on ($(grep -c '^Turning on' <<< "$out"))" || fail "R2.1 only $(grep -c '^Turning on' <<< "$out") items turned on" +vm_ssh 'sudo ls /home/isotest/.config/autostart/ 2>/dev/null' | grep -qi steamify && pass "R2.1 the first-login autostart is in place" || fail "R2.1 no first-login autostart" diff --git a/share/vmtest/menu/10-r1-menu.sh b/share/vmtest/menu/10-r1-menu.sh old mode 100644 new mode 100755 diff --git a/share/vmtest/toggles/10-r1-8-matrix.sh b/share/vmtest/toggles/10-r1-8-matrix.sh new file mode 100755 index 0000000..fd13098 --- /dev/null +++ b/share/vmtest/toggles/10-r1-8-matrix.sh @@ -0,0 +1,32 @@ +# R1.8 (and the matrix in the cachyos-vm-testing skill): switch items off and on and check the state each time. +A=gaming,theme,glyphs,single,launcher,notify,cec,machine,poweroff +apply() { info "apply: $1"; sf --defaults --options "$1" >/dev/null; [[ $? -eq 0 ]] && pass "options $1: exit 0" || fail "options $1: failed"; } +apply "$A" +expect_on gaming theme glyphs single launcher notify cec machine poweroff +[[ "$(current_display_manager)" == sddm ]] && pass "baseline: SDDM (single user)" || fail "baseline display manager: $(current_display_manager)" +# theme off +apply "${A/theme,/}" +expect_off theme; expect_on gaming single machine +# theme on again +apply "$A"; expect_on theme +# single user off: plasmalogin +apply "${A/single,/}" +expect_off single; expect_on gaming +[[ "$(current_display_manager)" == plasmalogin ]] && pass "single off: plasmalogin" || fail "single off: display manager $(current_display_manager)" +# single user on: SDDM again +apply "$A"; expect_on single +[[ "$(current_display_manager)" == sddm ]] && pass "single on: back to SDDM" || fail "single on: display manager $(current_display_manager)" +# Steam Machine support and power-off fix off: the driver goes, then comes back +apply gaming,theme,glyphs,single,launcher,notify,cec +expect_off machine poweroff +[[ ! -f /etc/modules-load.d/steamify-fremont-poweroff.conf ]] && pass "machine off: the power-off modules-load entry is gone" || fail "machine off: modules-load entry still there" +apply "$A"; expect_on machine poweroff +[[ $(dkms status | grep -c ': installed') -ge $((3 * $(ls /usr/lib/modules | wc -w))) ]] && pass "machine on: DKMS installed for every kernel" || fail "machine on: DKMS $(dkms status | tr '\n' ';')" +# HDMI-CEC off and on +apply "${A/cec,/}"; expect_off cec +apply "$A"; expect_on cec +# conversion off: single user goes with it, plasmalogin's autologin is CachyOS's again +apply launcher,notify +expect_off gaming single theme +[[ "$(current_display_manager)" == plasmalogin ]] && pass "conversion off: plasmalogin" || fail "conversion off: display manager $(current_display_manager)" +sed -n '/^\[Autologin\]/,/^\[/p' /etc/plasmalogin.conf 2>/dev/null | grep -q 'Session=plasma' && pass "conversion off: plasmalogin autologin is Session=plasma" || skip "conversion off: no plasmalogin Autologin section (a VM from vminstall.sh has no /etc/plasmalogin.conf, see TESTPLAN Known gaps)"