diff --git a/.gitignore b/.gitignore index d347748..276e348 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ share/* !share/vminstall-live.sh !share/vminstall-post.sh !share/bootloader-test/ +!share/vmtest/ # Belt and braces, anywhere in the tree *.iso diff --git a/TODO.md b/TODO.md index cd44ae7..0bbffed 100644 --- a/TODO.md +++ b/TODO.md @@ -33,12 +33,12 @@ TESTPLAN.md means adding its check here (say which row each check covers in a co Automatable (encode as checks): R1.1-R1.6, R1.7 (`vmcec.sh` already prints PASS/FAIL), R1.8 (toggle matrix), R1.9, R1.10, R2.1, G1-G3, F1-F7, H1, H3, H4 (session file), H5, and the boot loader matrix (done). -Not automatable without a person (stay manual, listed in the summary as SKIP): U1-U9 (screenshots), +Not automatable without a person (stay manual, listed in the summary as SKIP; no screenshot testing): U1-U9, R2.2 (first desktop login look), H-Real (real Steam Machine). - [ ] S1. Framework: `scripts/vmtest.sh` takes suites, resets the VM per block, tallies PASS/FAIL/SKIP, prints the skipped manual rows; move the loader matrix under suite `boot` -- [ ] S2. Suite `cli`: G1-G3, F1-F7 (exit codes, `features.state`, boot-desktop unit) +- [x] S2. Suite `cli` (55 checks pass): G1-G3, F1-F7 (exit codes, `features.state`, boot-desktop unit) - [ ] S3. Suite `menu`: R1.1-R1.6, R1.10 with scripted menu input (`vmrun.sh`) and `vmstate.sh` asserts - [ ] S4. Suite `hw`: H1, H3 (`WIZARD_BIOS_DRY_RUN=1`), H4, H5, R1.7 (`vmcec.sh --no-sleep`) - [ ] S5. Suite `installer`: R2.1 (`vminstallsim.sh`) diff --git a/scripts/vmsuite.sh b/scripts/vmsuite.sh new file mode 100755 index 0000000..2798ac6 --- /dev/null +++ b/scripts/vmsuite.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# One suite of the automated test (TESTPLAN.md rows that need no person) on the plain +# CachyOS VM (Plasma, no Steamify: ~/vms/steamify-vm, from vminstall.sh --iso with VM_STEAMIFY=skip). +# scripts/vmsuite.sh [--window] suites: the folders in share/vmtest/ (cli, ...) +# Every block share/vmtest//NN-name.sh starts from a fresh `ssh-ready` (vmreset.sh --fremont), +# runs in the guest as the VM user (repo on /mnt), and prints PASS/FAIL/SKIP lines; the first +# comment lines say which TESTPLAN rows it covers. Log: $VM_DIR..log. Exit status = FAILs. +# Env: ONLY= runs just that block; VM_DIR (default ~/vms/steamify-vm), REPO (default ../steamify-cachyos). +set -uo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(cd "$here/.." && pwd)" +suite="${1:?usage: $0 [--window]}" +[[ "${2:-}" == --window ]] || export VM_HEADLESS=1 +export VM_DIR="${VM_DIR:-$HOME/vms/steamify-vm}" +export REPO="${REPO:-$repo/../steamify-cachyos}" +dir="$repo/share/vmtest/$suite" +[[ -d "$dir" ]] || { echo "No suite $suite (folders in share/vmtest/)" >&2; exit 2; } +. "$here/common.sh" +log="$VM_DIR.$suite.log" +# Headless on a desktop: a Konsole following the log (CI has no desktop; vmtest.sh opens its own). +if [[ -n "${VM_HEADLESS:-}" && -z "${CI:-}${VMTEST_NO_VIEW:-}" && -n "${WAYLAND_DISPLAY:-}${DISPLAY:-}" ]] && command -v konsole >/dev/null; then + systemd-run --user -q --collect --unit="vmsuite-view-$suite-$$" konsole --hold -e tail -n +1 -F "$log" >/dev/null 2>&1 || true +fi +{ +for block in "$dir"/*.sh; do + [[ -z "${ONLY:-}" || "$(basename "$block")" == "$ONLY"* ]] || continue # ONLY=50: just that block + printf '\n##### %s %s\n' "$(date +%T)" "$(basename "$block" .sh)" + "$here/vmreset.sh" --fremont > /tmp/vmsuite-reset.out 2>&1 || { echo "FAIL vmreset.sh failed: $(tail -n 1 /tmp/vmsuite-reset.out)"; continue; } + cat "$repo/share/vmtest/prelude.sh" "$block" | vm_ssh -o ConnectTimeout=6 -o LogLevel=ERROR bash -s 2>&1 | sed 's/\x1b\[[0-9;]*m//g' +done +} 2>&1 | tee "$log" +fails="$(grep -c '^FAIL' "$log")" +echo; echo "== $suite: $(grep -c '^PASS' "$log") passed, $fails failed, $(grep -c '^SKIP' "$log") skipped (log: $log)" +grep '^FAIL' "$log" +exit "$fails" diff --git a/share/vmtest/cli/10-errors-list.sh b/share/vmtest/cli/10-errors-list.sh new file mode 100755 index 0000000..19efebf --- /dev/null +++ b/share/vmtest/cli/10-errors-list.sh @@ -0,0 +1,41 @@ +# G1-G3 (--defaults --list), F1, F2, F6 (bad arguments): read-only, nothing may change. +before=$(ls -R ~/.local/state/steamify 2>/dev/null | md5sum) +rejects() { # rejects : exit 1 and the message + local want="$1"; shift + out=$(bash steamify.sh "$@" 2>&1 < /dev/null); rc=$? + [[ $rc -eq 1 ]] && grep -q "$want" <<< "$out" && pass "steamify.sh $* -> exit 1 ($want)" || fail "steamify.sh $* -> exit $rc: $(sed 's/\x1b\[[0-9;]*m//g' <<< "$out" | grep -m1 -E 'ERROR|rror')" +} +# F1 +rejects "needs a value" --defaults --options +rejects "Unknown item" --defaults --options bogus +rejects "not an item" --defaults --options boot +rejects "can't be part of the install" --defaults --options bios +rejects "needs a value" --defaults --boot +rejects "gamescope or desktop" --defaults --boot sideways +rejects "Unknown option" --defaults --skip x +# F2 +rejects "needs the SteamOS conversion" --defaults --options theme --boot desktop +rejects "needs the SteamOS conversion" --defaults --boot desktop --options theme +# F6 +rejects "needs the SteamOS conversion" --boot desktop +# G1: valid JSON, exit 0, no stderr +out=$(bash steamify.sh --defaults --list 2>/tmp/list.err); rc=$? +[[ $rc -eq 0 ]] && pass "G1 --list exit 0" || fail "G1 --list exit $rc" +[[ ! -s /tmp/list.err ]] && pass "G1 --list prints nothing on stderr" || fail "G1 --list stderr: $(head -c 100 /tmp/list.err)" +python3 - "$out" <<'PY' +import json, sys +def r(ok, msg): print(("PASS " if ok else "FAIL ") + msg) +try: + d = json.loads(sys.argv[1]); r(isinstance(d, list) and d, "G1 --list is a JSON array") +except Exception as e: + r(False, f"G1 --list is not JSON: {e}"); sys.exit() +ids = {i.get("id"): i for i in d} +# G2: actions and retired items are not offered; poweroff needs machine, boot needs gaming +r(not ({"bios", "kpin", "hdmi"} & set(ids)), "G2 bios, kpin and hdmi (actions, retired) are left out") +r(not ({"vram", "steamgame"} & set(ids)), "G2 vram and steamgame (unavailable here) are left out") +r("machine" in str(ids.get("poweroff", {}).get("parent", "")), "G2 poweroff has the parent machine") +r("gaming" in str(ids.get("boot", {}).get("parent", "")), "G2 boot has the parent gaming") +PY +# G3 and nothing changed: the errors and --list must not touch the state +after=$(ls -R ~/.local/state/steamify 2>/dev/null | md5sum) +[[ "$before" == "$after" ]] && pass "F1/F2/F6/G1 changed nothing" || fail "the state directory changed" diff --git a/share/vmtest/cli/20-f3-everything-desktop.sh b/share/vmtest/cli/20-f3-everything-desktop.sh new file mode 100755 index 0000000..5d20de5 --- /dev/null +++ b/share/vmtest/cli/20-f3-everything-desktop.sh @@ -0,0 +1,7 @@ +# F3: every item and --boot desktop: all on, the boot-desktop unit enabled, VRAM left out with a warning. +out=$(sf --defaults --options gaming,theme,glyphs,single,launcher,notify,vram,cec,machine,poweroff --boot desktop); rc=$? +[[ $rc -eq 0 ]] && pass "F3 exit 0" || fail "F3 exit $rc" +expect_on gaming theme glyphs single launcher notify machine poweroff cec +grep -qi "VRAM booster" <<< "$out" && pass "F3 VRAM booster left out with a warning" || fail "F3 no VRAM warning" +is vram && fail "F3 vram is on (the VM has no dmem region)" || pass "F3 vram stays off" +systemctl is-enabled steamify-boot-desktop.service >/dev/null 2>&1 && pass "F3 boot-desktop unit enabled" || fail "F3 boot-desktop unit not enabled" diff --git a/share/vmtest/cli/30-f4-single-brings-conversion.sh b/share/vmtest/cli/30-f4-single-brings-conversion.sh new file mode 100755 index 0000000..b76c60a --- /dev/null +++ b/share/vmtest/cli/30-f4-single-brings-conversion.sh @@ -0,0 +1,6 @@ +# F4: --options theme,single,machine --boot desktop: single brings the conversion; the rest stays off. +sf --defaults --options theme,single,machine --boot desktop >/dev/null; rc=$? +[[ $rc -eq 0 ]] && pass "F4 exit 0" || fail "F4 exit $rc" +expect_on gaming theme single machine +expect_off poweroff cec glyphs launcher notify +systemctl is-enabled steamify-boot-desktop.service >/dev/null 2>&1 && pass "F4 boot-desktop unit enabled" || fail "F4 boot-desktop unit not enabled" diff --git a/share/vmtest/cli/40-f5-poweroff-brings-machine.sh b/share/vmtest/cli/40-f5-poweroff-brings-machine.sh new file mode 100755 index 0000000..096eddc --- /dev/null +++ b/share/vmtest/cli/40-f5-poweroff-brings-machine.sh @@ -0,0 +1,7 @@ +# F5: --options launcher,vram,poweroff: poweroff brings Steam Machine support; conversion off, plasmalogin unchanged. +dm_before=$(current_display_manager) +sf --defaults --options launcher,vram,poweroff >/dev/null; rc=$? +[[ $rc -eq 0 ]] && pass "F5 exit 0" || fail "F5 exit $rc" +expect_on machine poweroff launcher +expect_off gaming +[[ "$(current_display_manager)" == "$dm_before" ]] && pass "F5 display manager unchanged ($dm_before)" || fail "F5 display manager changed: $dm_before -> $(current_display_manager)" diff --git a/share/vmtest/cli/50-f7-boot-only.sh b/share/vmtest/cli/50-f7-boot-only.sh new file mode 100755 index 0000000..a5d5908 --- /dev/null +++ b/share/vmtest/cli/50-f7-boot-only.sh @@ -0,0 +1,12 @@ +# F7: on an install, --boot desktop, again, --boot gamescope: only Boot into changes; the menu ticks stay. +sf --defaults --options gaming,theme,single,launcher,notify,machine,poweroff >/dev/null +base=$(ticks | sed 's/boot //') +sf --boot desktop >/dev/null; rc=$? +[[ $rc -eq 0 ]] && pass "F7 --boot desktop exit 0" || fail "F7 --boot desktop exit $rc" +systemctl is-enabled steamify-boot-desktop.service >/dev/null 2>&1 && pass "F7 boot-desktop unit enabled" || fail "F7 unit not enabled after --boot desktop" +[[ "$(ticks | sed 's/boot //')" == "$base" ]] && pass "F7 the menu ticks are unchanged" || fail "F7 ticks changed: [$base] -> [$(ticks)]" +out=$(sf --boot desktop); grep -qi "already" <<< "$out" && pass "F7 second --boot desktop says already" || fail "F7 no 'Already starting in desktop': $(tail -n 2 <<< "$out" | tr '\n' ' ')" +sf --boot gamescope >/dev/null; rc=$? +[[ $rc -eq 0 ]] && pass "F7 --boot gamescope exit 0" || fail "F7 --boot gamescope exit $rc" +systemctl is-enabled steamify-boot-desktop.service >/dev/null 2>&1 && fail "F7 unit still enabled after --boot gamescope" || pass "F7 boot-desktop unit disabled again" +[[ "$(ticks | sed 's/boot //')" == "$base" ]] && pass "F7 ticks unchanged after --boot gamescope" || fail "F7 ticks changed after gamescope" diff --git a/share/vmtest/prelude.sh b/share/vmtest/prelude.sh new file mode 100644 index 0000000..91d9997 --- /dev/null +++ b/share/vmtest/prelude.sh @@ -0,0 +1,16 @@ +# Sent to the guest in front of every block (scripts/vmsuite.sh): helpers and Steamify's libs. +# ssh has no Plasma session: the theme item (Vapor layout) needs one (vmrun.sh does the same). +export XDG_RUNTIME_DIR=/run/user/$(id -u) WAYLAND_DISPLAY=wayland-0 DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus +pass() { echo "PASS $*"; }; fail() { echo "FAIL $*"; }; skip() { echo "SKIP $*"; } +sudo mountpoint -q /mnt || sudo mount -t 9p -o trans=virtio,version=9p2000.L repo /mnt +cd /mnt || exit 1 +for lib in common state packages login-manager single-user steam-desktop steam-machine fremont-poweroff vram-booster hdmi-refresh cec boot-session vapor-theme steamos-extras bios desktop-shortcut wizard-shortcut steam-game update-notifier; do + # shellcheck source=/dev/null + . "lib/$lib.sh" 2>/dev/null +done +sf() { bash /mnt/steamify.sh "$@" < /dev/null 2>&1 | sed 's/\x1b\[[0-9;]*m//g'; return "${PIPESTATUS[0]}"; } +# is : Steamify's own status function (what the menu ticks); isnt +is() { "${1}_status" 2>/dev/null; } +expect_on() { local i; for i in "$@"; do is "$i" && pass "$i is on" || fail "$i is off"; done; } +expect_off() { local i; for i in "$@"; do is "$i" && fail "$i is on" || pass "$i is off"; done; } +ticks() { local i; for i in gaming theme glyphs single launcher notify vram cec machine poweroff boot; do is "$i" && printf '%s ' "$i"; done; echo; }