mirror of
https://github.com/theupriser/steamify-cachyos-dev.git
synced 2026-10-05 12:45:56 +02:00
feat: vmsuite.sh and the cli suite: G1-G3, F1-F7 of TESTPLAN.md as unattended checks
Each block in share/vmtest/<suite>/ starts from a fresh ssh-ready snapshot, gets a Plasma session environment over ssh (the theme item needs one) and prints PASS/FAIL lines. ONLY=<prefix> runs one block. 55 checks pass on the plain CachyOS VM.
This commit is contained in:
theupriser committed
1 parent
508eb68ed5
commit
fe5251f758
9 files changed
+127
-2
No files matched your search
@@ -18,6 +18,7 @@ share/*
|
||||
!share/vminstall-live.sh
|
||||
!share/vminstall-post.sh
|
||||
!share/bootloader-test/
|
||||
!share/vmtest/
|
||||
|
||||
# Belt and braces, anywhere in the tree
|
||||
*.iso
|
||||
|
||||
@@ -33,12 +33,12 @@ TESTPLAN.md means adding its check here (say which row each check covers in a co
|
||||
|
||||
Automatable (encode as checks): R1.1-R1.6, R1.7 (`vmcec.sh` already prints PASS/FAIL), R1.8 (toggle matrix),
|
||||
R1.9, R1.10, R2.1, G1-G3, F1-F7, H1, H3, H4 (session file), H5, and the boot loader matrix (done).
|
||||
Not automatable without a person (stay manual, listed in the summary as SKIP): U1-U9 (screenshots),
|
||||
Not automatable without a person (stay manual, listed in the summary as SKIP; no screenshot testing): U1-U9,
|
||||
R2.2 (first desktop login look), H-Real (real Steam Machine).
|
||||
|
||||
- [ ] S1. Framework: `scripts/vmtest.sh` takes suites, resets the VM per block, tallies PASS/FAIL/SKIP,
|
||||
prints the skipped manual rows; move the loader matrix under suite `boot`
|
||||
- [ ] S2. Suite `cli`: G1-G3, F1-F7 (exit codes, `features.state`, boot-desktop unit)
|
||||
- [x] S2. Suite `cli` (55 checks pass): G1-G3, F1-F7 (exit codes, `features.state`, boot-desktop unit)
|
||||
- [ ] S3. Suite `menu`: R1.1-R1.6, R1.10 with scripted menu input (`vmrun.sh`) and `vmstate.sh` asserts
|
||||
- [ ] S4. Suite `hw`: H1, H3 (`WIZARD_BIOS_DRY_RUN=1`), H4, H5, R1.7 (`vmcec.sh --no-sleep`)
|
||||
- [ ] S5. Suite `installer`: R2.1 (`vminstallsim.sh`)
|
||||
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
# One suite of the automated test (TESTPLAN.md rows that need no person) on the plain
|
||||
# CachyOS VM (Plasma, no Steamify: ~/vms/steamify-vm, from vminstall.sh --iso <CachyOS ISO> with VM_STEAMIFY=skip).
|
||||
# scripts/vmsuite.sh <suite> [--window] suites: the folders in share/vmtest/ (cli, ...)
|
||||
# Every block share/vmtest/<suite>/NN-name.sh starts from a fresh `ssh-ready` (vmreset.sh --fremont),
|
||||
# runs in the guest as the VM user (repo on /mnt), and prints PASS/FAIL/SKIP lines; the first
|
||||
# comment lines say which TESTPLAN rows it covers. Log: $VM_DIR.<suite>.log. Exit status = FAILs.
|
||||
# Env: ONLY=<block name prefix> runs just that block; VM_DIR (default ~/vms/steamify-vm), REPO (default ../steamify-cachyos).
|
||||
set -uo pipefail
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
repo="$(cd "$here/.." && pwd)"
|
||||
suite="${1:?usage: $0 <suite> [--window]}"
|
||||
[[ "${2:-}" == --window ]] || export VM_HEADLESS=1
|
||||
export VM_DIR="${VM_DIR:-$HOME/vms/steamify-vm}"
|
||||
export REPO="${REPO:-$repo/../steamify-cachyos}"
|
||||
dir="$repo/share/vmtest/$suite"
|
||||
[[ -d "$dir" ]] || { echo "No suite $suite (folders in share/vmtest/)" >&2; exit 2; }
|
||||
. "$here/common.sh"
|
||||
log="$VM_DIR.$suite.log"
|
||||
# Headless on a desktop: a Konsole following the log (CI has no desktop; vmtest.sh opens its own).
|
||||
if [[ -n "${VM_HEADLESS:-}" && -z "${CI:-}${VMTEST_NO_VIEW:-}" && -n "${WAYLAND_DISPLAY:-}${DISPLAY:-}" ]] && command -v konsole >/dev/null; then
|
||||
systemd-run --user -q --collect --unit="vmsuite-view-$suite-$$" konsole --hold -e tail -n +1 -F "$log" >/dev/null 2>&1 || true
|
||||
fi
|
||||
{
|
||||
for block in "$dir"/*.sh; do
|
||||
[[ -z "${ONLY:-}" || "$(basename "$block")" == "$ONLY"* ]] || continue # ONLY=50: just that block
|
||||
printf '\n##### %s %s\n' "$(date +%T)" "$(basename "$block" .sh)"
|
||||
"$here/vmreset.sh" --fremont > /tmp/vmsuite-reset.out 2>&1 || { echo "FAIL vmreset.sh failed: $(tail -n 1 /tmp/vmsuite-reset.out)"; continue; }
|
||||
cat "$repo/share/vmtest/prelude.sh" "$block" | vm_ssh -o ConnectTimeout=6 -o LogLevel=ERROR bash -s 2>&1 | sed 's/\x1b\[[0-9;]*m//g'
|
||||
done
|
||||
} 2>&1 | tee "$log"
|
||||
fails="$(grep -c '^FAIL' "$log")"
|
||||
echo; echo "== $suite: $(grep -c '^PASS' "$log") passed, $fails failed, $(grep -c '^SKIP' "$log") skipped (log: $log)"
|
||||
grep '^FAIL' "$log"
|
||||
exit "$fails"
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
# G1-G3 (--defaults --list), F1, F2, F6 (bad arguments): read-only, nothing may change.
|
||||
before=$(ls -R ~/.local/state/steamify 2>/dev/null | md5sum)
|
||||
rejects() { # rejects <expected message> <args...>: exit 1 and the message
|
||||
local want="$1"; shift
|
||||
out=$(bash steamify.sh "$@" 2>&1 < /dev/null); rc=$?
|
||||
[[ $rc -eq 1 ]] && grep -q "$want" <<< "$out" && pass "steamify.sh $* -> exit 1 ($want)" || fail "steamify.sh $* -> exit $rc: $(sed 's/\x1b\[[0-9;]*m//g' <<< "$out" | grep -m1 -E 'ERROR|rror')"
|
||||
}
|
||||
# F1
|
||||
rejects "needs a value" --defaults --options
|
||||
rejects "Unknown item" --defaults --options bogus
|
||||
rejects "not an item" --defaults --options boot
|
||||
rejects "can't be part of the install" --defaults --options bios
|
||||
rejects "needs a value" --defaults --boot
|
||||
rejects "gamescope or desktop" --defaults --boot sideways
|
||||
rejects "Unknown option" --defaults --skip x
|
||||
# F2
|
||||
rejects "needs the SteamOS conversion" --defaults --options theme --boot desktop
|
||||
rejects "needs the SteamOS conversion" --defaults --boot desktop --options theme
|
||||
# F6
|
||||
rejects "needs the SteamOS conversion" --boot desktop
|
||||
# G1: valid JSON, exit 0, no stderr
|
||||
out=$(bash steamify.sh --defaults --list 2>/tmp/list.err); rc=$?
|
||||
[[ $rc -eq 0 ]] && pass "G1 --list exit 0" || fail "G1 --list exit $rc"
|
||||
[[ ! -s /tmp/list.err ]] && pass "G1 --list prints nothing on stderr" || fail "G1 --list stderr: $(head -c 100 /tmp/list.err)"
|
||||
python3 - "$out" <<'PY'
|
||||
import json, sys
|
||||
def r(ok, msg): print(("PASS " if ok else "FAIL ") + msg)
|
||||
try:
|
||||
d = json.loads(sys.argv[1]); r(isinstance(d, list) and d, "G1 --list is a JSON array")
|
||||
except Exception as e:
|
||||
r(False, f"G1 --list is not JSON: {e}"); sys.exit()
|
||||
ids = {i.get("id"): i for i in d}
|
||||
# G2: actions and retired items are not offered; poweroff needs machine, boot needs gaming
|
||||
r(not ({"bios", "kpin", "hdmi"} & set(ids)), "G2 bios, kpin and hdmi (actions, retired) are left out")
|
||||
r(not ({"vram", "steamgame"} & set(ids)), "G2 vram and steamgame (unavailable here) are left out")
|
||||
r("machine" in str(ids.get("poweroff", {}).get("parent", "")), "G2 poweroff has the parent machine")
|
||||
r("gaming" in str(ids.get("boot", {}).get("parent", "")), "G2 boot has the parent gaming")
|
||||
PY
|
||||
# G3 and nothing changed: the errors and --list must not touch the state
|
||||
after=$(ls -R ~/.local/state/steamify 2>/dev/null | md5sum)
|
||||
[[ "$before" == "$after" ]] && pass "F1/F2/F6/G1 changed nothing" || fail "the state directory changed"
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
# F3: every item and --boot desktop: all on, the boot-desktop unit enabled, VRAM left out with a warning.
|
||||
out=$(sf --defaults --options gaming,theme,glyphs,single,launcher,notify,vram,cec,machine,poweroff --boot desktop); rc=$?
|
||||
[[ $rc -eq 0 ]] && pass "F3 exit 0" || fail "F3 exit $rc"
|
||||
expect_on gaming theme glyphs single launcher notify machine poweroff cec
|
||||
grep -qi "VRAM booster" <<< "$out" && pass "F3 VRAM booster left out with a warning" || fail "F3 no VRAM warning"
|
||||
is vram && fail "F3 vram is on (the VM has no dmem region)" || pass "F3 vram stays off"
|
||||
systemctl is-enabled steamify-boot-desktop.service >/dev/null 2>&1 && pass "F3 boot-desktop unit enabled" || fail "F3 boot-desktop unit not enabled"
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
# F4: --options theme,single,machine --boot desktop: single brings the conversion; the rest stays off.
|
||||
sf --defaults --options theme,single,machine --boot desktop >/dev/null; rc=$?
|
||||
[[ $rc -eq 0 ]] && pass "F4 exit 0" || fail "F4 exit $rc"
|
||||
expect_on gaming theme single machine
|
||||
expect_off poweroff cec glyphs launcher notify
|
||||
systemctl is-enabled steamify-boot-desktop.service >/dev/null 2>&1 && pass "F4 boot-desktop unit enabled" || fail "F4 boot-desktop unit not enabled"
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
# F5: --options launcher,vram,poweroff: poweroff brings Steam Machine support; conversion off, plasmalogin unchanged.
|
||||
dm_before=$(current_display_manager)
|
||||
sf --defaults --options launcher,vram,poweroff >/dev/null; rc=$?
|
||||
[[ $rc -eq 0 ]] && pass "F5 exit 0" || fail "F5 exit $rc"
|
||||
expect_on machine poweroff launcher
|
||||
expect_off gaming
|
||||
[[ "$(current_display_manager)" == "$dm_before" ]] && pass "F5 display manager unchanged ($dm_before)" || fail "F5 display manager changed: $dm_before -> $(current_display_manager)"
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
# F7: on an install, --boot desktop, again, --boot gamescope: only Boot into changes; the menu ticks stay.
|
||||
sf --defaults --options gaming,theme,single,launcher,notify,machine,poweroff >/dev/null
|
||||
base=$(ticks | sed 's/boot //')
|
||||
sf --boot desktop >/dev/null; rc=$?
|
||||
[[ $rc -eq 0 ]] && pass "F7 --boot desktop exit 0" || fail "F7 --boot desktop exit $rc"
|
||||
systemctl is-enabled steamify-boot-desktop.service >/dev/null 2>&1 && pass "F7 boot-desktop unit enabled" || fail "F7 unit not enabled after --boot desktop"
|
||||
[[ "$(ticks | sed 's/boot //')" == "$base" ]] && pass "F7 the menu ticks are unchanged" || fail "F7 ticks changed: [$base] -> [$(ticks)]"
|
||||
out=$(sf --boot desktop); grep -qi "already" <<< "$out" && pass "F7 second --boot desktop says already" || fail "F7 no 'Already starting in desktop': $(tail -n 2 <<< "$out" | tr '\n' ' ')"
|
||||
sf --boot gamescope >/dev/null; rc=$?
|
||||
[[ $rc -eq 0 ]] && pass "F7 --boot gamescope exit 0" || fail "F7 --boot gamescope exit $rc"
|
||||
systemctl is-enabled steamify-boot-desktop.service >/dev/null 2>&1 && fail "F7 unit still enabled after --boot gamescope" || pass "F7 boot-desktop unit disabled again"
|
||||
[[ "$(ticks | sed 's/boot //')" == "$base" ]] && pass "F7 ticks unchanged after --boot gamescope" || fail "F7 ticks changed after gamescope"
|
||||
@@ -0,0 +1,16 @@
|
||||
# Sent to the guest in front of every block (scripts/vmsuite.sh): helpers and Steamify's libs.
|
||||
# ssh has no Plasma session: the theme item (Vapor layout) needs one (vmrun.sh does the same).
|
||||
export XDG_RUNTIME_DIR=/run/user/$(id -u) WAYLAND_DISPLAY=wayland-0 DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus
|
||||
pass() { echo "PASS $*"; }; fail() { echo "FAIL $*"; }; skip() { echo "SKIP $*"; }
|
||||
sudo mountpoint -q /mnt || sudo mount -t 9p -o trans=virtio,version=9p2000.L repo /mnt
|
||||
cd /mnt || exit 1
|
||||
for lib in common state packages login-manager single-user steam-desktop steam-machine fremont-poweroff vram-booster hdmi-refresh cec boot-session vapor-theme steamos-extras bios desktop-shortcut wizard-shortcut steam-game update-notifier; do
|
||||
# shellcheck source=/dev/null
|
||||
. "lib/$lib.sh" 2>/dev/null
|
||||
done
|
||||
sf() { bash /mnt/steamify.sh "$@" < /dev/null 2>&1 | sed 's/\x1b\[[0-9;]*m//g'; return "${PIPESTATUS[0]}"; }
|
||||
# is <item>: Steamify's own status function (what the menu ticks); isnt <item>
|
||||
is() { "${1}_status" 2>/dev/null; }
|
||||
expect_on() { local i; for i in "$@"; do is "$i" && pass "$i is on" || fail "$i is off"; done; }
|
||||
expect_off() { local i; for i in "$@"; do is "$i" && fail "$i is on" || pass "$i is off"; done; }
|
||||
ticks() { local i; for i in gaming theme glyphs single launcher notify vram cec machine poweroff boot; do is "$i" && printf '%s ' "$i"; done; echo; }
|
||||
Reference in new issue
Block a user