mirror of
https://github.com/theupriser/steamify-cachyos-dev.git
synced 2026-10-05 12:45:56 +02:00
feat: faster and parallel tests, a remote runner, and a CLAUDE.md hand-off
vmsuite.sh: the ssh-ready snapshot becomes one base image, every block gets a new qcow2 overlay (no snapshot restore), each suite has its own VM directory and ssh port (2301-2305, boot loaders 2401-2403), the host's package cache is bound over pacman's, VM_MEM defaults to 4G for tests (run.sh VM_MEM, default 8G). vmtest.sh runs up to MAX_PARALLEL=3 jobs at once, slowest first. scripts/vmtest-remote.sh runs vmtest.sh on another machine (the PC) over ssh and prints the summary. CLAUDE.md and the branch cleanup skill now live in the repo. Not proven as a whole: three VMs at once and the base image work, but a complete parallel run to the end has not finished yet (P1/P2 in TODO.md); the remote runner and WSL2 are untried (R1-R2).
This commit is contained in:
theupriser committed
1 parent
b895e28181
commit
f24ff3927e
10 files changed
+204
-26
No files matched your search
@@ -23,6 +23,8 @@ done
|
||||
$window || export VM_HEADLESS=1
|
||||
case "$loader" in limine|systemd-boot|grub) ;; *) echo "Unknown boot loader: $loader" >&2; exit 2 ;; esac
|
||||
export VM_DIR="${VM_DIR:-$HOME/vms/bl-$loader}"
|
||||
# Own ssh port per loader, so the loaders can run side by side (vmtest.sh).
|
||||
case "$loader" in limine) export VM_PORT="${VM_PORT:-2401}" ;; systemd-boot) export VM_PORT="${VM_PORT:-2402}" ;; grub) export VM_PORT="${VM_PORT:-2403}" ;; esac
|
||||
. "$here/common.sh"
|
||||
guest="$repo/share/bootloader-test"
|
||||
log="$VM_DIR.test.log"
|
||||
@@ -31,16 +33,16 @@ step() { printf '\n##### %s %s\n' "$(date +%T)" "$*"; }
|
||||
gssh() { vm_ssh -o ConnectTimeout=6 -o LogLevel=ERROR "$@"; }
|
||||
gscript() { gssh env "LOADER=$loader" bash -s < "$guest/$1"; } # the guest's login shell is fish
|
||||
waitssh() { local _; for _ in $(seq 60); do gssh 'uptime -p' 2>/dev/null | grep -q '^up' && return 0; sleep 5; done; echo "FAIL no SSH"; return 1; }
|
||||
running() { pgrep -f '^qemu-system' >/dev/null; }
|
||||
running() { pgrep -f "[h]ostfwd=tcp::$VM_PORT-" >/dev/null; }
|
||||
stop_vm() {
|
||||
running || return 0
|
||||
# Whichever VM runs: its key and host key are not this VM's.
|
||||
ssh -p "$VM_PORT" -i "$VM_SSH_KEY" -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR \
|
||||
"$VM_USER@$VM_HOST" 'sudo systemctl poweroff' >/dev/null 2>&1
|
||||
local _; for _ in $(seq 30); do running || return 0; sleep 2; done
|
||||
pkill -f '^qemu-system'; sleep 2
|
||||
pkill -f "[h]ostfwd=tcp::$VM_PORT-"; sleep 2
|
||||
}
|
||||
start_vm() { stop_vm; cp "$repo/run.sh" "$VM_DIR/run.sh"; (cd "$VM_DIR" && REPO="${REPO:-$repo/../steamify-cachyos}" setsid nohup ./run.sh --fremont > vm.log 2>&1 &); }
|
||||
start_vm() { stop_vm; cp "$repo/run.sh" "$VM_DIR/run.sh"; (cd "$VM_DIR" && VM_MEM="${VM_MEM:-4G}" REPO="${REPO:-$repo/../steamify-cachyos}" setsid nohup ./run.sh --fremont > vm.log 2>&1 &); }
|
||||
reboot_vm() { gssh 'sudo systemctl reboot' >/dev/null 2>&1; sleep 25; waitssh; }
|
||||
view_start # one shared Konsole (common.sh)
|
||||
# The guest checks print PASS/FAIL lines; the summary at the end counts them in the log.
|
||||
@@ -110,7 +112,7 @@ b="$(gssh uname -r)"
|
||||
|
||||
step "DONE"
|
||||
stop_vm
|
||||
} 2>&1 | tee "$log" | tee -a "$TEST_LOG" | sed 's/\x1b\[[0-9;]*m//g'
|
||||
} 2>&1 | tee "$log" | tee >(sed -u "s/^/[$loader] /" >> "$TEST_LOG") | sed 's/\x1b\[[0-9;]*m//g'
|
||||
fails="$(grep -c '^FAIL' "$log")"
|
||||
echo; echo "== $loader: $(grep -c '^PASS' "$log") passed, $fails failed (log: $log)"
|
||||
grep '^FAIL' "$log"
|
||||
|
||||
@@ -46,7 +46,7 @@ mkdir -p "$VM_DIR"
|
||||
iso="${iso:-$VM_DIR/cachyos.iso}"
|
||||
[[ -f "$iso" ]] || { echo "No ISO at $iso: run get-iso.sh (in $VM_DIR) or pass --iso <file>." >&2; exit 1; }
|
||||
iso="$(cd "$(dirname "$iso")" && pwd)/$(basename "$iso")"
|
||||
pgrep -f '^qemu-system' >/dev/null && { echo "A VM is running; power it off first (both use port $VM_PORT)." >&2; exit 1; }
|
||||
pgrep -f "[h]ostfwd=tcp::$VM_PORT-" >/dev/null && { echo "A VM is running on port $VM_PORT; power it off first." >&2; exit 1; }
|
||||
if [[ -f "$VM_DIR/disk.qcow2" ]]; then
|
||||
[[ "$force" == true ]] || { echo "$VM_DIR already has a disk.qcow2; use --force to replace it, or another VM_DIR." >&2; exit 1; }
|
||||
read -r -p "Replace $VM_DIR/disk.qcow2 and all its snapshots? Type YES: " answer < /dev/tty
|
||||
|
||||
+76
-13
@@ -1,37 +1,100 @@
|
||||
#!/bin/bash
|
||||
# One suite of the automated test (TESTPLAN.md rows that need no person) on the plain
|
||||
# CachyOS VM (Plasma, no Steamify: ~/vms/steamify-vm, from vminstall.sh --iso <CachyOS ISO> with VM_STEAMIFY=skip).
|
||||
# One suite of the automated test (TESTPLAN.md rows that need no person) on the plain CachyOS VM
|
||||
# (Plasma, no Steamify: ~/vms/steamify-vm, from vminstall.sh --iso <CachyOS ISO> with VM_STEAMIFY=skip).
|
||||
# scripts/vmsuite.sh <suite> [--window] suites: the folders in share/vmtest/ (cli, ...)
|
||||
# Every block share/vmtest/<suite>/NN-name.sh starts from a fresh `ssh-ready` (vmreset.sh --fremont),
|
||||
# runs in the guest as the VM user (repo on /mnt; NN-name.host.sh blocks run on the host instead), and prints PASS/FAIL/SKIP lines; the first
|
||||
# comment lines say which TESTPLAN rows it covers; `# env: VAR=value` there is set for the VM start (e.g. BIOS_VERSION=F7F0107). Log: $VM_DIR.<suite>.log. Exit status = FAILs.
|
||||
# Env: ONLY=<block name prefix> runs just that block; VM_DIR (default ~/vms/steamify-vm), REPO (default ../steamify-cachyos).
|
||||
# scripts/vmsuite.sh --prepare only make the base image (below)
|
||||
# Every block share/vmtest/<suite>/NN-name.sh starts from a fresh copy of the VM's `ssh-ready` state,
|
||||
# runs in the guest as the VM user (repo on /mnt; NN-name.host.sh blocks run on the host instead: reboots,
|
||||
# host scripts), and prints PASS/FAIL/SKIP lines; the first comment lines say which TESTPLAN rows it
|
||||
# covers; `# env: VAR=value` there is set for the VM start (e.g. BIOS_VERSION=F7F0107).
|
||||
# Fast and parallel: the `ssh-ready` snapshot is converted once into $SRC/base-ssh-ready.qcow2; each suite
|
||||
# has its own directory ~/vms/run-<suite> and ssh port, where every block gets a new qcow2 overlay of the
|
||||
# base (instant, no snapshot restore), and the host's package cache (VM_CACHE, ~/vms/pkg-cache) is bound
|
||||
# over pacman's cache, so packages are downloaded once for all. Log: $VM_DIR.log. Exit status = FAILs.
|
||||
# Env: ONLY=<block name prefix> runs just that block; SRC (default ~/vms/steamify-vm); REPO (default ../steamify-cachyos).
|
||||
set -uo pipefail
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
repo="$(cd "$here/.." && pwd)"
|
||||
suite="${1:?usage: $0 <suite> [--window]}"
|
||||
src="${SRC:-$HOME/vms/steamify-vm}"
|
||||
base="$src/base-ssh-ready.qcow2"
|
||||
|
||||
prepare_base() { # once, under a lock (parallel suites): the ssh-ready snapshot as a stand-alone image
|
||||
( flock 9
|
||||
if [[ ! -f "$base" || "$src/vars.ssh-ready.fd" -nt "$base" ]]; then
|
||||
echo "Making $base from the ssh-ready snapshot (once, a few minutes)..." >&2
|
||||
qemu-img convert -f qcow2 -O qcow2 -l snapshot.name=ssh-ready "$src/disk.qcow2" "$base.tmp" && mv "$base.tmp" "$base"
|
||||
fi ) 9>"$src/.base.lock"
|
||||
[[ -f "$base" ]]
|
||||
}
|
||||
if [[ "${1:-}" == --prepare ]]; then prepare_base; exit $?; fi
|
||||
|
||||
suite="${1:?usage: $0 <suite> [--window] | --prepare}"
|
||||
[[ "${2:-}" == --window ]] || export VM_HEADLESS=1
|
||||
export VM_DIR="${VM_DIR:-$HOME/vms/steamify-vm}"
|
||||
export REPO="${REPO:-$repo/../steamify-cachyos}"
|
||||
dir="$repo/share/vmtest/$suite"
|
||||
[[ -d "$dir" ]] || { echo "No suite $suite (folders in share/vmtest/)" >&2; exit 2; }
|
||||
case "$suite" in cli) port=2301 ;; menu) port=2302 ;; hw) port=2303 ;; installer) port=2304 ;; toggles) port=2305 ;; *) port=2390 ;; esac
|
||||
export VM_PORT="${VM_PORT:-$port}"
|
||||
export VM_DIR="$HOME/vms/run-$suite"
|
||||
export REPO="${REPO:-$repo/../steamify-cachyos}"
|
||||
export VM_CACHE="${VM_CACHE-$HOME/vms/pkg-cache}"
|
||||
export VM_MEM="${VM_MEM:-4G}" # three VMs at once next to a desktop: 8G each pushed the host into swap
|
||||
prepare_base || { echo "FAIL no base image" >&2; exit 1; }
|
||||
mkdir -p "$VM_DIR"
|
||||
for f in vm-user ssh-key; do cp "$src/$f" "$VM_DIR/$f"; done
|
||||
echo "$REPO" > "$VM_DIR/repo-path"; [[ -e "$VM_DIR/share" ]] || ln -s "$repo/share" "$VM_DIR/share"
|
||||
. "$here/common.sh"
|
||||
log="$VM_DIR.$suite.log"
|
||||
log="$VM_DIR.log"
|
||||
view_start # one shared Konsole (common.sh), headless on a desktop
|
||||
|
||||
vm_running() { pgrep -f "[h]ostfwd=tcp::$VM_PORT-" >/dev/null; }
|
||||
vm_stop() {
|
||||
vm_running || return 0
|
||||
vm_ssh -o ConnectTimeout=4 'sudo systemctl poweroff' >/dev/null 2>&1
|
||||
local _; for _ in $(seq 20); do vm_running || return 0; sleep 1; done
|
||||
pkill -f "[h]ostfwd=tcp::$VM_PORT-"; sleep 2
|
||||
}
|
||||
# A fresh VM for a block: new overlay of the base, the snapshot's UEFI vars, Plasma up, repo mounted, package cache bound.
|
||||
vm_fresh() { # vm_fresh [VAR=value...]: extra environment for the VM start (BIOS_VERSION=...)
|
||||
vm_stop
|
||||
cp "$repo/run.sh" "$VM_DIR/run.sh"; cp "$src/vars.ssh-ready.fd" "$VM_DIR/vars.fd"
|
||||
rm -f "$VM_DIR/disk.qcow2" "$VM_DIR/known_hosts"
|
||||
qemu-img create -q -f qcow2 -b "$base" -F qcow2 "$VM_DIR/disk.qcow2" || return 1
|
||||
(cd "$VM_DIR" && env "$@" nohup ./run.sh --fremont > vm.log 2>&1 &)
|
||||
local _; for _ in $(seq 100); do vm_ssh -o ConnectTimeout=3 true 2>/dev/null && break; sleep 3; done
|
||||
vm_ssh -o LogLevel=ERROR bash -s <<'REMOTE'
|
||||
sudo mount -t 9p -o trans=virtio,version=9p2000.L repo /mnt
|
||||
sudo mkdir -p /etc/plasmalogin.conf.d
|
||||
printf '[Autologin]\nUser=%s\nSession=plasma.desktop\n' "$USER" | sudo tee /etc/plasmalogin.conf.d/00-test-autologin.conf >/dev/null
|
||||
for _ in $(seq 10); do pgrep -u "$USER" -x plasmashell >/dev/null && break; sleep 3; done
|
||||
pgrep -u "$USER" -x plasmashell >/dev/null || { sudo systemctl reset-failed plasmalogin; sudo systemctl restart plasmalogin; }
|
||||
for _ in $(seq 40); do pgrep -u "$USER" -x plasmashell >/dev/null && break; sleep 3; done
|
||||
pkill -u "$USER" systemsettings; pkill -u "$USER" cachyos-hello
|
||||
for f in /etc/pacman.d/*mirrorlist*; do sudo sed -i '/krfoss/s/^Server/#Server/' "$f"; done
|
||||
# the host's package cache over pacman's
|
||||
sudo mkdir -p /var/cache/steamify-pkg
|
||||
if sudo mount -t 9p -o trans=virtio,version=9p2000.L cache /var/cache/steamify-pkg 2>/dev/null; then
|
||||
sudo mkdir -p /var/cache/steamify-pkg/pkg; sudo mount --bind /var/cache/steamify-pkg/pkg /var/cache/pacman/pkg
|
||||
fi
|
||||
pgrep -u "$USER" -x plasmashell >/dev/null && echo "Logged in to Plasma." || { echo "Plasma did not start." >&2; exit 1; }
|
||||
REMOTE
|
||||
}
|
||||
|
||||
{
|
||||
printf '\n===== vmsuite.sh %s (%s) =====\n' "$suite" "$(date +%T)"
|
||||
for block in "$dir"/*.sh; do
|
||||
[[ -z "${ONLY:-}" || "$(basename "$block")" == "$ONLY"* ]] || continue # ONLY=50: just that block
|
||||
printf '\n##### %s %s\n' "$(date +%T)" "$(basename "$block" .sh)"
|
||||
env $(sed -n 's/^# env: //p' "$block" | head -n 1) "$here/vmreset.sh" --fremont > /tmp/vmsuite-reset.out 2>&1 || { echo "FAIL vmreset.sh failed: $(tail -n 1 /tmp/vmsuite-reset.out)"; continue; }
|
||||
vm_fresh $(sed -n 's/^# env: //p' "$block" | head -n 1) > /tmp/vmsuite-$suite-reset.out 2>&1 \
|
||||
|| { echo "FAIL the VM did not come up: $(tail -n 1 /tmp/vmsuite-$suite-reset.out)"; continue; }
|
||||
if [[ "$block" == *.host.sh ]]; then # runs on the host (reboots, host scripts); prelude-host.sh has the helpers
|
||||
( . "$repo/share/vmtest/prelude-host.sh"; . "$block" ) 2>&1 | sed -u 's/\x1b\[[0-9;]*m//g'
|
||||
continue
|
||||
fi
|
||||
cat "$repo/share/vmtest/prelude.sh" "$block" | vm_ssh -o ConnectTimeout=6 -o LogLevel=ERROR bash -s 2>&1 | sed -u 's/\x1b\[[0-9;]*m//g'
|
||||
done
|
||||
} 2>&1 | tee "$log" | tee -a "$TEST_LOG"
|
||||
vm_stop
|
||||
} 2>&1 | tee "$log" | sed -u "s/^/[$suite] /" >> "$TEST_LOG"
|
||||
fails="$(grep -c '^FAIL' "$log")"
|
||||
echo; echo "== $suite: $(grep -c '^PASS' "$log") passed, $fails failed, $(grep -c '^SKIP' "$log") skipped (log: $log)"
|
||||
echo "== $suite: $(grep -c '^PASS' "$log") passed, $fails failed, $(grep -c '^SKIP' "$log") skipped (log: $log)"
|
||||
grep '^FAIL' "$log"
|
||||
exit "$fails"
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# Run the whole automated test on another machine (the PC with the VMs) from this one (e.g. a laptop):
|
||||
# update the repo there, start scripts/vmtest.sh in a detached `screen`, wait for the summary, print it.
|
||||
# scripts/vmtest-remote.sh <ssh-host> [--no-wait] [vmtest.sh args...] e.g. scripts/vmtest-remote.sh pc cli hw
|
||||
# The branch you are on must be pushed (the remote pulls it). The remote needs this repo, ../steamify-cachyos,
|
||||
# ../steammachine-cachyos-live-iso (for the loader list and the ISO), qemu + OVMF + screen, KVM, and the VMs in
|
||||
# ~/vms (see the vm-install skill). Env: REMOTE_REPO (default ~/projects/steamify/steamify-cachyos-dev).
|
||||
set -uo pipefail
|
||||
host="${1:?usage: $0 <ssh-host> [--no-wait] [vmtest.sh args...]}"; shift
|
||||
wait=true; args=()
|
||||
for a in "$@"; do [[ "$a" == --no-wait ]] && wait=false || args+=("$a"); done
|
||||
repo="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
branch="$(git -C "$repo" branch --show-current)"
|
||||
rr="${REMOTE_REPO:-~/projects/steamify/steamify-cachyos-dev}"
|
||||
git -C "$repo" ls-remote --exit-code --heads origin "$branch" >/dev/null 2>&1 || { echo "Push $branch first: the remote pulls it." >&2; exit 1; }
|
||||
[[ -z "$(git -C "$repo" status --porcelain)" ]] || echo "Note: uncommitted changes here are NOT tested; the remote runs what is pushed." >&2
|
||||
ssh "$host" "rm -f ~/vms/last-test.txt; cd $rr && git fetch -q origin && git checkout -q $branch && git pull -q --ff-only && scripts/vmtest.sh --screen ${args[*]:-}" || exit 1
|
||||
$wait || { echo "Started on $host; the summary lands in ~/vms/last-test.txt there (screen -r vmtest to watch)."; exit 0; }
|
||||
until ssh "$host" 'grep -q "^== total" ~/vms/last-test.txt 2>/dev/null'; do sleep 30; done
|
||||
ssh "$host" 'cat ~/vms/last-test.txt'
|
||||
ssh "$host" 'test "$(grep -c "^FAIL" ~/vms/last-test.txt)" -eq 0'
|
||||
+23
-6
@@ -7,6 +7,7 @@
|
||||
# <suite> cli, menu, hw, installer, toggles (folders in share/vmtest/)
|
||||
# --install install the boot loader VMs first from the newest Steamify ISO (asks before replacing a disk)
|
||||
# --window show the VMs' windows (default: headless; one shared Konsole follows ~/vms/test.log)
|
||||
# MAX_PARALLEL=3 (env) VMs at once; the slowest suites start first
|
||||
# --screen run in a detached `screen` session named vmtest: `screen -r vmtest` to watch, Ctrl-a d to leave
|
||||
# The suites run on the plain CachyOS VM ~/vms/steamify-vm (Plasma, from vminstall.sh --iso <CachyOS ISO>
|
||||
# with VM_STEAMIFY=skip); the boot loader test on ~/vms/bl-<loader> (Steamify ISO).
|
||||
@@ -46,19 +47,35 @@ advertised="$(sed -n 's/.*"bootloader:\([^"]*\)".*/\1/p' "$iso_repo/archiso/airo
|
||||
$want_boot && [[ ${#loaders[@]} -eq 0 ]] && read -r -a loaders <<< "$advertised"
|
||||
tested="limine systemd-boot grub" # the loaders vmbootloadertest.sh has a way to boot the other kernel for
|
||||
summary="$HOME/vms/last-test.txt"
|
||||
maxp="${MAX_PARALLEL:-3}" # VMs at once: 8 GB RAM each, 6 vCPUs each
|
||||
# Jobs, the slowest first so the last ones are short: suites, then the boot loaders.
|
||||
jobs_list=()
|
||||
for s in toggles hw cli menu installer; do [[ " ${suites[*]} " == *" $s "* ]] && jobs_list+=("suite:$s"); done
|
||||
for s in "${suites[@]}"; do [[ " toggles hw cli menu installer " == *" $s "* ]] || jobs_list+=("suite:$s"); done
|
||||
for l in "${loaders[@]}"; do jobs_list+=("loader:$l"); done
|
||||
run_job() { # run_job <kind:name>: output in /tmp/vmtest-<name>.out, exit status in /tmp/vmtest-<name>.rc
|
||||
local kind="${1%%:*}" name="${1#*:}"
|
||||
if [[ "$kind" == loader ]]; then bash "$here/vmbootloadertest.sh" "$name" $install $window > "/tmp/vmtest-$name.out" 2>&1
|
||||
else bash "$here/vmsuite.sh" "$name" $window > "/tmp/vmtest-$name.out" 2>&1; fi
|
||||
echo $? > "/tmp/vmtest-$name.rc"
|
||||
}
|
||||
{
|
||||
echo "vmtest.sh $(date '+%F %T') branch $(git -C "$repo" branch --show-current) $(git -C "$repo" rev-parse --short HEAD)"
|
||||
total=0
|
||||
for l in $advertised; do
|
||||
[[ " $tested " == *" $l "* ]] || { echo "FAIL the ISO advertises $l, but there is no test for it (scripts/vmbootloadertest.sh)"; total=$((total + 1)); }
|
||||
done
|
||||
for l in "${loaders[@]}"; do
|
||||
bash "$here/vmbootloadertest.sh" "$l" $install $window > "/tmp/vmtest-$l.out" 2>&1; rc=$?
|
||||
sed -n '/^== /,$p' "/tmp/vmtest-$l.out"; total=$((total + rc))
|
||||
[[ ${#suites[@]} -gt 0 ]] && { bash "$here/vmsuite.sh" --prepare || { echo "FAIL no base image for the suites"; total=$((total + 1)); }; }
|
||||
rm -f /tmp/vmtest-*.rc
|
||||
for job in "${jobs_list[@]}"; do
|
||||
run_job "$job" &
|
||||
while [[ $(jobs -rp | wc -l) -ge $maxp ]]; do wait -n; done
|
||||
done
|
||||
for s in "${suites[@]}"; do
|
||||
bash "$here/vmsuite.sh" "$s" $window > "/tmp/vmtest-$s.out" 2>&1; rc=$?
|
||||
sed -n '/^== /,$p' "/tmp/vmtest-$s.out"; total=$((total + rc))
|
||||
wait
|
||||
for job in "${jobs_list[@]}"; do
|
||||
name="${job#*:}"
|
||||
sed -n '/^== /,$p' "/tmp/vmtest-$name.out"
|
||||
total=$((total + $(cat "/tmp/vmtest-$name.rc" 2>/dev/null || echo 1)))
|
||||
done
|
||||
echo
|
||||
echo "SKIP (needs a person, TESTPLAN.md): U1-U9 the app's screens, R2.2 the first desktop login, H-Real the real Steam Machine"
|
||||
|
||||
Reference in new issue
Block a user