feat: vmisobuild.sh (ISO build in the VM, package cache per VM), vmisoboot.sh (ISO VM for Calamares), qmpshot.py/qmpclick.py; run.sh VM_CACHE/VM_CPUS/VM_NOGL/VM_PORT

run.sh

@@ -10,6 +10,8 @@
 # VM_ISO=<path> boots that ISO for `install` instead of cachyos.iso; with
 # VM_KERNEL/VM_INITRD/VM_APPEND its kernel is booted directly with those
 # parameters (scripts/vminstall.sh, the unattended install).
+# VM_CPUS=<n>: guest CPUs (6); ISO builds use all host cores but 2.
+# VM_PORT=<n>: the host port for the guest's SSH (2222), e.g. a second VM.
 # BIOS_VERSION=F7F0107 makes the guest report that BIOS version (DMI), e.g.
 # to test the wizard's BIOS update item; the firmware itself doesn't change.
 # The repo is shared into the guest; mount it there with:
@@ -26,11 +28,18 @@ repo="${REPO:-$HOME/projects/cachyos-gamescope-boot}"
 cdrom=()
 direct=()
 [[ -n "${VM_KERNEL:-}" ]] && direct=(-kernel "$VM_KERNEL" -initrd "$VM_INITRD" -append "$VM_APPEND")
+# VM_CACHE=<host dir>: shared read-write as 9p tag `cache` (the ISO build's
+# package cache, scripts/vmisobuild.sh; one per VM, e.g. $VM_DIR/iso-cache),
+# so snapshot resets don't lose it.
+[[ -n "${VM_CACHE:-}" ]] && { mkdir -p "$VM_CACHE"; direct+=(-virtfs "local,path=$VM_CACHE,mount_tag=cache,security_model=mapped-xattr"); }
 # VM_SERIAL=<file>: the guest's serial console (console=ttyS0) logged into
 # that file, and reachable as a socket next to it (<file>.sock) to type into.
 [[ -n "${VM_SERIAL:-}" ]] && direct+=(-chardev "socket,id=ser0,path=$VM_SERIAL.sock,server=on,wait=off,logfile=$VM_SERIAL" -serial chardev:ser0)
 smbios=()
-gpu=virtio-vga-gl,xres=1920,yres=1080
+gpu=virtio-vga-gl,xres=1920,yres=1080 gl=on
+# VM_NOGL=1: no virgl (software rendering in the guest), so QMP screendump
+# works (scripts/qmpshot.py); for the live ISO and Calamares.
+[[ -n "${VM_NOGL:-}" ]] && { gpu=virtio-vga,xres=1920,yres=1080; gl=off; }
 [[ -n "${BIOS_VERSION:-}" ]] && smbios+=(-smbios "type=0,version=$BIOS_VERSION")
 for arg in "$@"; do
     case "$arg" in
@@ -56,14 +65,14 @@ ovmf=/usr/share/OVMF; ovmf_code=OVMF_CODE_4M.fd; ovmf_vars=OVMF_VARS_4M.fd
 cat ~/.ssh/*.pub > share/host-keys.pub
 
 exec qemu-system-x86_64 \
-    -enable-kvm -machine q35,memory-backend=mem -cpu host -smp 6 -m 8G \
+    -enable-kvm -machine q35,memory-backend=mem -cpu host -smp "${VM_CPUS:-6}" -m 8G \
     -object memory-backend-memfd,id=mem,size=8G,share=on \
     -drive if=pflash,format=raw,readonly=on,file="$ovmf/$ovmf_code" \
     -drive if=pflash,format=raw,file=vars.fd \
     -drive file=disk.qcow2,if=virtio \
-    -device "$gpu" -display gtk,gl=on,zoom-to-fit=off \
+    -device "$gpu" -display gtk,gl=$gl,zoom-to-fit=off \
     -device qemu-xhci -device usb-tablet \
-    -nic user,model=virtio-net-pci,hostfwd=tcp::2222-:22 \
+    -nic user,model=virtio-net-pci,hostfwd=tcp::${VM_PORT:-2222}-:22 \
     -virtfs local,path="$repo",mount_tag=repo,security_model=mapped-xattr \
     -virtfs local,path="$PWD/share",mount_tag=vmtools,security_model=mapped-xattr,readonly=on \
     -qmp unix:"$PWD/qmp.sock",server=on,wait=off \

scripts/qmpclick.py

@@ -0,0 +1,16 @@
+# qmpclick.py <qmp socket> <screen w> <screen h> <x,y>...: left-clicks at
+# those guest-screen pixels (the usb-tablet from run.sh, absolute 0..32767).
+import json, socket, sys, time
+sock, w, h = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
+s = socket.socket(socket.AF_UNIX); s.connect(sock); f = s.makefile("rw"); f.readline()
+def cmd(c):
+    f.write(json.dumps(c) + "\n"); f.flush(); return f.readline()
+cmd({"execute": "qmp_capabilities"})
+def ev(*events): cmd({"execute": "input-send-event", "arguments": {"events": list(events)}})
+for pos in sys.argv[4:]:
+    x, y = (int(v) for v in pos.split(","))
+    ev({"type": "abs", "data": {"axis": "x", "value": x * 32767 // (w - 1)}},
+       {"type": "abs", "data": {"axis": "y", "value": y * 32767 // (h - 1)}})
+    time.sleep(0.2)
+    ev({"type": "btn", "data": {"down": True, "button": "left"}}); time.sleep(0.08)
+    ev({"type": "btn", "data": {"down": False, "button": "left"}}); time.sleep(0.8)

scripts/qmpshot.py

@@ -0,0 +1,27 @@
+# qmpshot.py <qmp socket> <out.png>: screenshot of the VM's screen over QMP
+# (screendump). Needs a VM without virgl (run.sh with VM_NOGL=1): with
+# virgl QEMU answers "no surface". Before the guest's desktop is up it shows
+# whatever is on screen (boot menu, Calamares, ...).
+import json, os, socket, sys, tempfile
+
+sock, out = sys.argv[1], sys.argv[2]
+s = socket.socket(socket.AF_UNIX); s.connect(sock); f = s.makefile("rw"); f.readline()
+def cmd(c):
+    f.write(json.dumps(c) + "\n"); f.flush()
+    while True:
+        r = json.loads(f.readline())
+        if "event" not in r:
+            return r
+cmd({"execute": "qmp_capabilities"})
+ppm = tempfile.mktemp(suffix=".ppm")
+r = cmd({"execute": "screendump", "arguments": {"filename": ppm, "format": "ppm"}})
+if "error" in r:
+    raise SystemExit("screendump: %s" % r["error"]["desc"])
+try:
+    from PIL import Image
+    Image.open(ppm).save(out)
+    os.remove(ppm)
+except ImportError:
+    os.replace(ppm, os.path.splitext(out)[0] + ".ppm")
+    print("PIL missing: wrote a .ppm instead", file=sys.stderr)
+print(out)

scripts/vminstall.sh

@@ -79,14 +79,14 @@ echo "ISO: $iso  key: $VM_SSH_KEY"
 # --- The ISO's kernel and initramfs, to boot it with our parameters.
 boot="$VM_DIR/iso-boot"
 mkdir -p "$boot"
-loop="$(udisksctl loop-setup -r -f "$iso" | grep -o '/dev/loop[0-9]*')"
+loop="$(udisksctl loop-setup --no-user-interaction -r -f "$iso" | grep -o '/dev/loop[0-9]*')"
-cleanup_loop() { udisksctl unmount -b "${loop}p1" >/dev/null 2>&1 || true; udisksctl loop-delete -b "$loop" >/dev/null 2>&1 || true; }
+cleanup_loop() { udisksctl unmount --no-user-interaction -b "${loop}p1" >/dev/null 2>&1 || true; udisksctl loop-delete --no-user-interaction -b "$loop" >/dev/null 2>&1 || true; }
 trap cleanup_loop EXIT
 mnt=""
 for _ in $(seq 10); do
     mnt="$(findmnt -nro TARGET "${loop}p1" 2>/dev/null || true)"
     [[ -n "$mnt" ]] && break
-    udisksctl mount -b "${loop}p1" >/dev/null 2>&1 || true; sleep 1
+    udisksctl mount --no-user-interaction -b "${loop}p1" >/dev/null 2>&1 || true; sleep 1
 done
 [[ -n "$mnt" ]] || { echo "Couldn't mount the ISO." >&2; exit 1; }
 rm -f "$boot"/*   # copied read-only from the ISO

scripts/vmisoboot.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Boot a live ISO (the Steam Machine ISO) in its own VM, the normal way, to go
+# through Calamares by hand or driven with scripts/qmpkey.py / qmptype.py,
+# with screenshots of every page (scripts/qmpshot.py; no virgl, so QMP
+# screendump works). Unlike vminstall.sh nothing is automated: the wizard,
+# its Steamify pages and the installer's Steamify step run as on real
+# hardware. Starts in the background; the VM's window is where you watch.
+#   scripts/vmisoboot.sh [--fresh] [--iso <file>] [run.sh flags...]   e.g. --fremont
+#   --fresh      a new, empty disk (else the existing one: e.g. boot the
+#                installed system with `./run.sh` in $VM_DIR afterwards)
+#   --iso <file> default: the newest out/desktop/*.iso of the host's
+#                steammachine-cachyos-live-iso checkout
+# Env: ISO_VM_DIR (~/vms/iso-vm), ISO_VM_PORT (2223: runs next to the test
+#      VM on 2222).
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+repo="$(cd "$here/.." && pwd)"
+dir="${ISO_VM_DIR:-$HOME/vms/iso-vm}"
+port="${ISO_VM_PORT:-2223}"
+fresh=false iso="" flags=()
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --fresh) fresh=true; shift ;;
+        --iso) iso="${2:?--iso needs a file}"; shift 2 ;;
+        *) flags+=("$1"); shift ;;
+    esac
+done
+if [[ -z "$iso" ]]; then
+    iso="$(ls -t "$(dirname "$repo")"/steammachine-cachyos-live-iso/out/desktop/*.iso 2>/dev/null | head -1)"
+    [[ -n "$iso" ]] || { echo "No ISO in steammachine-cachyos-live-iso/out/desktop: build one (vmisobuild.sh) or pass --iso." >&2; exit 1; }
+fi
+iso="$(cd "$(dirname "$iso")" && pwd)/$(basename "$iso")"
+mkdir -p "$dir"
+cd "$dir"
+[[ -S qmp.sock ]] && python3 -c "import socket,sys; s=socket.socket(socket.AF_UNIX); s.connect(sys.argv[1])" qmp.sock 2>/dev/null &&
+    { echo "The ISO VM in $dir is running already." >&2; exit 1; }
+cp "$repo/run.sh" run.sh
+[[ -e share ]] || ln -s "$repo/share" share
+if [[ "$fresh" == true ]]; then rm -f disk.qcow2 vars.fd; fi
+REPO="${REPO:-$(dirname "$repo")/steamify-cachyos}" VM_NOGL=1 VM_PORT="$port" VM_ISO="$iso" \
+    nohup ./run.sh install "${flags[@]}" > vm.log 2>&1 &
+echo "ISO VM: $iso in $dir (SSH port $port once installed)."
+echo "Screenshot: python3 $here/qmpshot.py $dir/qmp.sock shot.png"
+echo "Keys:       python3 $here/qmpkey.py $dir/qmp.sock down ret   /   qmptype.py $dir/qmp.sock \"text\""

scripts/vmisobuild.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Build the Steam Machine ISO (steammachine-cachyos-live-iso) in the test VM
+# (directly, no container: build-live-modules.sh with sudo, buildiso.sh as
+# the user, it sudos itself),
+# visibly (a Konsole on the VM's desktop follows the log), with the package
+# downloads cached on the host, and copy the ISO back.
+#   scripts/vmisobuild.sh [--steamify <guest path>]
+#     --steamify  a Steamify checkout in the guest (e.g. /mnt, the mounted
+#                 REPO) instead of the newest release
+# The VM must run with its cache shared (one per VM, next to its disk), and
+# gets all host cores but 2 for the build: start it with
+#   VM_CACHE=$VM_DIR/iso-cache VM_CPUS=$(( $(nproc) - 2 )) scripts/vmreset.sh --fremont
+# Env: VM_DIR (see common.sh), ISO_BRANCH (feat/steamify), ISO_OUT (the host's
+#      steammachine-cachyos-live-iso/out/desktop).
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+. "$here/common.sh"
+steamify=""
+[[ "${1:-}" == --steamify ]] && steamify="${2:?--steamify needs a guest path}"
+branch="${ISO_BRANCH:-feat/steamify}"
+# Where the ISO lands: out/desktop of the host's live-ISO checkout (next to
+# this repo), where a local build would put it.
+out="${ISO_OUT:-$(dirname "$(cd "$here/.." && pwd)")/steammachine-cachyos-live-iso/out/desktop}"
+mkdir -p "$out"
+
+{ printf 'branch=%q steamify=%q\n' "$branch" "$steamify"; cat << 'REMOTE'
+set -e
+export XDG_RUNTIME_DIR=/run/user/$(id -u) DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus
+# The package cache on the host (9p tag `cache`), kept across snapshot resets,
+# is the VM's pacman cache: the build tools and mkarchiso (which installs the
+# ISO's packages with the build system's cache) both use it. No container:
+# the VM is CachyOS already, and a snapshot reset removes the build tools.
+sudo mkdir -p /var/cache/steamify-iso
+mountpoint -q /var/cache/steamify-iso || sudo mount -t 9p -o trans=virtio,version=9p2000.L cache /var/cache/steamify-iso ||
+    { echo "No 9p share 'cache': start the VM with VM_CACHE=<host dir> (run.sh)." >&2; exit 1; }
+sudo mkdir -p /var/cache/steamify-iso/pkg
+mountpoint -q /var/cache/pacman/pkg || sudo mount --bind /var/cache/steamify-iso/pkg /var/cache/pacman/pkg
+[ -z "$steamify" ] || mountpoint -q /mnt || sudo mount -t 9p -o trans=virtio,version=9p2000.L repo /mnt
+# archlinux.cachyos.org serves packages without their .sig (404), and pacman
+# doesn't try the next mirror for the signature of a cached package: the
+# build fails. mkarchiso uses this mirrorlist, so skip it here.
+sudo sed -i '/archlinux\.cachyos\.org/s/^Server/#Server/' /etc/pacman.d/mirrorlist
+sudo pacman -Sy --needed --noconfirm archiso mkinitcpio-archiso git squashfs-tools grub >/dev/null
+iso=~/projects/steammachine-cachyos-live-iso
+if [ -d "$iso/.git" ]; then git -C "$iso" fetch -q && git -C "$iso" checkout -q "$branch" && git -C "$iso" reset -q --hard "origin/$branch"
+else mkdir -p ~/projects && git clone -q -b "$branch" https://github.com/theupriser/steammachine-cachyos-live-iso "$iso"; fi
+cd "$iso" && echo "ISO repo: $(git log --oneline -1)"
+./steamify-prepare.sh $steamify | tail -1
+sudo rm -rf build out
+cat > ~/projects/iso-build.sh << S
+#!/bin/bash
+cd $iso
+sudo ./build-live-modules.sh && { ./buildiso.sh -p desktop -w || ./buildiso.sh -p desktop -c -w; }
+echo "== BUILD EXIT \$?"
+ls -la out/desktop/*.iso 2>/dev/null
+S
+chmod +x ~/projects/iso-build.sh; : > ~/projects/iso-build.log
+systemd-run --user --collect -q -u "isobuild-$(date +%s)" bash -c "~/projects/iso-build.sh > ~/projects/iso-build.log 2>&1"
+systemd-run --user -q --setenv=XDG_CURRENT_DESKTOP=KDE konsole --separate --hold -e bash -c "tail -n 50 -F ~/projects/iso-build.log"
+REMOTE
+} | vm_ssh bash -s
+echo "Building in the VM (its Konsole shows the log)..."
+until vm_ssh 'grep -q "^== BUILD EXIT" ~/projects/iso-build.log'; do sleep 30; done
+iso="$(vm_ssh "bash -c 'ls -t ~/projects/steammachine-cachyos-live-iso/out/desktop/*.iso 2>/dev/null | head -1'")"
+[[ -n "$iso" ]] || { echo "No ISO was built; see ~/projects/iso-build.log in the VM." >&2; exit 1; }
+vm_scp "$VM_USER@$VM_HOST:$iso" "$out/"
+echo "ISO: $out/$(basename "$iso")  (cache: $VM_DIR/iso-cache)"

scripts/vmreset.sh

@@ -9,6 +9,7 @@
 #      VM_LOG (QEMU output, default $VM_DIR/vm.log).
 set -euo pipefail
 . "$(dirname "$0")/common.sh"
+repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
 SNAPSHOT="${SNAPSHOT:-ssh-ready}"
 VM_LOG="${VM_LOG:-$VM_DIR/vm.log}"
 
@@ -21,6 +22,8 @@ fi
 
 echo "Restoring snapshot '$SNAPSHOT' in $VM_DIR..."
 cd "$VM_DIR"
+# Always this repo's run.sh: a copy in $VM_DIR falls behind (VM_CACHE, ...).
+cp "$repo_dir/run.sh" run.sh
 qemu-img snapshot -a "$SNAPSHOT" disk.qcow2
 cp "vars.$SNAPSHOT.fd" vars.fd