diff --git a/run.sh b/run.sh index c2d6bc5..bddcc99 100755 --- a/run.sh +++ b/run.sh @@ -10,6 +10,8 @@ # VM_ISO= boots that ISO for `install` instead of cachyos.iso; with # VM_KERNEL/VM_INITRD/VM_APPEND its kernel is booted directly with those # parameters (scripts/vminstall.sh, the unattended install). +# VM_CPUS=: guest CPUs (6); ISO builds use all host cores but 2. +# VM_PORT=: the host port for the guest's SSH (2222), e.g. a second VM. # BIOS_VERSION=F7F0107 makes the guest report that BIOS version (DMI), e.g. # to test the wizard's BIOS update item; the firmware itself doesn't change. # The repo is shared into the guest; mount it there with: @@ -26,11 +28,18 @@ repo="${REPO:-$HOME/projects/cachyos-gamescope-boot}" cdrom=() direct=() [[ -n "${VM_KERNEL:-}" ]] && direct=(-kernel "$VM_KERNEL" -initrd "$VM_INITRD" -append "$VM_APPEND") +# VM_CACHE=: shared read-write as 9p tag `cache` (the ISO build's +# package cache, scripts/vmisobuild.sh; one per VM, e.g. $VM_DIR/iso-cache), +# so snapshot resets don't lose it. +[[ -n "${VM_CACHE:-}" ]] && { mkdir -p "$VM_CACHE"; direct+=(-virtfs "local,path=$VM_CACHE,mount_tag=cache,security_model=mapped-xattr"); } # VM_SERIAL=: the guest's serial console (console=ttyS0) logged into # that file, and reachable as a socket next to it (.sock) to type into. [[ -n "${VM_SERIAL:-}" ]] && direct+=(-chardev "socket,id=ser0,path=$VM_SERIAL.sock,server=on,wait=off,logfile=$VM_SERIAL" -serial chardev:ser0) smbios=() -gpu=virtio-vga-gl,xres=1920,yres=1080 +gpu=virtio-vga-gl,xres=1920,yres=1080 gl=on +# VM_NOGL=1: no virgl (software rendering in the guest), so QMP screendump +# works (scripts/qmpshot.py); for the live ISO and Calamares. +[[ -n "${VM_NOGL:-}" ]] && { gpu=virtio-vga,xres=1920,yres=1080; gl=off; } [[ -n "${BIOS_VERSION:-}" ]] && smbios+=(-smbios "type=0,version=$BIOS_VERSION") for arg in "$@"; do case "$arg" in @@ -56,14 +65,14 @@ ovmf=/usr/share/OVMF; ovmf_code=OVMF_CODE_4M.fd; ovmf_vars=OVMF_VARS_4M.fd cat ~/.ssh/*.pub > share/host-keys.pub exec qemu-system-x86_64 \ - -enable-kvm -machine q35,memory-backend=mem -cpu host -smp 6 -m 8G \ + -enable-kvm -machine q35,memory-backend=mem -cpu host -smp "${VM_CPUS:-6}" -m 8G \ -object memory-backend-memfd,id=mem,size=8G,share=on \ -drive if=pflash,format=raw,readonly=on,file="$ovmf/$ovmf_code" \ -drive if=pflash,format=raw,file=vars.fd \ -drive file=disk.qcow2,if=virtio \ - -device "$gpu" -display gtk,gl=on,zoom-to-fit=off \ + -device "$gpu" -display gtk,gl=$gl,zoom-to-fit=off \ -device qemu-xhci -device usb-tablet \ - -nic user,model=virtio-net-pci,hostfwd=tcp::2222-:22 \ + -nic user,model=virtio-net-pci,hostfwd=tcp::${VM_PORT:-2222}-:22 \ -virtfs local,path="$repo",mount_tag=repo,security_model=mapped-xattr \ -virtfs local,path="$PWD/share",mount_tag=vmtools,security_model=mapped-xattr,readonly=on \ -qmp unix:"$PWD/qmp.sock",server=on,wait=off \ diff --git a/scripts/qmpclick.py b/scripts/qmpclick.py new file mode 100644 index 0000000..cef8be5 --- /dev/null +++ b/scripts/qmpclick.py @@ -0,0 +1,16 @@ +# qmpclick.py ...: left-clicks at +# those guest-screen pixels (the usb-tablet from run.sh, absolute 0..32767). +import json, socket, sys, time +sock, w, h = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) +s = socket.socket(socket.AF_UNIX); s.connect(sock); f = s.makefile("rw"); f.readline() +def cmd(c): + f.write(json.dumps(c) + "\n"); f.flush(); return f.readline() +cmd({"execute": "qmp_capabilities"}) +def ev(*events): cmd({"execute": "input-send-event", "arguments": {"events": list(events)}}) +for pos in sys.argv[4:]: + x, y = (int(v) for v in pos.split(",")) + ev({"type": "abs", "data": {"axis": "x", "value": x * 32767 // (w - 1)}}, + {"type": "abs", "data": {"axis": "y", "value": y * 32767 // (h - 1)}}) + time.sleep(0.2) + ev({"type": "btn", "data": {"down": True, "button": "left"}}); time.sleep(0.08) + ev({"type": "btn", "data": {"down": False, "button": "left"}}); time.sleep(0.8) diff --git a/scripts/qmpshot.py b/scripts/qmpshot.py new file mode 100644 index 0000000..79db39b --- /dev/null +++ b/scripts/qmpshot.py @@ -0,0 +1,27 @@ +# qmpshot.py : screenshot of the VM's screen over QMP +# (screendump). Needs a VM without virgl (run.sh with VM_NOGL=1): with +# virgl QEMU answers "no surface". Before the guest's desktop is up it shows +# whatever is on screen (boot menu, Calamares, ...). +import json, os, socket, sys, tempfile + +sock, out = sys.argv[1], sys.argv[2] +s = socket.socket(socket.AF_UNIX); s.connect(sock); f = s.makefile("rw"); f.readline() +def cmd(c): + f.write(json.dumps(c) + "\n"); f.flush() + while True: + r = json.loads(f.readline()) + if "event" not in r: + return r +cmd({"execute": "qmp_capabilities"}) +ppm = tempfile.mktemp(suffix=".ppm") +r = cmd({"execute": "screendump", "arguments": {"filename": ppm, "format": "ppm"}}) +if "error" in r: + raise SystemExit("screendump: %s" % r["error"]["desc"]) +try: + from PIL import Image + Image.open(ppm).save(out) + os.remove(ppm) +except ImportError: + os.replace(ppm, os.path.splitext(out)[0] + ".ppm") + print("PIL missing: wrote a .ppm instead", file=sys.stderr) +print(out) diff --git a/scripts/vminstall.sh b/scripts/vminstall.sh index 3cd28b5..10c6f23 100755 --- a/scripts/vminstall.sh +++ b/scripts/vminstall.sh @@ -79,14 +79,14 @@ echo "ISO: $iso key: $VM_SSH_KEY" # --- The ISO's kernel and initramfs, to boot it with our parameters. boot="$VM_DIR/iso-boot" mkdir -p "$boot" -loop="$(udisksctl loop-setup -r -f "$iso" | grep -o '/dev/loop[0-9]*')" -cleanup_loop() { udisksctl unmount -b "${loop}p1" >/dev/null 2>&1 || true; udisksctl loop-delete -b "$loop" >/dev/null 2>&1 || true; } +loop="$(udisksctl loop-setup --no-user-interaction -r -f "$iso" | grep -o '/dev/loop[0-9]*')" +cleanup_loop() { udisksctl unmount --no-user-interaction -b "${loop}p1" >/dev/null 2>&1 || true; udisksctl loop-delete --no-user-interaction -b "$loop" >/dev/null 2>&1 || true; } trap cleanup_loop EXIT mnt="" for _ in $(seq 10); do mnt="$(findmnt -nro TARGET "${loop}p1" 2>/dev/null || true)" [[ -n "$mnt" ]] && break - udisksctl mount -b "${loop}p1" >/dev/null 2>&1 || true; sleep 1 + udisksctl mount --no-user-interaction -b "${loop}p1" >/dev/null 2>&1 || true; sleep 1 done [[ -n "$mnt" ]] || { echo "Couldn't mount the ISO." >&2; exit 1; } rm -f "$boot"/* # copied read-only from the ISO diff --git a/scripts/vmisoboot.sh b/scripts/vmisoboot.sh new file mode 100755 index 0000000..258f75d --- /dev/null +++ b/scripts/vmisoboot.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Boot a live ISO (the Steam Machine ISO) in its own VM, the normal way, to go +# through Calamares by hand or driven with scripts/qmpkey.py / qmptype.py, +# with screenshots of every page (scripts/qmpshot.py; no virgl, so QMP +# screendump works). Unlike vminstall.sh nothing is automated: the wizard, +# its Steamify pages and the installer's Steamify step run as on real +# hardware. Starts in the background; the VM's window is where you watch. +# scripts/vmisoboot.sh [--fresh] [--iso ] [run.sh flags...] e.g. --fremont +# --fresh a new, empty disk (else the existing one: e.g. boot the +# installed system with `./run.sh` in $VM_DIR afterwards) +# --iso default: the newest out/desktop/*.iso of the host's +# steammachine-cachyos-live-iso checkout +# Env: ISO_VM_DIR (~/vms/iso-vm), ISO_VM_PORT (2223: runs next to the test +# VM on 2222). +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(cd "$here/.." && pwd)" +dir="${ISO_VM_DIR:-$HOME/vms/iso-vm}" +port="${ISO_VM_PORT:-2223}" +fresh=false iso="" flags=() +while [[ $# -gt 0 ]]; do + case "$1" in + --fresh) fresh=true; shift ;; + --iso) iso="${2:?--iso needs a file}"; shift 2 ;; + *) flags+=("$1"); shift ;; + esac +done +if [[ -z "$iso" ]]; then + iso="$(ls -t "$(dirname "$repo")"/steammachine-cachyos-live-iso/out/desktop/*.iso 2>/dev/null | head -1)" + [[ -n "$iso" ]] || { echo "No ISO in steammachine-cachyos-live-iso/out/desktop: build one (vmisobuild.sh) or pass --iso." >&2; exit 1; } +fi +iso="$(cd "$(dirname "$iso")" && pwd)/$(basename "$iso")" +mkdir -p "$dir" +cd "$dir" +[[ -S qmp.sock ]] && python3 -c "import socket,sys; s=socket.socket(socket.AF_UNIX); s.connect(sys.argv[1])" qmp.sock 2>/dev/null && + { echo "The ISO VM in $dir is running already." >&2; exit 1; } +cp "$repo/run.sh" run.sh +[[ -e share ]] || ln -s "$repo/share" share +if [[ "$fresh" == true ]]; then rm -f disk.qcow2 vars.fd; fi +REPO="${REPO:-$(dirname "$repo")/steamify-cachyos}" VM_NOGL=1 VM_PORT="$port" VM_ISO="$iso" \ + nohup ./run.sh install "${flags[@]}" > vm.log 2>&1 & +echo "ISO VM: $iso in $dir (SSH port $port once installed)." +echo "Screenshot: python3 $here/qmpshot.py $dir/qmp.sock shot.png" +echo "Keys: python3 $here/qmpkey.py $dir/qmp.sock down ret / qmptype.py $dir/qmp.sock \"text\"" diff --git a/scripts/vmisobuild.sh b/scripts/vmisobuild.sh new file mode 100755 index 0000000..0752672 --- /dev/null +++ b/scripts/vmisobuild.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Build the Steam Machine ISO (steammachine-cachyos-live-iso) in the test VM +# (directly, no container: build-live-modules.sh with sudo, buildiso.sh as +# the user, it sudos itself), +# visibly (a Konsole on the VM's desktop follows the log), with the package +# downloads cached on the host, and copy the ISO back. +# scripts/vmisobuild.sh [--steamify ] +# --steamify a Steamify checkout in the guest (e.g. /mnt, the mounted +# REPO) instead of the newest release +# The VM must run with its cache shared (one per VM, next to its disk), and +# gets all host cores but 2 for the build: start it with +# VM_CACHE=$VM_DIR/iso-cache VM_CPUS=$(( $(nproc) - 2 )) scripts/vmreset.sh --fremont +# Env: VM_DIR (see common.sh), ISO_BRANCH (feat/steamify), ISO_OUT (the host's +# steammachine-cachyos-live-iso/out/desktop). +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +. "$here/common.sh" +steamify="" +[[ "${1:-}" == --steamify ]] && steamify="${2:?--steamify needs a guest path}" +branch="${ISO_BRANCH:-feat/steamify}" +# Where the ISO lands: out/desktop of the host's live-ISO checkout (next to +# this repo), where a local build would put it. +out="${ISO_OUT:-$(dirname "$(cd "$here/.." && pwd)")/steammachine-cachyos-live-iso/out/desktop}" +mkdir -p "$out" + +{ printf 'branch=%q steamify=%q\n' "$branch" "$steamify"; cat << 'REMOTE' +set -e +export XDG_RUNTIME_DIR=/run/user/$(id -u) DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus +# The package cache on the host (9p tag `cache`), kept across snapshot resets, +# is the VM's pacman cache: the build tools and mkarchiso (which installs the +# ISO's packages with the build system's cache) both use it. No container: +# the VM is CachyOS already, and a snapshot reset removes the build tools. +sudo mkdir -p /var/cache/steamify-iso +mountpoint -q /var/cache/steamify-iso || sudo mount -t 9p -o trans=virtio,version=9p2000.L cache /var/cache/steamify-iso || + { echo "No 9p share 'cache': start the VM with VM_CACHE= (run.sh)." >&2; exit 1; } +sudo mkdir -p /var/cache/steamify-iso/pkg +mountpoint -q /var/cache/pacman/pkg || sudo mount --bind /var/cache/steamify-iso/pkg /var/cache/pacman/pkg +[ -z "$steamify" ] || mountpoint -q /mnt || sudo mount -t 9p -o trans=virtio,version=9p2000.L repo /mnt +# archlinux.cachyos.org serves packages without their .sig (404), and pacman +# doesn't try the next mirror for the signature of a cached package: the +# build fails. mkarchiso uses this mirrorlist, so skip it here. +sudo sed -i '/archlinux\.cachyos\.org/s/^Server/#Server/' /etc/pacman.d/mirrorlist +sudo pacman -Sy --needed --noconfirm archiso mkinitcpio-archiso git squashfs-tools grub >/dev/null +iso=~/projects/steammachine-cachyos-live-iso +if [ -d "$iso/.git" ]; then git -C "$iso" fetch -q && git -C "$iso" checkout -q "$branch" && git -C "$iso" reset -q --hard "origin/$branch" +else mkdir -p ~/projects && git clone -q -b "$branch" https://github.com/theupriser/steammachine-cachyos-live-iso "$iso"; fi +cd "$iso" && echo "ISO repo: $(git log --oneline -1)" +./steamify-prepare.sh $steamify | tail -1 +sudo rm -rf build out +cat > ~/projects/iso-build.sh << S +#!/bin/bash +cd $iso +sudo ./build-live-modules.sh && { ./buildiso.sh -p desktop -w || ./buildiso.sh -p desktop -c -w; } +echo "== BUILD EXIT \$?" +ls -la out/desktop/*.iso 2>/dev/null +S +chmod +x ~/projects/iso-build.sh; : > ~/projects/iso-build.log +systemd-run --user --collect -q -u "isobuild-$(date +%s)" bash -c "~/projects/iso-build.sh > ~/projects/iso-build.log 2>&1" +systemd-run --user -q --setenv=XDG_CURRENT_DESKTOP=KDE konsole --separate --hold -e bash -c "tail -n 50 -F ~/projects/iso-build.log" +REMOTE +} | vm_ssh bash -s +echo "Building in the VM (its Konsole shows the log)..." +until vm_ssh 'grep -q "^== BUILD EXIT" ~/projects/iso-build.log'; do sleep 30; done +iso="$(vm_ssh "bash -c 'ls -t ~/projects/steammachine-cachyos-live-iso/out/desktop/*.iso 2>/dev/null | head -1'")" +[[ -n "$iso" ]] || { echo "No ISO was built; see ~/projects/iso-build.log in the VM." >&2; exit 1; } +vm_scp "$VM_USER@$VM_HOST:$iso" "$out/" +echo "ISO: $out/$(basename "$iso") (cache: $VM_DIR/iso-cache)" diff --git a/scripts/vmreset.sh b/scripts/vmreset.sh index 29c3947..dcbe901 100755 --- a/scripts/vmreset.sh +++ b/scripts/vmreset.sh @@ -9,6 +9,7 @@ # VM_LOG (QEMU output, default $VM_DIR/vm.log). set -euo pipefail . "$(dirname "$0")/common.sh" +repo_dir="$(cd "$(dirname "$0")/.." && pwd)" SNAPSHOT="${SNAPSHOT:-ssh-ready}" VM_LOG="${VM_LOG:-$VM_DIR/vm.log}" @@ -21,6 +22,8 @@ fi echo "Restoring snapshot '$SNAPSHOT' in $VM_DIR..." cd "$VM_DIR" +# Always this repo's run.sh: a copy in $VM_DIR falls behind (VM_CACHE, ...). +cp "$repo_dir/run.sh" run.sh qemu-img snapshot -a "$SNAPSHOT" disk.qcow2 cp "vars.$SNAPSHOT.fd" vars.fd