ci: stream the VM tests' progress into the job log

vmtest.yml: newest Steamify ISO release from the mirror (secret GIT_UPRISER_URL), KVM on the hosted runner, boot loaders limine/systemd-boot/grub two at a time, then the suites on a plain VM from the same ISO.

.github/actions/vm-runner/action.yml

@@ -0,0 +1,37 @@
+name: VM runner
+description: KVM, qemu, the big disk for the VMs, the package cache and the cached Steamify ISO (at /mnt/work/iso/steamify.iso)
+inputs:
+  iso-tag:
+    description: The Steamify ISO release the iso job cached
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: KVM, qemu, disk
+      shell: bash
+      run: |
+        echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+        sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm && sudo udevadm settle || true
+        for _ in $(seq 10); do [ -w /dev/kvm ] && break; sleep 1; done
+        ls -l /dev/kvm 2>&1 || true
+        grep -cE 'vmx|svm' /proc/cpuinfo || true
+        [ -w /dev/kvm ] || { echo "::error::no usable /dev/kvm on this runner (some GitHub runners have none: re-run the job)"; exit 1; }
+        sudo apt-get update -qq
+        sudo apt-get install -y -qq qemu-system-x86 qemu-utils ovmf libarchive-tools openssh-client
+        # The VMs and the ISO on /mnt (the big disk); ~/vms points there.
+        sudo mkdir -p /mnt/work && sudo chown "$USER" /mnt/work
+        mkdir -p /mnt/work/vms/pkg-cache /mnt/work/iso "$HOME/.ssh" && ln -s /mnt/work/vms "$HOME/vms"
+        ssh-keygen -q -t ed25519 -N "" -C steamify-vm -f "$HOME/.ssh/steamify-vm_ed25519"
+        nproc; free -g | sed -n 2p; df -h /mnt | tail -n 1
+    - name: Cache - the VMs' packages
+      uses: actions/cache@v4
+      with:
+        path: /mnt/work/vms/pkg-cache
+        key: vmpkg-${{ github.run_id }}
+        restore-keys: vmpkg-
+    - name: Cache - the ISO (from the iso job)
+      uses: actions/cache/restore@v4
+      with:
+        path: /mnt/work/iso
+        key: steamify-iso-${{ inputs.iso-tag }}
+        fail-on-cache-miss: true

.github/workflows/vmtest.yml

@@ -1,19 +1,20 @@
-# Steamify's own test run on the Gitea mirror (git.upriser.nl, its own runner; skipped on GitHub): the same as
+# Steamify's own VM tests on GitHub's runners (4 vCPUs, 16 GB, KVM; skipped on the Gitea mirror): the same as
 # `scripts/vmtest.sh` in steamify-cachyos-dev, on this commit's steamify.sh. The base is the newest Steamify ISO
-# release of the mirror (steamify-cachyos-live-iso): the boot loader tests (limine, systemd-boot, grub) install
+# release (steamify-cachyos-live-iso, built and kept on the git.upriser.nl mirror: GitHub releases take at
-# it with this commit's Steamify, then the suites (cli, menu, hw, installer, toggles) run on a plain VM from
+# most 2 GB per file): the boot loader tests (limine, systemd-boot, grub) install it with this commit's
-# the same ISO (its Steamify step skipped: the suites start from a system without Steamify).
+# Steamify, then the suites (cli, menu, hw, installer, toggles) run on a plain VM from the same ISO (its
-# The runner needs /dev/kvm and privileged containers, about 12 GB of RAM for two VMs at a time (6 GB to
+# Steamify step skipped: the suites start from a system without Steamify).
-# install, 4 GB to test, 2 vCPUs each).
+# The ISO is fetched once (the iso job, cached) and restored by the others.
-name: Steamify VM tests (Gitea)
+# Needs the secret GIT_UPRISER_URL (the mirror's address, for the ISO). Every job has its own runner, so everything runs in parallel: one job per boot loader and per suite, each with its own VMs (2 vCPUs, 4-6 GB).
+name: Steamify VM tests
 
+# Only on main and the release branches (and by hand): the runners' time is limited, so not on every pull request.
 on:
   push:
     branches: [main, 'release/**']
-  pull_request:
   workflow_dispatch:
 
-# A new run on the same branch or pull request stops the old one.
+# A new run on the same branch stops the old one.
 concurrency:
   group: vmtest-${{ github.ref }}
   cancel-in-progress: true
@@ -21,17 +22,18 @@ concurrency:
 jobs:
   iso:
     name: Newest Steamify ISO
-    if: github.server_url != 'https://github.com'
+    if: github.server_url == 'https://github.com'
     runs-on: ubuntu-latest
     outputs:
       tag: ${{ steps.find.outputs.tag }}
-      url: ${{ steps.find.outputs.url }}
     steps:
       - name: Find the newest ISO release on the mirror
         id: find
+        env:
+          MIRROR: ${{ secrets.GIT_UPRISER_URL }}
         run: |
-          api="${{ github.server_url }}/api/v1/repos/theupriser/steamify-cachyos-live-iso/releases?limit=20"
+          [ -n "$MIRROR" ] || { echo "::error::the secret GIT_UPRISER_URL is not set"; exit 1; }
-          read -r tag url < <(curl -fsS "$api" | python3 -c '
+          read -r tag url < <(curl -fsS "${MIRROR%/}/api/v1/repos/theupriser/steamify-cachyos-live-iso/releases?limit=20" | python3 -c '
           import json, sys
           for r in json.load(sys.stdin):
               for a in r["assets"]:
@@ -41,53 +43,29 @@ jobs:
           [ -n "$tag" ] || { echo "::error::no ISO release on the mirror"; exit 1; }
           echo "Using $tag"
           { echo "tag=$tag"; echo "url=$url"; } >> "$GITHUB_OUTPUT"
+      - name: Cache - the ISO
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: /mnt/work/iso
+          key: steamify-iso-${{ steps.find.outputs.tag }}
+      - name: Download the ISO (once for every job)
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          sudo mkdir -p /mnt/work/iso && sudo chown -R "$USER" /mnt/work
+          curl -fL --retry 5 --retry-all-errors -o /mnt/work/iso/steamify.iso "${{ steps.find.outputs.url }}"
+          ls -la /mnt/work/iso
 
   boot:
     name: Boot loader ${{ matrix.loader }}
     needs: iso
     runs-on: ubuntu-latest
-    timeout-minutes: 90
+    timeout-minutes: 120
-    container:
-      image: docker.io/cachyos/cachyos:latest
-      options: --privileged
     strategy:
       fail-fast: false
-      max-parallel: 2
       matrix:
         loader: [limine, systemd-boot, grub]
-    defaults:
-      run:
-        shell: bash
     steps:
-      - name: Node and git
-        id: node
-        run: |
-          pacman-key --init && pacman-key --populate
-          pacman -Syu --noconfirm --needed nodejs git
-          echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
-      - name: Cache - pacman packages
-        uses: actions/cache@v4
-        with:
-          path: /var/cache/pacman/pkg
-          key: pacman-test-${{ steps.node.outputs.week }}
-          restore-keys: pacman-test-
-      - name: Cache - the VMs' packages
-        uses: actions/cache@v4
-        with:
-          path: /root/vms/pkg-cache
-          key: vmpkg-steamify-${{ steps.node.outputs.week }}
-          restore-keys: vmpkg-
-      - name: Tools and KVM
-        run: |
-          pacman -S --noconfirm --needed qemu-desktop edk2-ovmf openssh libarchive procps-ng util-linux python curl sudo
-          [ -c /dev/kvm ] || { echo "::error::no /dev/kvm in this container: the runner needs KVM (and privileged containers)"; exit 1; }
-      - name: Cache - the ISO
-        uses: actions/cache@v4
-        with:
-          path: iso
-          key: steamify-iso-${{ needs.iso.outputs.tag }}
-
-      # steamify-cachyos-dev (the scripts) and this commit of steamify-cachyos side by side (9p `repo` in the VMs).
       - uses: actions/checkout@v4
         with:
           path: steamify-cachyos
@@ -95,73 +73,41 @@ jobs:
         with:
           repository: theupriser/steamify-cachyos-dev
           path: steamify-cachyos-dev
-      - name: ISO
+      - uses: ./steamify-cachyos/.github/actions/vm-runner
-        run: |
+        with:
-          mkdir -p iso
+          iso-tag: ${{ needs.iso.outputs.tag }}
-          [ -n "$(ls iso/*.iso 2>/dev/null)" ] || curl -fL --retry 5 --retry-all-errors -o "iso/${{ needs.iso.outputs.tag }}.iso" "${{ needs.iso.outputs.url }}"
       - name: Boot loader test (${{ matrix.loader }})
         env:
           CI: "1"
           VM_TIMEZONE: UTC
-          VM_CPUS: "2"
+          VM_CPUS: "4"
           VM_INSTALL_MEM: 6G
         run: |
-          mkdir -p ~/.ssh ~/vms/pkg-cache
-          ssh-keygen -q -t ed25519 -N "" -C steamify-vm -f ~/.ssh/steamify-vm_ed25519
           cd steamify-cachyos-dev
-          VM_ISO="$(ls "$GITHUB_WORKSPACE"/iso/*.iso | head -n 1)" scripts/vmbootloadertest.sh "${{ matrix.loader }}" --install
+          touch ~/vms/test.log; tail -n +1 -F ~/vms/test.log & tailpid=$!   # the test's progress, as it happens
+          trap 'kill $tailpid 2>/dev/null' EXIT
+          VM_ISO=/mnt/work/iso/steamify.iso scripts/vmbootloadertest.sh "${{ matrix.loader }}" --install
       - name: Logs
         if: always()
-        uses: christopherhx/gitea-upload-artifact@v4
+        uses: actions/upload-artifact@v4
         with:
           name: boot-${{ matrix.loader }}
           path: |
-            /root/vms/bl-${{ matrix.loader }}.test.log
+            /mnt/work/vms/bl-${{ matrix.loader }}.test.log
-            /root/vms/bl-${{ matrix.loader }}/vm.log
+            /mnt/work/vms/bl-${{ matrix.loader }}/vm.log
           retention-days: 7
           if-no-files-found: ignore
 
   suites:
-    name: Suites (vmtest.sh)
+    name: Suite ${{ matrix.suite }}
-    needs: [iso, boot]
+    needs: iso
     runs-on: ubuntu-latest
-    timeout-minutes: 150
+    timeout-minutes: 180
-    container:
+    strategy:
-      image: docker.io/cachyos/cachyos:latest
+      fail-fast: false
-      options: --privileged
+      matrix:
-    defaults:
+        suite: [toggles, hw, cli, 'menu installer']
-      run:
-        shell: bash
     steps:
-      - name: Node and git
-        id: node
-        run: |
-          pacman-key --init && pacman-key --populate
-          pacman -Syu --noconfirm --needed nodejs git
-          echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
-      - name: Cache - pacman packages
-        uses: actions/cache@v4
-        with:
-          path: /var/cache/pacman/pkg
-          key: pacman-test-${{ steps.node.outputs.week }}
-          restore-keys: pacman-test-
-      - name: Cache - the VMs' packages
-        uses: actions/cache@v4
-        with:
-          path: /root/vms/pkg-cache
-          key: vmpkg-steamify-${{ steps.node.outputs.week }}
-          restore-keys: vmpkg-
-      - name: Tools and KVM
-        run: |
-          pacman -S --noconfirm --needed qemu-desktop edk2-ovmf openssh libarchive procps-ng util-linux python curl sudo
-          [ -c /dev/kvm ] || { echo "::error::no /dev/kvm in this container: the runner needs KVM (and privileged containers)"; exit 1; }
-      - name: Cache - the ISO
-        uses: actions/cache@v4
-        with:
-          path: iso
-          key: steamify-iso-${{ needs.iso.outputs.tag }}
-
-      # vmtest.sh wants the live-iso repository too (which boot loaders the ISO advertises).
       - uses: actions/checkout@v4
         with:
           path: steamify-cachyos
@@ -169,41 +115,42 @@ jobs:
         with:
           repository: theupriser/steamify-cachyos-dev
           path: steamify-cachyos-dev
+      # vmtest.sh reads which boot loaders the ISO advertises from this repository.
       - uses: actions/checkout@v4
         with:
           repository: theupriser/steamify-cachyos-live-iso
           path: steamify-cachyos-live-iso
-      - name: ISO
+      - uses: ./steamify-cachyos/.github/actions/vm-runner
-        run: |
+        with:
-          mkdir -p iso
+          iso-tag: ${{ needs.iso.outputs.tag }}
-          [ -n "$(ls iso/*.iso 2>/dev/null)" ] || curl -fL --retry 5 --retry-all-errors -o "iso/${{ needs.iso.outputs.tag }}.iso" "${{ needs.iso.outputs.url }}"
+      # A plain VM for the suites: the ISO's installer with its Steamify step skipped (the suites start without Steamify).
       - name: Base VM from the ISO
         env:
           CI: "1"
           VM_TIMEZONE: UTC
-          VM_CPUS: "2"
+          VM_CPUS: "4"
           VM_INSTALL_MEM: 6G
           VM_STEAMIFY: skip
         run: |
-          mkdir -p ~/.ssh ~/vms/pkg-cache
-          ssh-keygen -q -t ed25519 -N "" -C steamify-vm -f ~/.ssh/steamify-vm_ed25519
           cd steamify-cachyos-dev
-          VM_DIR="$HOME/vms/steamify-vm" scripts/vminstall.sh --iso "$(ls "$GITHUB_WORKSPACE"/iso/*.iso | head -n 1)"
+          VM_DIR="$HOME/vms/steamify-vm" scripts/vminstall.sh --iso /mnt/work/iso/steamify.iso
-      - name: vmtest.sh (every suite)
+      - name: vmtest.sh ${{ matrix.suite }}
         env:
           CI: "1"
           VM_CPUS: "2"
           MAX_PARALLEL: "2"
         run: |
           cd steamify-cachyos-dev
-          scripts/vmtest.sh cli menu hw installer toggles
+          touch ~/vms/test.log; tail -n +1 -F ~/vms/test.log & tailpid=$!   # the suites' progress, as it happens
+          trap 'kill $tailpid 2>/dev/null' EXIT
+          scripts/vmtest.sh ${{ matrix.suite }}
       - name: Logs
         if: always()
-        uses: christopherhx/gitea-upload-artifact@v4
+        uses: actions/upload-artifact@v4
         with:
-          name: suites
+          name: suite-${{ strategy.job-index }}
           path: |
-            /root/vms/last-test.txt
+            /mnt/work/vms/last-test.txt
-            /root/vms/run-*.log
+            /mnt/work/vms/run-*.log
           retention-days: 7
           if-no-files-found: ignore