Distribute images as a bundle, and stop paying for a boot nobody watches

.github/workflows/build.yml

@@ -79,7 +79,7 @@ jobs:
       - name: Install Dependencies
         run: |
           sudo apt update
-          sudo apt install -y fdisk dosfstools rsync curl python3
+          sudo apt install -y fdisk dosfstools rsync curl python3 zstd
 
           # qemu only for the qemu-backed architectures; vax uses SIMH.
           if [ -n '${{ matrix.architecture.qemu }}' ]; then
@@ -135,6 +135,11 @@ jobs:
             '${{ matrix.architecture.name }}' \
             $extra_args
 
+      # packer keeps the ISO in the workspace, which the action synchronizes
+      # into the guest: half a gigabyte with no use there.
+      - name: Remove the install media
+        run: rm -rf packer_cache
+
       # When the install goes wrong, packer only reports that it gave up
       # waiting for SSH. The guest console says what actually happened.
       #
@@ -187,17 +192,24 @@ jobs:
 
       - name: Create File
         if: matrix.architecture.name != 'vax'
-        run: echo 'host to guest' > host_to_guest.txt
+        run: |
+          echo 'host to guest' > host_to_guest.txt
+          # Read by the guest to check its clock against the host's.
+          date -u +%s > host_epoch.txt
 
       # qemu architectures are booted by cross-platform-actions/action.
+      #
+      # Pinned to the branch that can read the bundle rather than master, which
+      # still builds a .qcow2 URL. Change back to master once action#162 has
+      # merged.
       - name: Test Image
         if: matrix.architecture.name != 'vax'
-        uses: cross-platform-actions/action@master
+        uses: cross-platform-actions/action@claude/netbsd-11-microvm-kernel-wsrfrg
         with:
           operating_system: netbsd
           architecture: ${{ matrix.architecture.name }}
           version: '${{ matrix.version }}'
-          image_url: http://localhost:8080/netbsd-${{ matrix.version }}-${{ matrix.architecture.name }}.qcow2
+          image_url: http://localhost:8080/netbsd-${{ matrix.version }}-${{ matrix.architecture.name }}.tar.zst
           shutdown_vm: false
           shell: bash
           run: |
@@ -213,6 +225,9 @@ jobs:
             printf 'whoami: '; whoami
             printf 'hostname: '; hostname
             echo 'ls -lah:'; ls -lah
+            # What the synchronization actually carried in. The workspace holds
+            # the built image as well as the checkout, so this is worth seeing.
+            printf 'du -sh .: '; du -sh .
             echo 'env | sort:'; env | sort
             [ "`uname -s`" = 'NetBSD' ]
             [ "`uname -r`" = '${{ matrix.version }}' ]
@@ -256,27 +271,91 @@ jobs:
             cat host_to_guest.txt | grep -q 'host to guest'
             echo 'guest to host' > guest_to_host.txt
 
+            # No DHCP client any more (provision.sh freezes the lease), so
+            # assert the network it ships with works rather than trusting it.
+            echo 'network:'
+            # ifconfig lives in /sbin, which isn't in the PATH of the user the
+            # commands run as.
+            PATH="/sbin:/usr/sbin:$PATH"
+            [ ! -f /var/run/dhcpcd.pid ]
+            ifconfig -l
+            netstat -rn -f inet | grep -q '^default'
+            grep -q '^nameserver' /etc/resolv.conf
+
+            # HTTPS is attempted everywhere, but only 10 and later can pass it:
+            # certctl(8), which populates the system CA store, arrived in
+            # NetBSD 10, so on 9.x this fails on trust rather than on
+            # connectivity, which is not what is being asserted here. Tolerate
+            # that only there -- the uname test keeps a broken CA store on a
+            # release that has one from passing quietly. The clock is checked
+            # against the host's below in either case.
+            if curl -fsS -o /dev/null https://github.com; then
+              echo 'https: ok'
+            else
+              echo 'https: no CA store, asserting plain http instead'
+              [ "$(uname -r | cut -d. -f1)" -lt 10 ]
+              curl -fsS -o /dev/null http://github.com
+            fi
+
+            # Duplicate address detection is off, so the address is usable as
+            # soon as it is configured and rc doesn't wait for it. Assert the
+            # probe count, the absence of a `tentative` flag, and that the wait
+            # never ran -- it announces itself on the console, which rc logs.
+            [ "$(sysctl -n net.inet.ip.dad_count)" -eq 0 ]
+            [ "$(sysctl -n net.inet6.ip6.dad_count)" -eq 0 ]
+            interface=$(route -n get default | awk '$1 == "interface:" { print $2 }')
+            ifconfig "$interface" inet
+            ! ifconfig "$interface" inet | grep -q tentative
+            ! sudo grep -q 'duplicate address detection' /var/run/rc.log
+
+            # The clock comes from the emulated RTC now. The HTTPS fetch above
+            # already proves the right year, so this only catches a large skew.
+            echo 'clock:'
+            guest_epoch=$(date -u +%s)
+            host_epoch=$(cat host_epoch.txt)
+            skew=$((guest_epoch - host_epoch))
+            echo "guest is ${skew}s ahead of the host"
+            [ "$skew" -gt -60 ] && [ "$skew" -lt 600 ]
+
+            # The per-script boot timeline, when built with boot_timestamps.
+            echo 'rc log:'
+            sudo cat /var/run/rc.log || true
+
       - name: Verify File Synchronization
         if: matrix.architecture.name != 'vax'
         run: cat guest_to_host.txt | grep -q 'guest to host'
 
+      # The action still logs in with a generated key, but the image accepts an
+      # empty password now, which is how it will log in once it stops building
+      # the resources disk. The VM above is still up (shutdown_vm: false), so
+      # assert that path directly with every other method disabled.
+      - name: Test Passwordless Login
+        if: matrix.architecture.name != 'vax'
+        run: |
+          ssh -p 2847 \
+            -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            -o BatchMode=yes -o PubkeyAuthentication=no \
+            -o PasswordAuthentication=no -o GSSAPIAuthentication=no \
+            -o ConnectTimeout=60 runner@127.0.0.1 \
+            'echo "logged in as $(whoami) without a credential"'
+
       # cross-platform-actions/action is qemu-based and can't run a VAX
-      # image, so boot the built image under SIMH directly: decompress the
-      # distributed artifact to RAW (what a consumer does), autoboot the
-      # disk via EXPECT/SEND, wait for sshd, and run uname as the secondary
-      # user over ssh, the way the consumer logs in.
+      # image, so boot the built image under SIMH directly: unpack the
+      # distributed bundle (what a consumer does), autoboot the disk it holds
+      # via EXPECT/SEND, wait for sshd, and run uname as the secondary user
+      # over ssh, the way the consumer logs in.
       - name: Test Image (vax)
         if: matrix.architecture.name == 'vax'
         run: |
           set -eux
 
-          zstd -dc "output/netbsd-${{ matrix.version }}-vax.img.zst" > disk.raw
+          zstd -dc "output/netbsd-${{ matrix.version }}-vax.tar.zst" | tar -x
 
           # Autoboot the disk hands-free; forward host 2222 -> guest ssh.
           {
             echo "set cpu 64m"
             echo "set rq0 ra92"
-            echo "attach rq0 disk.raw"
+            echo "attach rq0 disk.img"
             echo "attach xq nat:tcp=2222:10.0.2.15:22"
             echo 'expect ">>>" send "boot dua0\r"; go'
             echo "boot cpu"
@@ -363,9 +442,9 @@ jobs:
         run: |
           version="${github_ref_name#v}"
 
-          # Ship only the distributable image: the compressed RAW for vax
-          # (not the 1.5 GB raw .img or console logs), the qcow2 otherwise.
-          artifacts="${{ matrix.architecture.name == 'vax' && 'output/*.img.zst' || 'output/*.qcow2' }}"
+          # One artifact per architecture now: the bundle holds the disk and,
+          # where a release publishes one, the microvm kernel (see build.sh).
+          artifacts="output/*.tar.zst"
 
           # Matrix jobs run in parallel; the first to arrive creates the
           # draft release, the rest upload their artifacts to it.

build.sh

@@ -59,6 +59,94 @@ download_install_media() {
 
 download_install_media || true
 
+# Turn the image's zero ranges into holes, which is what makes `tar --sparse`
+# worth asking for: tar uses SEEK_HOLE rather than looking for zeroes itself,
+# and the image arrives from the builder fully allocated, so it finds none and
+# archives all 12 GB. Digging first takes both ends down to the ~700 MB the
+# installation uses.
+#
+# `fallocate` is Linux-only and works in place. Elsewhere a raw-to-raw qemu-img
+# conversion skips the zero blocks and so writes a sparse file.
+sparsify() {
+  image="$1"
+
+  if command -v fallocate > /dev/null 2>&1; then
+    fallocate --dig-holes "$image"
+  else
+    qemu-img convert -f raw -O raw "$image" "$image.sparse"
+    mv "$image.sparse" "$image"
+  fi
+
+  echo "image: $(du -h "$image" | cut -f1) on disk, \
+$(ls -l "$image" | awk '{print $5}') bytes virtual"
+}
+
+# One artifact per image, holding a RAW `disk.img` and, where the release has
+# one, a `kernel`. The members are named generically so a consumer needs one
+# code path per platform. -19 --long=27 is the knee of the size/speed curve, and
+# a 128 MiB window is zstd's default decompression limit, so the consumer needs
+# no --long to unpack.
+bundle_image() {
+  raw="output/netbsd-$OS_VERSION-$ARCHITECTURE.img"
+  bundle="output/netbsd-$OS_VERSION-$ARCHITECTURE.tar.zst"
+
+  # Staged inside output/ so that renaming a 12 GB file stays a rename rather
+  # than becoming a copy onto another file system.
+  rm -rf output/bundle
+  mkdir -p output/bundle
+  mv "$raw" output/bundle/disk.img
+
+  members='disk.img'
+
+  if [ -f output/kernel ]; then
+    mv output/kernel output/bundle/kernel
+    members="$members kernel"
+  fi
+
+  sparsify output/bundle/disk.img
+
+  # shellcheck disable=SC2086
+  tar --sparse -C output/bundle -cf - $members \
+    | zstd -19 --long=27 -T0 -o "$bundle" -f
+
+  rm -rf output/bundle
+  ls -l "$bundle"
+}
+
+# The kernel for QEMU's `microvm` machine type. It goes beside the disk rather
+# than inside the image, because that machine type cannot boot from a disk: the
+# consumer passes it to QEMU with `-kernel`.
+#
+# Only releases with the MICROVM configuration publish one, so a release without
+# it is not an error. The archive mirror is tried second, as older releases move
+# there.
+download_microvm_kernel() {
+  [ "$ARCHITECTURE" = 'x86-64' ] || return 0
+
+  image_architecture=$(awk -F'"' '/^ *image *=/ { print $2 }' \
+    "var_files/$ARCHITECTURE.pkrvars.hcl")
+  target=output/kernel
+  mkdir -p output
+
+  for base in \
+    "https://cdn.netbsd.org/pub/NetBSD/NetBSD-$OS_VERSION" \
+    "https://archive.netbsd.org/pub/NetBSD-archive/NetBSD-$OS_VERSION"
+  do
+    url="$base/$image_architecture/binary/kernel/netbsd-MICROVM.gz"
+    echo "microvm kernel: trying $url"
+    rm -f /tmp/microvm-kernel.gz
+    curl -fL --connect-timeout 30 -o /tmp/microvm-kernel.gz "$url" || continue
+
+    gunzip -c /tmp/microvm-kernel.gz > "$target"
+    echo "microvm kernel: obtained from $url"
+    ls -l "$target"
+    return 0
+  done
+
+  echo "microvm kernel: NetBSD $OS_VERSION publishes none"
+  return 1
+}
+
 # NetBSD/VAX is built by the SIMH plugin from a separate template. It
 # shares var_files/common.pkrvars.hcl (the user/password identity) with
 # the qemu builds, but not the qemu-specific layers — the VAX template
@@ -108,16 +196,10 @@ if [ "$ARCHITECTURE" = "vax" ]; then
     "$@" \
     netbsd-vax.pkr.hcl
 
-  # Compress the RAW disk image for distribution. The qemu architectures
-  # get compression for free from qcow2; the SIMH RAW image does not, so
-  # we zstd it here. Build time is irrelevant, so compress hard: -19
-  # --long=27 is the knee of the size/speed curve (~90 MB from ~1.5 GB)
-  # and a 128 MiB window is exactly zstd's default decompression limit,
-  # so the consumer needs no --long flag to `zstd -d`. The consumer
-  # stream-decompresses back to RAW (least runtime overhead under SIMH):
-  #   curl -sL <url>/netbsd-<ver>-vax.img.zst | zstd -dc > disk.raw
-  image="output/netbsd-${OS_VERSION}-vax.img"
-  zstd -19 --long=27 -f "$image" -o "$image.zst"
+  # Same bundle as the qemu architectures, so the consumer has one code path.
+  # SIMH attaches RAW with the least runtime overhead, so the image here is RAW
+  # to begin with; there is no microvm kernel for vax.
+  bundle_image
 else
   packer init netbsd.pkr.hcl
 
@@ -129,4 +211,8 @@ else
     -var-file "var_files/$OS_VERSION/common.pkrvars.hcl" \
     "$@" \
     netbsd.pkr.hcl
+
+  # Before bundling: the kernel is a member of the bundle, not its own artifact.
+  download_microvm_kernel || true
+  bundle_image
 fi

changelog.md

@@ -5,6 +5,43 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Removed
+- The `rc.local` hook that mounted the consumer's FAT resources disk and
+    installed the SSH key from it. The passwordless login replaces it
+
+### Added
+- Publish the kernel for QEMU's `microvm` machine type inside the image bundle,
+    where a NetBSD release provides one
+- The `runner` user can log in over SSH without a credential on every
+    architecture, not just VAX, which lets a consumer stop building the FAT
+    resources disk that carries a generated key
+- New `boot_timestamps` build variable, which makes `/etc/rc` print a
+    timestamped line to the console as each `rc.d` script starts
+
+### Changed
+- Distribute every image as a bundle, `netbsd-<version>-<architecture>.tar.zst`,
+    holding a RAW `disk.img` and, where the release has one, a `kernel`. qcow2 is
+    gone: its own compression has to keep the image writable, so it compresses
+    worse than a solid stream, and the consumer pays that on every job. The 11.0
+    x86-64 image goes from 497 MiB to 268 MiB
+    ([action#151](https://github.com/cross-platform-actions/action/issues/151))
+- Turn the image's zero ranges into holes before archiving it. The image comes
+    out of the builder fully allocated, and `tar --sparse` asks the file system
+    where the holes are rather than looking for zeroes itself, so without this
+    the archive carries all 12 GB and the consumer writes all 12 GB back out
+    when it unpacks: 47 seconds, against the 4 that converting the qcow2 took
+- Freeze the address the hypervisor hands out into the static network
+    configuration and disable the DHCP client, taking it off the boot path to
+    `sshd`
+- Don't run `ntpdate` at boot. It blocks the boot on network round trips to
+    correct an offset that is already close to zero, since the emulated RTC is
+    seeded from the host clock. `ntpd` stays enabled, now with `-g`
+- Don't do duplicate address detection at all, and don't wait for it in
+    `/etc/rc.d/network`. An address is unusable while it is probed, so `sshd`
+    answered nothing for several seconds after it started listening. There is no
+    second host on the hypervisor's user mode network that could hold the
+    address
+
 ### Fixed
 - Stop the boot-time `ntpdate` being able to stall a NetBSD/VAX guest
     indefinitely. It runs inline in the boot sequence, ahead of `sshd`, and the

netbsd.pkr.hcl

@@ -169,13 +169,19 @@ variable "package_repository" {
   description = "The binary package repository to install the packages from. An empty value keeps the one the installer configured"
 }
 
+variable "boot_timestamps" {
+  default = false
+  type = bool
+  description = "Print a timestamped line to the console as every rc.d script starts, to attribute the boot time to individual scripts"
+}
+
 locals {
   iso_target_extension = "iso"
   iso_target_path = "packer_cache"
   iso_full_target_path = "${local.iso_target_path}/${sha1(var.checksum)}.${local.iso_target_extension}"
 
   image = "NetBSD-${var.os_version}-${var.architecture.image}.${local.iso_target_extension}"
-  vm_name = "netbsd-${var.os_version}-${var.architecture.name}.qcow2"
+  vm_name = "netbsd-${var.os_version}-${var.architecture.name}.img"
   full_remote_path = "images/${var.os_version}/${local.image}?key=NetBSD"
 }
 
@@ -185,10 +191,13 @@ source "qemu" "qemu" {
   memory = var.memory
   net_device = "virtio-net"
 
-  disk_compression = true
   disk_interface = "virtio"
   disk_size = var.disk_size
-  format = "qcow2"
+
+  # RAW, because that is what gets distributed (compressed with zstd, see
+  # build.sh). Asking the builder for it directly avoids converting the image
+  # afterwards. disk_compression is a qcow2-only option, so it's gone with it.
+  format = "raw"
 
   headless = var.headless
   use_default_display = var.use_default_display
@@ -340,7 +349,7 @@ source "qemu" "qemu" {
     ["-device", "virtio-scsi-pci"],
     ["-device", "scsi-hd,drive=drive0,bootindex=0"],
     ["-device", "scsi-cd,drive=drive1,bootindex=1"],
-    ["-drive", "if=none,file={{ .OutputDir }}/{{ .Name }},id=drive0,cache=writeback,discard=ignore,format=qcow2"],
+    ["-drive", "if=none,file={{ .OutputDir }}/{{ .Name }},id=drive0,cache=writeback,discard=ignore,format=raw"],
     ["-drive", "if=none,file=${local.iso_full_target_path},id=drive1,media=disk,format=raw,readonly=on"],
     ["-netdev", "user,id=user.0,hostfwd=tcp::{{ .SSHHostPort }}-:22,ipv6=off"]
   ]
@@ -388,7 +397,8 @@ build {
     environment_vars = [
       "SECONDARY_USER=${var.secondary_user_username}",
       "BOOT_CONSOLE=${var.boot_console}",
-      "PACKAGE_REPOSITORY=${var.package_repository}"
+      "PACKAGE_REPOSITORY=${var.package_repository}",
+      "BOOT_TIMESTAMPS=${var.boot_timestamps}"
     ]
   }