Work

.github/scripts/build-erofs-utils.sh

@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+# Build erofs-utils from source, patched, for the image-backed tests.
+#
+# INTERIM. This script exists because erofs-utils has no release binary aqua
+# could pin: it is a C project distributed as source, so the pinned toolchain
+# cannot carry an mkfs.erofs and every image-backed test skips itself without
+# one. The end state is a forkcloser release of erofs-utils (the patches this
+# script applies are already ours) pinned in aqua.yaml like any other tool, at
+# which point this script and its CI steps go away and `just test` alone is
+# the whole story.
+#
+# Until then, the doctrine ci.yaml states is honored as far as source builds
+# allow: exact versions, sha256-verified downloads, and one implementation
+# shared by every job that needs it (a divergence between two copies of this
+# was the classic failure of the workflow this replaced). What stays
+# unpinned, knowingly: the build-dependency packages from the runner's own
+# apt/brew repositories.
+#
+# Where it lands — and why that matters: `just` runs every recipe under limen's
+# HERMETIC PATH (aqua's bin plus the base system dirs; see .limen/just/main.just),
+# so a `make install` into /usr/local is invisible to `just test` and every
+# image-backed test skips itself while the log says the build succeeded. The
+# binary therefore installs into the project's own tool dir, build/erofs-utils/,
+# whose bin/ the root Justfile prepends to PATH — the one declared exception to
+# the hermetic list, and the same location on every OS (the windows cross-build
+# is copied there by CI). No sudo, no system prefix.
+#
+# Usage:
+#   build-erofs-utils.sh native    build and install for this host (linux or
+#                                  macOS) into build/erofs-utils/bin/mkfs.erofs
+#   build-erofs-utils.sh windows   cross-compile a static mkfs.erofs.exe with
+#                                  MinGW-w64 (linux host) into
+#                                  build/erofs-utils/bin/mkfs.erofs.exe
+# Set SKIP_DEPS=1 to skip the apt/brew build-dependency step (a developer
+# machine that already has autotools and lz4).
+set -euo pipefail
+
+EROFS_UTILS_VERSION="1.9.3"
+EROFS_UTILS_SHA256="17bfa54f4d370838c61081fce44022815a0366e282d777389589184414d5adc5"
+LZ4_VERSION="1.10.0"
+LZ4_SHA256="537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b"
+MINGW_HOST="x86_64-w64-mingw32"
+
+repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+patches="$repo/.github/workflows/patches/erofs-utils"
+headers="$repo/.github/workflows/mingw-compat-headers"
+work="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/erofs-utils-build"
+prefix="$repo/build/erofs-utils"
+
+die() {
+    echo "build-erofs-utils: $*" >&2
+    exit 1
+}
+
+# fetch <url> <sha256> <dest>: download and verify, or fail closed.
+fetch() {
+    local url=$1 sum=$2 dest=$3 actual
+    curl -fsSL -o "$dest" "$url"
+    if command -v sha256sum > /dev/null 2>&1; then
+        actual=$(sha256sum "$dest" | awk '{print $1}')
+    else
+        actual=$(shasum -a 256 "$dest" | awk '{print $1}')
+    fi
+    [ "$actual" = "$sum" ] || die "checksum mismatch for $url: want $sum, got $actual"
+}
+
+# unpack_erofs_utils: fetch, verify, extract, and patch the source into $work;
+# prints the source directory.
+unpack_erofs_utils() {
+    local tarball="$work/erofs-utils-${EROFS_UTILS_VERSION}.tar.gz"
+    local src="$work/erofs-utils-${EROFS_UTILS_VERSION}"
+    mkdir -p "$work"
+    rm -rf "$src"
+    fetch "https://github.com/erofs/erofs-utils/archive/refs/tags/v${EROFS_UTILS_VERSION}.tar.gz" \
+        "$EROFS_UTILS_SHA256" "$tarball"
+    tar -xzf "$tarball" -C "$work"
+    local p
+    for p in "$patches"/*.patch; do
+        patch -d "$src" -p1 < "$p" > /dev/null
+    done
+    echo "$src"
+}
+
+nproc_portable() {
+    nproc 2> /dev/null || sysctl -n hw.ncpu
+}
+
+build_native() {
+    if [ -z "${SKIP_DEPS:-}" ]; then
+        case "$(uname -s)" in
+            Linux)
+                sudo apt-get update -qq
+                sudo apt-get install -y -qq autoconf automake libtool pkg-config libz-dev liblz4-dev uuid-dev
+                ;;
+            Darwin)
+                brew install autoconf automake libtool pkg-config lz4
+                ;;
+            *) die "native build supports linux and macOS only (got $(uname -s))" ;;
+        esac
+    fi
+    local src
+    src=$(unpack_erofs_utils)
+    (
+        cd "$src"
+        ./autogen.sh
+        # configure caps the block size at the BUILD host's page size (bumped
+        # to 16K only when the build CPU is aarch64), so the same source
+        # yields a different mkfs per runner. Pin it: the 16384 leg of
+        # TestReadReferenceImage skips itself otherwise.
+        MAX_BLOCK_SIZE=16384 ./configure --enable-lz4 --prefix="$prefix"
+        make -j"$(nproc_portable)"
+        make install
+    )
+    "$prefix/bin/mkfs.erofs" -V
+    echo "installed $prefix/bin/mkfs.erofs"
+}
+
+build_windows() {
+    [ "$(uname -s)" = "Linux" ] || die "the windows cross-build needs a linux host"
+    if [ -z "${SKIP_DEPS:-}" ]; then
+        sudo apt-get update -qq
+        sudo apt-get install -y -qq autoconf automake libtool pkg-config mingw-w64
+    fi
+    mkdir -p "$work"
+
+    # lz4, static, into the mingw sysroot — the only library the cross build
+    # links; everything else is configured out below.
+    local lz4_tarball="$work/lz4-${LZ4_VERSION}.tar.gz"
+    fetch "https://github.com/lz4/lz4/archive/refs/tags/v${LZ4_VERSION}.tar.gz" "$LZ4_SHA256" "$lz4_tarball"
+    rm -rf "$work/lz4-${LZ4_VERSION}"
+    tar -xzf "$lz4_tarball" -C "$work"
+    (
+        cd "$work/lz4-${LZ4_VERSION}/lib"
+        make -j"$(nproc_portable)" \
+            CC="${MINGW_HOST}-gcc" AR="${MINGW_HOST}-ar" WINDRES="${MINGW_HOST}-windres" \
+            TARGET_OS=Windows BUILD_STATIC=yes BUILD_SHARED=no \
+            CFLAGS="-O3 -DXXH_NAMESPACE=LZ4_"
+        sudo make PREFIX="/usr/${MINGW_HOST}" install
+    )
+
+    local src
+    src=$(unpack_erofs_utils)
+    # The compat headers stand in for the POSIX surface MinGW lacks; they are
+    # force-included into every translation unit below.
+    sudo cp -r "$headers"/* "/usr/${MINGW_HOST}/include/"
+    (
+        cd "$src"
+        ./autogen.sh
+        # PKG_CONFIG_LIBDIR (not _PATH): _PATH prepends to the host's search
+        # dirs, so host .pc files leak into the cross build — v1.9.3's libxml2
+        # auto-probe found the runner's libxml-2.0.pc and put -lxml2 on a link
+        # line no mingw library can satisfy. _LIBDIR replaces the search path
+        # outright: only the mingw sysroot (where the cross-compiled lz4
+        # installs its .pc) is visible, and every other auto-probe fails closed.
+        # MAX_BLOCK_SIZE: same pin as the native build, same reason.
+        PKG_CONFIG_LIBDIR="/usr/${MINGW_HOST}/lib/pkgconfig" \
+            MAX_BLOCK_SIZE=16384 \
+            ./configure \
+            --host="${MINGW_HOST}" \
+            --disable-shared \
+            --enable-lz4 \
+            --without-zlib \
+            --disable-lzma \
+            --without-libzstd \
+            --without-selinux \
+            --without-uuid \
+            --without-openssl \
+            --without-libxml2 \
+            --disable-fuse \
+            --disable-debug \
+            --disable-dependency-tracking \
+            CFLAGS="-O2 -g -D_FILE_OFFSET_BITS=64" \
+            LDFLAGS="-Wl,-Bstatic -static-libgcc -L/usr/${MINGW_HOST}/lib" \
+            liblz4_LIBS="/usr/${MINGW_HOST}/lib/liblz4.a"
+        make -j"$(nproc_portable)" -C lib CPPFLAGS="-D_GNU_SOURCE -include posix_compat.h"
+        make -j"$(nproc_portable)" -C mkfs CPPFLAGS="-D_GNU_SOURCE -include posix_compat.h" LIBS="-llz4"
+        "${MINGW_HOST}-strip" mkfs/mkfs.erofs.exe
+    )
+    mkdir -p "$prefix/bin"
+    cp "$src/mkfs/mkfs.erofs.exe" "$prefix/bin/mkfs.erofs.exe"
+    echo "installed $prefix/bin/mkfs.erofs.exe"
+}
+
+case "${1:-}" in
+    native) build_native ;;
+    windows) build_windows ;;
+    *) die "usage: $0 native|windows" ;;
+esac

.github/workflows/ci.yaml

@@ -1,6 +1,9 @@
-# DO NOT EDIT MANUALLY.
-# This workflow is generic — no project-specific content — and is destined to
-# become part of the canonical baseline limen distributes.
+# Seeded from limen's canonical ci.yaml and the project's own from there:
+# `verify` and `gate` are the canonical shape, verbatim; the erofs-utils
+# pieces (a build step in `verify`, plus `mkfs-windows`, `verify-windows-image`
+# and `fuzz`) are this project's, and are INTERIM — see
+# .github/scripts/build-erofs-utils.sh for why they exist and what retires
+# them. Everything they add still answers to the canonical `gate`.
 #
 # Design: minimal GitHub glue around the same tooling every developer runs
 # locally. The only marketplace action is GitHub's own checkout, pinned by
@@ -80,9 +83,143 @@ jobs:
       - name: Lint
         run: just lint
 
+      # INTERIM: the image-backed tests shell out to mkfs.erofs, which the
+      # pinned toolchain cannot carry (no release binary to pin), so it is
+      # built from source here — checksum-verified, patched, one script for
+      # every job — into build/erofs-utils/bin, the one directory the root
+      # Justfile adds to limen's hermetic PATH (a /usr/local install would be
+      # invisible to `just test`). Not on windows: no native build there; the
+      # windows-2025 leg WITH an image is `verify-windows-image` below, fed by
+      # a cross-compile. The windows legs of this matrix run `just test`
+      # as-is and their image tests skip themselves — `mkfs-info` says so.
+      - name: Build erofs-utils (interim)
+        if: runner.os != 'Windows'
+        run: .github/scripts/build-erofs-utils.sh native
+
+      - name: Test
+        env:
+          # The legs that just built mkfs.erofs must FAIL if `just test`
+          # cannot see it — a silent fall-back to "every image test skipped"
+          # is a coverage regression, not a pass. Windows legs: not required.
+          EROFS_REQUIRE_MKFS: ${{ runner.os != 'Windows' && '1' || '' }}
+        run: just test
+
+  # INTERIM: cross-compile a static mkfs.erofs.exe so the windows leg below
+  # can run the image-backed tests. Artifact upload/download are GitHub's own
+  # actions, SHA-pinned — the same trust class as checkout — and go away with
+  # the rest of the erofs-utils machinery.
+  mkfs-windows:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Cross-compile mkfs.erofs for windows
+        run: .github/scripts/build-erofs-utils.sh windows
+
+      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: mkfs-erofs-windows
+          path: build/erofs-utils/bin/mkfs.erofs.exe
+          if-no-files-found: error
+
+  # INTERIM: the windows-2025 leg WITH an image. Same recipe as every other
+  # leg — `just test` — with mkfs.erofs on PATH; the difference from the
+  # canonical windows legs of `verify` is only that the image tests do not
+  # skip. Windows-arm is not covered with an image (the cross build is
+  # x86_64 and would run under emulation; not worth a leg for an interim).
+  verify-windows-image:
+    needs: [mkfs-windows]
+    runs-on: windows-2025
+    timeout-minutes: 45
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      # Into the same project-local tool dir the native builds use: the root
+      # Justfile's PATH addition finds it, no GITHUB_PATH plumbing needed.
+      - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
+        with:
+          name: mkfs-erofs-windows
+          path: build/erofs-utils/bin
+
+      - name: Install aqua (pinned, checksum-verified)
+        uses: ./.github/actions/setup-aqua
+
+      - name: Install pinned tools
+        run: aqua install --only-link
+
       - name: Test
+        env:
+          # This leg exists for the image tests: fail if mkfs.erofs is not seen.
+          EROFS_REQUIRE_MKFS: "1"
+          # mkfs.erofs uses TMPDIR for its temporaries; windows does not set it.
+          TMPDIR: ${{ runner.temp }}
         run: just test
 
+  # INTERIM in its plumbing, permanent in intent: a short fuzz of every Fuzz*
+  # target, once, on linux, with mkfs.erofs available (several targets round-
+  # trip through it). The recipe is limen's `do::test::go::fuzz` (verdict by
+  # crasher, hiccup retried once); only the erofs-utils build step is interim.
+  fuzz:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Install aqua (pinned, checksum-verified)
+        uses: ./.github/actions/setup-aqua
+
+      - name: Install pinned tools
+        run: aqua install --only-link
+
+      - name: Build erofs-utils (interim)
+        run: .github/scripts/build-erofs-utils.sh native
+
+      # The generated corpus is what makes fuzzing cumulative: each run
+      # starts from every interesting input earlier runs discovered rather
+      # than from the seeds. Key on the fuzz test sources so a changed
+      # target restarts its own corpus; restore-keys keep the rest. GitHub's
+      # own action, SHA-pinned.
+      - id: fuzzdir
+        run: echo "dir=$(go env GOCACHE)/fuzz" >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ steps.fuzzdir.outputs.dir }}
+          key: fuzz-corpus-${{ runner.os }}-${{ hashFiles('**/*_fuzz_test.go') }}
+          restore-keys: |
+            fuzz-corpus-${{ runner.os }}-
+
+      # Several targets round-trip through mkfs.erofs; without it they skip
+      # and the fuzz run is quietly worth less. Same guard as the test legs.
+      - name: Fuzz
+        env:
+          EROFS_REQUIRE_MKFS: "1"
+        run: |
+          just mkfs-info
+          just do test go fuzz
+
+      # Surface crashers as artifacts: the log names the target, but the
+      # input itself is what reproduces the bug locally.
+      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        if: failure()
+        with:
+          name: fuzz-crashers
+          path: testdata/fuzz/
+          if-no-files-found: ignore
+
   # The ONE required status check (see defaultRequiredChecks in
   # internal/github/audit.go). Branch protection names contexts as strings, so
   # requiring the matrix legs directly would bake this workflow's runner list
@@ -91,7 +228,7 @@ jobs:
   # whatever its shape, to a single stable name: change the legs above freely,
   # the ruleset never moves.
   gate:
-    needs: [verify]
+    needs: [verify, verify-windows-image, fuzz]
     # always(), and the result asserted explicitly. Without always() a failed
     # or cancelled dependency SKIPS this job instead of failing it, and a
     # skipped required check does not block a merge — branch protection that
@@ -102,11 +239,15 @@ jobs:
     timeout-minutes: 5
     permissions: {}
     steps:
-      - name: Every verify leg succeeded
+      - name: Every leg succeeded
         env:
           # Via env, never interpolated into the script: the shell sees data,
           # not something the expression layer can rewrite into code.
           RESULT: ${{ needs.verify.result }}
+          RESULT_WINDOWS_IMAGE: ${{ needs.verify-windows-image.result }}
+          RESULT_FUZZ: ${{ needs.fuzz.result }}
         run: |
           printf 'verify: %s\n' "$RESULT"
-          [ "$RESULT" = "success" ]
+          printf 'verify-windows-image: %s\n' "$RESULT_WINDOWS_IMAGE"
+          printf 'fuzz: %s\n' "$RESULT_FUZZ"
+          [ "$RESULT" = "success" ] && [ "$RESULT_WINDOWS_IMAGE" = "success" ] && [ "$RESULT_FUZZ" = "success" ]