From 26e946cd33e1e7fbe2b12a726de085e266a0856e Mon Sep 17 00:00:00 2001 From: apostasie Date: Sat, 15 Aug 2026 19:55:11 -0700 Subject: [PATCH 1/5] Cleanup Signed-off-by: apostasie --- .github/scripts/build-erofs-utils.sh | 168 ++++++++++ .github/workflows/ci.yaml | 145 ++++++++- .github/workflows/erofs-utils-integration.yml | 291 ------------------ Justfile | 64 ++++ 4 files changed, 371 insertions(+), 297 deletions(-) create mode 100755 .github/scripts/build-erofs-utils.sh delete mode 100644 .github/workflows/erofs-utils-integration.yml diff --git a/.github/scripts/build-erofs-utils.sh b/.github/scripts/build-erofs-utils.sh new file mode 100755 index 0000000..327eea1 --- /dev/null +++ b/.github/scripts/build-erofs-utils.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Build erofs-utils from source, patched, for the image-backed tests. +# +# INTERIM. This script exists because erofs-utils has no release binary aqua +# could pin: it is a C project distributed as source, so the pinned toolchain +# cannot carry an mkfs.erofs and every image-backed test skips itself without +# one. The end state is a forkcloser release of erofs-utils (the patches this +# script applies are already ours) pinned in aqua.yaml like any other tool, at +# which point this script and its CI steps go away and `just test` alone is +# the whole story. +# +# Until then, the doctrine ci.yaml states is honored as far as source builds +# allow: exact versions, sha256-verified downloads, and one implementation +# shared by every job that needs it (a divergence between two copies of this +# was the classic failure of the workflow this replaced). What stays +# unpinned, knowingly: the build-dependency packages from the runner's own +# apt/brew repositories. +# +# Usage: +# build-erofs-utils.sh native build and `make install` for this host +# (linux or macOS); mkfs.erofs lands on PATH +# build-erofs-utils.sh windows cross-compile a static mkfs.erofs.exe with +# MinGW-w64 (linux host); prints its path +set -euo pipefail + +EROFS_UTILS_VERSION="1.9.3" +EROFS_UTILS_SHA256="17bfa54f4d370838c61081fce44022815a0366e282d777389589184414d5adc5" +LZ4_VERSION="1.10.0" +LZ4_SHA256="537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b" +MINGW_HOST="x86_64-w64-mingw32" + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +patches="$repo/.github/workflows/patches/erofs-utils" +headers="$repo/.github/workflows/mingw-compat-headers" +work="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/erofs-utils-build" + +die() { + echo "build-erofs-utils: $*" >&2 + exit 1 +} + +# fetch : download and verify, or fail closed. +fetch() { + local url=$1 sum=$2 dest=$3 actual + curl -fsSL -o "$dest" "$url" + if command -v sha256sum > /dev/null 2>&1; then + actual=$(sha256sum "$dest" | awk '{print $1}') + else + actual=$(shasum -a 256 "$dest" | awk '{print $1}') + fi + [ "$actual" = "$sum" ] || die "checksum mismatch for $url: want $sum, got $actual" +} + +# unpack_erofs_utils: fetch, verify, extract, and patch the source into $work; +# prints the source directory. +unpack_erofs_utils() { + local tarball="$work/erofs-utils-${EROFS_UTILS_VERSION}.tar.gz" + local src="$work/erofs-utils-${EROFS_UTILS_VERSION}" + mkdir -p "$work" + rm -rf "$src" + fetch "https://github.com/erofs/erofs-utils/archive/refs/tags/v${EROFS_UTILS_VERSION}.tar.gz" \ + "$EROFS_UTILS_SHA256" "$tarball" + tar -xzf "$tarball" -C "$work" + local p + for p in "$patches"/*.patch; do + patch -d "$src" -p1 < "$p" > /dev/null + done + echo "$src" +} + +nproc_portable() { + nproc 2> /dev/null || sysctl -n hw.ncpu +} + +build_native() { + case "$(uname -s)" in + Linux) + sudo apt-get update -qq + sudo apt-get install -y -qq autoconf automake libtool pkg-config libz-dev liblz4-dev uuid-dev + ;; + Darwin) + brew install autoconf automake libtool pkg-config lz4 + ;; + *) die "native build supports linux and macOS only (got $(uname -s))" ;; + esac + local src + src=$(unpack_erofs_utils) + ( + cd "$src" + ./autogen.sh + # configure caps the block size at the BUILD host's page size (bumped + # to 16K only when the build CPU is aarch64), so the same source + # yields a different mkfs per runner. Pin it: the 16384 leg of + # TestReadReferenceImage skips itself otherwise. + MAX_BLOCK_SIZE=16384 ./configure --enable-lz4 + make -j"$(nproc_portable)" + sudo make install + ) + mkfs.erofs -V +} + +build_windows() { + [ "$(uname -s)" = "Linux" ] || die "the windows cross-build needs a linux host" + sudo apt-get update -qq + sudo apt-get install -y -qq autoconf automake libtool pkg-config mingw-w64 + mkdir -p "$work" + + # lz4, static, into the mingw sysroot — the only library the cross build + # links; everything else is configured out below. + local lz4_tarball="$work/lz4-${LZ4_VERSION}.tar.gz" + fetch "https://github.com/lz4/lz4/archive/refs/tags/v${LZ4_VERSION}.tar.gz" "$LZ4_SHA256" "$lz4_tarball" + rm -rf "$work/lz4-${LZ4_VERSION}" + tar -xzf "$lz4_tarball" -C "$work" + ( + cd "$work/lz4-${LZ4_VERSION}/lib" + make -j"$(nproc_portable)" \ + CC="${MINGW_HOST}-gcc" AR="${MINGW_HOST}-ar" WINDRES="${MINGW_HOST}-windres" \ + TARGET_OS=Windows BUILD_STATIC=yes BUILD_SHARED=no \ + CFLAGS="-O3 -DXXH_NAMESPACE=LZ4_" + sudo make PREFIX="/usr/${MINGW_HOST}" install + ) + + local src + src=$(unpack_erofs_utils) + # The compat headers stand in for the POSIX surface MinGW lacks; they are + # force-included into every translation unit below. + sudo cp -r "$headers"/* "/usr/${MINGW_HOST}/include/" + ( + cd "$src" + ./autogen.sh + # PKG_CONFIG_LIBDIR (not _PATH): _PATH prepends to the host's search + # dirs, so host .pc files leak into the cross build — v1.9.3's libxml2 + # auto-probe found the runner's libxml-2.0.pc and put -lxml2 on a link + # line no mingw library can satisfy. _LIBDIR replaces the search path + # outright: only the mingw sysroot (where the cross-compiled lz4 + # installs its .pc) is visible, and every other auto-probe fails closed. + # MAX_BLOCK_SIZE: same pin as the native build, same reason. + PKG_CONFIG_LIBDIR="/usr/${MINGW_HOST}/lib/pkgconfig" \ + MAX_BLOCK_SIZE=16384 \ + ./configure \ + --host="${MINGW_HOST}" \ + --disable-shared \ + --enable-lz4 \ + --without-zlib \ + --disable-lzma \ + --without-libzstd \ + --without-selinux \ + --without-uuid \ + --without-openssl \ + --without-libxml2 \ + --disable-fuse \ + --disable-debug \ + --disable-dependency-tracking \ + CFLAGS="-O2 -g -D_FILE_OFFSET_BITS=64" \ + LDFLAGS="-Wl,-Bstatic -static-libgcc -L/usr/${MINGW_HOST}/lib" \ + liblz4_LIBS="/usr/${MINGW_HOST}/lib/liblz4.a" + make -j"$(nproc_portable)" -C lib CPPFLAGS="-D_GNU_SOURCE -include posix_compat.h" + make -j"$(nproc_portable)" -C mkfs CPPFLAGS="-D_GNU_SOURCE -include posix_compat.h" LIBS="-llz4" + "${MINGW_HOST}-strip" mkfs/mkfs.erofs.exe + ) + echo "$src/mkfs/mkfs.erofs.exe" +} + +case "${1:-}" in + native) build_native ;; + windows) build_windows ;; + *) die "usage: $0 native|windows" ;; +esac diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2067ef7..4303df7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,9 @@ -# DO NOT EDIT MANUALLY. -# This workflow is generic — no project-specific content — and is destined to -# become part of the canonical baseline limen distributes. +# Seeded from limen's canonical ci.yaml and the project's own from there: +# `verify` and `gate` are the canonical shape, verbatim; the erofs-utils +# pieces (a build step in `verify`, plus `mkfs-windows`, `verify-windows-image` +# and `fuzz`) are this project's, and are INTERIM — see +# .github/scripts/build-erofs-utils.sh for why they exist and what retires +# them. Everything they add still answers to the canonical `gate`. # # Design: minimal GitHub glue around the same tooling every developer runs # locally. The only marketplace action is GitHub's own checkout, pinned by @@ -80,9 +83,135 @@ jobs: - name: Lint run: just lint + # INTERIM: the image-backed tests shell out to mkfs.erofs, which the + # pinned toolchain cannot carry (no release binary to pin), so it is + # built from source here — checksum-verified, patched, one script for + # every job. Not on windows: no native build; the windows-2025 leg with + # an image is `verify-windows-image` below, fed by a cross-compile. The + # windows legs of this matrix run `just test` as-is, and `mkfs-info` + # says so in the log: image tests skip themselves there. + - name: Build erofs-utils (interim) + if: runner.os != 'Windows' + run: .github/scripts/build-erofs-utils.sh native + - name: Test run: just test + # INTERIM: cross-compile a static mkfs.erofs.exe so the windows leg below + # can run the image-backed tests. Artifact upload/download are GitHub's own + # actions, SHA-pinned — the same trust class as checkout — and go away with + # the rest of the erofs-utils machinery. + mkfs-windows: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Cross-compile mkfs.erofs for windows + run: | + exe=$(.github/scripts/build-erofs-utils.sh windows | tail -n 1) + mkdir -p build/mkfs-windows + cp "$exe" build/mkfs-windows/mkfs.erofs.exe + + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: mkfs-erofs-windows + path: build/mkfs-windows/mkfs.erofs.exe + if-no-files-found: error + + # INTERIM: the windows-2025 leg WITH an image. Same recipe as every other + # leg — `just test` — with mkfs.erofs on PATH; the difference from the + # canonical windows legs of `verify` is only that the image tests do not + # skip. Windows-arm is not covered with an image (the cross build is + # x86_64 and would run under emulation; not worth a leg for an interim). + verify-windows-image: + needs: [mkfs-windows] + runs-on: windows-2025 + timeout-minutes: 45 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: mkfs-erofs-windows + path: build/mkfs-windows + + - name: Put mkfs.erofs on PATH + run: | + echo "$GITHUB_WORKSPACE/build/mkfs-windows" >> "$GITHUB_PATH" + + - name: Install aqua (pinned, checksum-verified) + uses: ./.github/actions/setup-aqua + + - name: Install pinned tools + run: aqua install --only-link + + - name: Test + env: + # mkfs.erofs uses TMPDIR for its temporaries; windows does not set it. + TMPDIR: ${{ runner.temp }} + run: | + mkfs.erofs -V + just test + + # INTERIM in its plumbing, permanent in intent: a short fuzz of every Fuzz* + # target, once, on linux, with mkfs.erofs available (several targets round- + # trip through it). The recipe is the project's `fuzz` until limen ships + # `do::test::go::fuzz`; the erofs-utils build step leaves with the rest. + fuzz: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install aqua (pinned, checksum-verified) + uses: ./.github/actions/setup-aqua + + - name: Install pinned tools + run: aqua install --only-link + + - name: Build erofs-utils (interim) + run: .github/scripts/build-erofs-utils.sh native + + # The generated corpus is what makes fuzzing cumulative: each run + # starts from every interesting input earlier runs discovered rather + # than from the seeds. Key on the fuzz test sources so a changed + # target restarts its own corpus; restore-keys keep the rest. GitHub's + # own action, SHA-pinned. + - id: fuzzdir + run: echo "dir=$(go env GOCACHE)/fuzz" >> "$GITHUB_OUTPUT" + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.fuzzdir.outputs.dir }} + key: fuzz-corpus-${{ runner.os }}-${{ hashFiles('**/*_fuzz_test.go') }} + restore-keys: | + fuzz-corpus-${{ runner.os }}- + + - name: Fuzz + run: just fuzz + + # Surface crashers as artifacts: the log names the target, but the + # input itself is what reproduces the bug locally. + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + if: failure() + with: + name: fuzz-crashers + path: testdata/fuzz/ + if-no-files-found: ignore + # The ONE required status check (see defaultRequiredChecks in # internal/github/audit.go). Branch protection names contexts as strings, so # requiring the matrix legs directly would bake this workflow's runner list @@ -91,7 +220,7 @@ jobs: # whatever its shape, to a single stable name: change the legs above freely, # the ruleset never moves. gate: - needs: [verify] + needs: [verify, verify-windows-image, fuzz] # always(), and the result asserted explicitly. Without always() a failed # or cancelled dependency SKIPS this job instead of failing it, and a # skipped required check does not block a merge — branch protection that @@ -102,11 +231,15 @@ jobs: timeout-minutes: 5 permissions: {} steps: - - name: Every verify leg succeeded + - name: Every leg succeeded env: # Via env, never interpolated into the script: the shell sees data, # not something the expression layer can rewrite into code. RESULT: ${{ needs.verify.result }} + RESULT_WINDOWS_IMAGE: ${{ needs.verify-windows-image.result }} + RESULT_FUZZ: ${{ needs.fuzz.result }} run: | printf 'verify: %s\n' "$RESULT" - [ "$RESULT" = "success" ] + printf 'verify-windows-image: %s\n' "$RESULT_WINDOWS_IMAGE" + printf 'fuzz: %s\n' "$RESULT_FUZZ" + [ "$RESULT" = "success" ] && [ "$RESULT_WINDOWS_IMAGE" = "success" ] && [ "$RESULT_FUZZ" = "success" ] diff --git a/.github/workflows/erofs-utils-integration.yml b/.github/workflows/erofs-utils-integration.yml deleted file mode 100644 index 7d7c8eb..0000000 --- a/.github/workflows/erofs-utils-integration.yml +++ /dev/null @@ -1,291 +0,0 @@ -# Inherited, and not yet folded into the limen pipeline. -# -# Everything here exists for one reason: the tests that read a real image shell -# out to mkfs.erofs, and erofs-utils is a C project distributed as source — it -# has no release binary aqua could pin, so `just test` under the hermetic PATH -# cannot reach it and every image-backed test skips itself. This workflow builds -# erofs-utils (patched) from source and runs the suite against it, on linux, -# macos, and — via a MinGW cross-compile — windows, plus the fuzz targets. -# -# Retiring it means giving the pinned toolchain an mkfs.erofs, after which these -# jobs become `just` recipes like any other and this file goes away. Until then -# ci.yaml ("ci") is the authority on everything that does NOT need an image, and -# this workflow covers only what it cannot. -# -# Deliberately absent: a lint job. ci.yaml runs `just lint` — the repo's pinned -# golangci-lint (aqua.yaml), its .golangci.yml, once per supported GOOS. The job -# that used to live here ran an action-supplied golangci-lint v2.1 against the -# same code, so the two could disagree about the same tree. -name: erofs-utils integration - -on: - push: - branches: [main, ci-test] - pull_request: - branches: [main] - -permissions: - contents: read - -env: - EROFS_UTILS_VERSION: v1.9.3 - -jobs: - build-and-test: - name: Build & Test (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version: "1.25" - cache: false - - - name: Install erofs-utils build dependencies (Linux) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y autoconf automake libtool pkg-config libz-dev liblz4-dev uuid-dev - - - name: Install erofs-utils build dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install autoconf automake libtool pkg-config lz4 - - - name: Build and install erofs-utils - run: | - curl -L https://github.com/erofs/erofs-utils/archive/refs/tags/${EROFS_UTILS_VERSION}.tar.gz | tar -xzf - - cd erofs-utils-${EROFS_UTILS_VERSION#v} - for p in "${GITHUB_WORKSPACE}"/.github/workflows/patches/erofs-utils/*.patch; do - patch -p1 < "$p" - done - ./autogen.sh - # configure caps the block size at the BUILD host's page size - # (bumped to 16K only when the build CPU is aarch64), so the - # same source yields a different mkfs per runner. Pin it: the - # 16384 leg of TestReadReferenceImage skips itself otherwise. - MAX_BLOCK_SIZE=16384 ./configure --enable-lz4 - make -j"$(nproc 2>/dev/null || sysctl -n hw.ncpu)" - sudo make install - mkfs.erofs -V - - - name: Build - run: go build ./... - - - name: Test - run: go test -v ./... - - fuzz: - name: Fuzz - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version: stable - cache: false - - - name: Install erofs-utils build dependencies - run: | - sudo apt-get update - sudo apt-get install -y autoconf automake libtool pkg-config libz-dev liblz4-dev uuid-dev - - - name: Build and install erofs-utils - run: | - curl -L https://github.com/erofs/erofs-utils/archive/refs/tags/${EROFS_UTILS_VERSION}.tar.gz | tar -xzf - - cd erofs-utils-${EROFS_UTILS_VERSION#v} - for p in "${GITHUB_WORKSPACE}"/.github/workflows/patches/erofs-utils/*.patch; do - patch -p1 < "$p" - done - ./autogen.sh - # configure caps the block size at the BUILD host's page size - # (bumped to 16K only when the build CPU is aarch64), so the - # same source yields a different mkfs per runner. Pin it: the - # 16384 leg of TestReadReferenceImage skips itself otherwise. - MAX_BLOCK_SIZE=16384 ./configure --enable-lz4 - make -j"$(nproc)" - sudo make install - mkfs.erofs -V - - # The generated corpus is what makes fuzzing cumulative: each run - # starts from every interesting input earlier runs discovered rather - # than from the seeds. Key on the fuzz test sources so a changed - # target restarts its own corpus; restore-keys keep the rest. - - id: fuzzdir - run: echo "dir=$(go env GOCACHE)/fuzz" >> "$GITHUB_OUTPUT" - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ steps.fuzzdir.outputs.dir }} - key: fuzz-corpus-${{ runner.os }}-${{ hashFiles('**/*_fuzz_test.go') }} - restore-keys: | - fuzz-corpus-${{ runner.os }}- - - - name: Fuzz - run: | - fuzz_time=10s - # Each target is driven by `go test -fuzz` itself, not a - # prebuilt binary: only the go tool's fuzz build compiles in the - # coverage counters, and without them the engine mutates blind - # ("not built with coverage instrumentation ... may be - # inefficient" — it was random byte-flipping, not fuzzing). The - # per-target rebuild is a cached second or two; the first run - # below warms it. - # - # A real fuzz failure always writes the failing input under - # testdata/fuzz//. The coordinator can also report - # "context deadline exceeded" when a worker is mid-iteration as - # fuzztime expires — that is a shutdown hiccup, not a finding, - # and Wide targets (200-entry ReadDir per iteration) hit it - # most. So the verdict comes from the crasher, not the exit - # code: exit 1 without a new testdata file is retried once - # (a second hiccup in a row is treated as real). - targets=$(go test -list 'Fuzz.*' . 2>/dev/null | grep '^Fuzz') - echo "targets: $(echo "$targets" | wc -w)" - fail=0 - for target in $targets; do - echo "::group::$target" - before=$(find "testdata/fuzz/$target" -type f 2>/dev/null | wc -l) - ok=0 - for attempt in 1 2; do - if go test -fuzz="^${target}\$" -run='^$' -fuzztime=$fuzz_time -timeout=180s . ; then - ok=1; break - fi - after=$(find "testdata/fuzz/$target" -type f 2>/dev/null | wc -l) - if [ "$after" -gt "$before" ]; then - echo "::error::$target: new crasher written to testdata/fuzz/$target" - break - fi - echo "$target: exit without a crasher (attempt $attempt) — coordinator shutdown hiccup, retrying" - done - if [ "$ok" = 1 ]; then echo "PASS: $target"; else fail=1; fi - echo "::endgroup::" - done - exit $fail - - # Surface crashers as artifacts: the log names the target, but the - # input itself is what reproduces the bug locally. - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - if: failure() - with: - name: fuzz-crashers - path: testdata/fuzz/ - if-no-files-found: ignore - - cross-compile-mkfs-windows: - name: Cross-compile mkfs.erofs for Windows - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Install MinGW-w64 toolchain - run: | - sudo apt-get update - sudo apt-get install -y autoconf automake libtool pkg-config mingw-w64 - - - name: Cross-compile lz4 for Windows - env: - MINGW_HOST: x86_64-w64-mingw32 - run: | - curl -L https://github.com/lz4/lz4/archive/refs/tags/v1.10.0.tar.gz | tar -xzf - - cd lz4-1.10.0/lib - make -j"$(nproc)" \ - CC=${MINGW_HOST}-gcc \ - AR=${MINGW_HOST}-ar \ - WINDRES=${MINGW_HOST}-windres \ - TARGET_OS=Windows \ - BUILD_STATIC=yes \ - BUILD_SHARED=no \ - CFLAGS="-O3 -DXXH_NAMESPACE=LZ4_" - sudo make PREFIX=/usr/${MINGW_HOST} install - - - name: Cross-compile erofs-utils for Windows - env: - MINGW_HOST: x86_64-w64-mingw32 - run: | - curl -L https://github.com/erofs/erofs-utils/archive/refs/tags/${EROFS_UTILS_VERSION}.tar.gz | tar -xzf - - cd erofs-utils-${EROFS_UTILS_VERSION#v} - - # Install compat headers for MinGW - sudo cp -r "${GITHUB_WORKSPACE}/.github/workflows/mingw-compat-headers"/* /usr/${MINGW_HOST}/include/ - - # Apply Windows compatibility patches - for p in "${GITHUB_WORKSPACE}"/.github/workflows/patches/erofs-utils/*.patch; do - patch -p1 < "$p" - done - - ./autogen.sh - # PKG_CONFIG_LIBDIR (not _PATH): _PATH prepends to the host's - # search dirs, so host .pc files leak into the cross build — - # v1.9.3's libxml2 auto-probe found the runner's libxml-2.0.pc - # and put -lxml2 on a link line no mingw library can satisfy. - # _LIBDIR replaces the search path outright: only the mingw - # sysroot (where the cross-compiled lz4 installs its .pc) is - # visible, and every other auto-probe fails closed. - PKG_CONFIG_LIBDIR=/usr/${MINGW_HOST}/lib/pkgconfig \ - MAX_BLOCK_SIZE=16384 \ - ./configure \ - --host=${MINGW_HOST} \ - --disable-shared \ - --enable-lz4 \ - --without-zlib \ - --disable-lzma \ - --without-libzstd \ - --without-selinux \ - --without-uuid \ - --without-openssl \ - --without-libxml2 \ - --disable-fuse \ - --disable-debug \ - --disable-dependency-tracking \ - CFLAGS="-O2 -g -D_FILE_OFFSET_BITS=64" \ - LDFLAGS="-Wl,-Bstatic -static-libgcc -L/usr/${MINGW_HOST}/lib" \ - liblz4_LIBS="/usr/${MINGW_HOST}/lib/liblz4.a" - - make -j"$(nproc)" -C lib CPPFLAGS="-D_GNU_SOURCE -include posix_compat.h" - make -j"$(nproc)" -C mkfs CPPFLAGS="-D_GNU_SOURCE -include posix_compat.h" LIBS="-llz4" - - ${MINGW_HOST}-strip mkfs/mkfs.erofs.exe - - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: mkfs-erofs-windows - path: erofs-utils-*/mkfs/mkfs.erofs.exe - - test-windows: - name: Build & Test (windows-latest) - runs-on: windows-latest - needs: cross-compile-mkfs-windows - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version: "1.25" - cache: false - - - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - name: mkfs-erofs-windows - - - name: Install mkfs.erofs - shell: pwsh - run: | - New-Item -ItemType Directory -Force -Path "$env:USERPROFILE\bin" - Copy-Item erofs-utils-*/mkfs/mkfs.erofs.exe "$env:USERPROFILE\bin\" - "$env:USERPROFILE\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 - - - name: Verify mkfs.erofs - shell: bash - run: mkfs.erofs -V - - - name: Build - run: go build ./... - - - name: Test - env: - # mkfs.erofs uses TMPDIR for temporary files; Windows doesn't set it - TMPDIR: ${{ runner.temp }} - run: go test -v ./... diff --git a/Justfile b/Justfile index 094114a..760897d 100644 --- a/Justfile +++ b/Justfile @@ -22,3 +22,67 @@ mkfs-info: else echo "mkfs.erofs: NOT FOUND — every image-backed test will skip itself" fi + +# Fuzz smoke: every Fuzz* target, each for a short budget — a regression net, +# not a campaign. TEMPORARY, and verbatim what limen's `do::test::go::fuzz` +# will be once a release ships it; the day `just do test go fuzz` exists, this +# recipe is deleted and CI's fuzz job calls that instead. Not part of `test`: +# it needs mkfs.erofs on PATH to be worth anything (see mkfs-info) and runs +# once, on linux, in CI. +# +# Driven by `go test -fuzz` itself, never a prebuilt test binary: only the go +# tool's fuzz build compiles in the coverage counters, and without them the +# engine mutates blind. The verdict comes from the crasher, not the exit code: +# a real finding always writes the failing input under testdata/fuzz// +# (commit it — it is a regression test from then on), while the coordinator +# can also exit 1 with "context deadline exceeded" when a worker is +# mid-iteration as fuzztime expires — a shutdown hiccup, not a finding, and +# the Wide targets hit it most. So exit 1 WITH a new testdata file fails; exit +# 1 without one is retried once, and a second in a row is treated as real. The +# generated corpus lives under $(go env GOCACHE)/fuzz — CI caches it between +# runs so fuzzing is cumulative. +[doc('Run every Fuzz* target for TEST_GO_FUZZ_TIME (default 10s) each')] +fuzz: + #!/usr/bin/env bash + set -euo pipefail + fuzztime="${TEST_GO_FUZZ_TIME:-10s}" + # Crasher count for a target: the directory does not exist until the first + # finding, and a missing directory is zero, not an error (pipefail). + crashers() { + if [ -d "$1" ]; then find "$1" -type f | wc -l; else echo 0; fi + } + found=0 + fail=0 + for pkg in $(go list ./...); do + targets=$(go test -count=1 -run '^$' -list '^Fuzz' "$pkg" | grep '^Fuzz' || true) + [ -n "$targets" ] || continue + dir=$(go list -f '{{{{.Dir}}' "$pkg") + for target in $targets; do + found=1 + echo "fuzzing $pkg $target for $fuzztime" + before=$(crashers "$dir/testdata/fuzz/$target") + ok=0 + for attempt in 1 2; do + if go test -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" \ + -run "^${target}\$" -fuzz "^${target}\$" -fuzztime "$fuzztime" "$pkg"; then + ok=1 + break + fi + after=$(crashers "$dir/testdata/fuzz/$target") + if [ "$after" -gt "$before" ]; then + echo "$target: new crasher written under $dir/testdata/fuzz/$target — commit it as a regression test" >&2 + break + fi + echo "$target: exit without a crasher (attempt $attempt) — coordinator shutdown hiccup, retrying" >&2 + done + if [ "$ok" -eq 1 ]; then + echo "PASS: $target" + else + fail=1 + fi + done + done + if [ "$found" -eq 0 ]; then + echo "no Fuzz* targets in this module — nothing to fuzz" + fi + exit "$fail" From 108183a5ceb75152273552b50f69fefe2a33aed7 Mon Sep 17 00:00:00 2001 From: apostasie Date: Sat, 15 Aug 2026 20:12:01 -0700 Subject: [PATCH 2/5] Limen up / fuzz Signed-off-by: apostasie --- .github/workflows/ci.yaml | 6 ++-- .limen/just/test-go.just | 65 +++++++++++++++++++++++++++++++++++++++ Justfile | 64 -------------------------------------- aqua-checksums.json | 20 ++++++------ aqua.yaml | 2 +- 5 files changed, 79 insertions(+), 78 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4303df7..77d0bb5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -165,8 +165,8 @@ jobs: # INTERIM in its plumbing, permanent in intent: a short fuzz of every Fuzz* # target, once, on linux, with mkfs.erofs available (several targets round- - # trip through it). The recipe is the project's `fuzz` until limen ships - # `do::test::go::fuzz`; the erofs-utils build step leaves with the rest. + # trip through it). The recipe is limen's `do::test::go::fuzz` (verdict by + # crasher, hiccup retried once); only the erofs-utils build step is interim. fuzz: runs-on: ubuntu-24.04 timeout-minutes: 30 @@ -201,7 +201,7 @@ jobs: fuzz-corpus-${{ runner.os }}- - name: Fuzz - run: just fuzz + run: just do test go fuzz # Surface crashers as artifacts: the log names the target, but the # input itself is what reproduces the bug locally. diff --git a/.limen/just/test-go.just b/.limen/just/test-go.just index 2d246c1..9631be9 100644 --- a/.limen/just/test-go.just +++ b/.limen/just/test-go.just @@ -56,6 +56,71 @@ bench: (_banner "test go" "bench") CGO_ENABLED="{{ go_cgo }}" \ go test -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" -run '^$' -bench . -benchmem ./... +# Fuzz smoke: every Fuzz* target in every package, each for a short budget — +# a regression net, not a campaign. The seed corpus of the target runs first +# (that is what -run selects), then the engine mutates for TEST_GO_FUZZ_TIME +# (default 10s; export from the root Justfile to raise it). Driven by +# `go test -fuzz` itself, never a prebuilt test binary: only the go tool's +# fuzz build compiles in the coverage counters, and without them the engine +# mutates blind. `go test -fuzz` accepts one target per invocation, hence the +# loop; targets are discovered with -list rather than grep so build tags and +# _test.go placement are Go's call, not a regex's. +# +# The verdict comes from the crasher, not the exit code. A real finding always +# writes the failing input under the package's testdata/fuzz// (commit +# it — it is a regression test from then on). But the coordinator can also +# exit 1 with "context deadline exceeded" when a worker is mid-iteration as +# fuzztime expires — a shutdown hiccup, not a finding, and slow targets hit it +# most. So: exit 1 WITH a new testdata file is a failure; exit 1 without one +# is retried once, and a second in a row is treated as real. The generated +# corpus lives under $(go env GOCACHE)/fuzz — cache that between CI runs and +# fuzzing becomes cumulative. A tree with no fuzz targets says so and passes. +fuzz: (_banner "test go" "fuzz") + #!/usr/bin/env bash + set -euo pipefail + export CGO_ENABLED="{{ go_cgo }}" + fuzztime="${TEST_GO_FUZZ_TIME:-10s}" + # Crasher count for a target: the directory does not exist until the first + # finding, and a missing directory is zero, not an error (pipefail). + crashers() { + if [ -d "$1" ]; then find "$1" -type f | wc -l; else echo 0; fi + } + found=0 + fail=0 + for pkg in $(go list ./...); do + targets=$(go test -count=1 -run '^$' -list '^Fuzz' "$pkg" | grep '^Fuzz' || true) + [ -n "$targets" ] || continue + dir=$(go list -f '{{{{.Dir}}' "$pkg") + for target in $targets; do + found=1 + echo "fuzzing $pkg $target for $fuzztime" + before=$(crashers "$dir/testdata/fuzz/$target") + ok=0 + for attempt in 1 2; do + if go test -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" \ + -run "^${target}\$" -fuzz "^${target}\$" -fuzztime "$fuzztime" "$pkg"; then + ok=1 + break + fi + after=$(crashers "$dir/testdata/fuzz/$target") + if [ "$after" -gt "$before" ]; then + echo "$target: new crasher written under $dir/testdata/fuzz/$target — commit it as a regression test" >&2 + break + fi + echo "$target: exit without a crasher (attempt $attempt) — coordinator shutdown hiccup, retrying" >&2 + done + if [ "$ok" -eq 1 ]; then + echo "PASS: $target" + else + fail=1 + fi + done + done + if [ "$found" -eq 0 ]; then + echo "no Fuzz* targets in this module — nothing to fuzz" + fi + exit "$fail" + # Coverage: per-function summary, an HTML report under build/coverage/, and an # optional minimum gate — export TEST_GO_COVER_MIN := '80' (integer percent) # from the root Justfile to enforce a floor; unset or 0 reports without gating. diff --git a/Justfile b/Justfile index 760897d..094114a 100644 --- a/Justfile +++ b/Justfile @@ -22,67 +22,3 @@ mkfs-info: else echo "mkfs.erofs: NOT FOUND — every image-backed test will skip itself" fi - -# Fuzz smoke: every Fuzz* target, each for a short budget — a regression net, -# not a campaign. TEMPORARY, and verbatim what limen's `do::test::go::fuzz` -# will be once a release ships it; the day `just do test go fuzz` exists, this -# recipe is deleted and CI's fuzz job calls that instead. Not part of `test`: -# it needs mkfs.erofs on PATH to be worth anything (see mkfs-info) and runs -# once, on linux, in CI. -# -# Driven by `go test -fuzz` itself, never a prebuilt test binary: only the go -# tool's fuzz build compiles in the coverage counters, and without them the -# engine mutates blind. The verdict comes from the crasher, not the exit code: -# a real finding always writes the failing input under testdata/fuzz// -# (commit it — it is a regression test from then on), while the coordinator -# can also exit 1 with "context deadline exceeded" when a worker is -# mid-iteration as fuzztime expires — a shutdown hiccup, not a finding, and -# the Wide targets hit it most. So exit 1 WITH a new testdata file fails; exit -# 1 without one is retried once, and a second in a row is treated as real. The -# generated corpus lives under $(go env GOCACHE)/fuzz — CI caches it between -# runs so fuzzing is cumulative. -[doc('Run every Fuzz* target for TEST_GO_FUZZ_TIME (default 10s) each')] -fuzz: - #!/usr/bin/env bash - set -euo pipefail - fuzztime="${TEST_GO_FUZZ_TIME:-10s}" - # Crasher count for a target: the directory does not exist until the first - # finding, and a missing directory is zero, not an error (pipefail). - crashers() { - if [ -d "$1" ]; then find "$1" -type f | wc -l; else echo 0; fi - } - found=0 - fail=0 - for pkg in $(go list ./...); do - targets=$(go test -count=1 -run '^$' -list '^Fuzz' "$pkg" | grep '^Fuzz' || true) - [ -n "$targets" ] || continue - dir=$(go list -f '{{{{.Dir}}' "$pkg") - for target in $targets; do - found=1 - echo "fuzzing $pkg $target for $fuzztime" - before=$(crashers "$dir/testdata/fuzz/$target") - ok=0 - for attempt in 1 2; do - if go test -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" \ - -run "^${target}\$" -fuzz "^${target}\$" -fuzztime "$fuzztime" "$pkg"; then - ok=1 - break - fi - after=$(crashers "$dir/testdata/fuzz/$target") - if [ "$after" -gt "$before" ]; then - echo "$target: new crasher written under $dir/testdata/fuzz/$target — commit it as a regression test" >&2 - break - fi - echo "$target: exit without a crasher (attempt $attempt) — coordinator shutdown hiccup, retrying" >&2 - done - if [ "$ok" -eq 1 ]; then - echo "PASS: $target" - else - fail=1 - fi - done - done - if [ "$found" -eq 0 ]; then - echo "no Fuzz* targets in this module — nothing to fuzz" - fi - exit "$fail" diff --git a/aqua-checksums.json b/aqua-checksums.json index 04d5fc8..b975ef1 100644 --- a/aqua-checksums.json +++ b/aqua-checksums.json @@ -46,28 +46,28 @@ "algorithm": "sha256" }, { - "id": "github_release/github.com/farcloser/limen/v0.0.12/limen_0.0.12_darwin_arm64.tar.gz", - "checksum": "EDCA1AF957C0F14178F6B29D473EC899F479BB3AF2A240020FAD5965F044AC99", + "id": "github_release/github.com/farcloser/limen/v0.0.13/limen_0.0.13_darwin_arm64.tar.gz", + "checksum": "8739EA67AE2404A33E56EC9E89DE228C4DB73BFE37E2E814D7592289576FDBF8", "algorithm": "sha256" }, { - "id": "github_release/github.com/farcloser/limen/v0.0.12/limen_0.0.12_linux_amd64.tar.gz", - "checksum": "551BC67781A4FD18F1941A9BE68320DB730B982DA62BBF41988F88E721B5EE65", + "id": "github_release/github.com/farcloser/limen/v0.0.13/limen_0.0.13_linux_amd64.tar.gz", + "checksum": "C5A3990CF1307CB14DA8986FF45AD634A569DD35B3E536F7694C7BD9E9BF4557", "algorithm": "sha256" }, { - "id": "github_release/github.com/farcloser/limen/v0.0.12/limen_0.0.12_linux_arm64.tar.gz", - "checksum": "13C508CC4ECE232D033AA4E336DA96EFAE35E6C247F7BFE52F8E38824EC502B1", + "id": "github_release/github.com/farcloser/limen/v0.0.13/limen_0.0.13_linux_arm64.tar.gz", + "checksum": "6D19477293359E262C2E6E882974355B1F339A5249EBC1D42620783BBD23B3B0", "algorithm": "sha256" }, { - "id": "github_release/github.com/farcloser/limen/v0.0.12/limen_0.0.12_windows_amd64.tar.gz", - "checksum": "3CFE919FF7CBD49C0DACA0397B5E76330861C96CE4575E997B9D3C4833AE05E0", + "id": "github_release/github.com/farcloser/limen/v0.0.13/limen_0.0.13_windows_amd64.tar.gz", + "checksum": "942522F4AB937B9DDDCFE2E369E2AF13AF64C1041934CA20D098AA9C8321FC20", "algorithm": "sha256" }, { - "id": "github_release/github.com/farcloser/limen/v0.0.12/limen_0.0.12_windows_arm64.tar.gz", - "checksum": "8DA23959424BABC09D0AED2798F1E3393A7DC2F594AE4BC8A098BADDAB03F210", + "id": "github_release/github.com/farcloser/limen/v0.0.13/limen_0.0.13_windows_arm64.tar.gz", + "checksum": "C331B8BA5495406BC1A5C6DB62549B4AB562FDD808DFB71B18D3FD6ADE3FCE42", "algorithm": "sha256" }, { diff --git a/aqua.yaml b/aqua.yaml index 9ea8106..95d5914 100644 --- a/aqua.yaml +++ b/aqua.yaml @@ -32,7 +32,7 @@ packages: - name: github.com/farcloser/godolint/cmd/godolint@v0.1.0 registry: local # --- farcloser tools (local registry; standard once registered upstream) --- - - name: farcloser/limen@v0.0.12 # renovate: depName=farcloser/limen + - name: farcloser/limen@v0.0.13 # renovate: depName=farcloser/limen registry: local # --- toolchain + binary-release tools (standard registry, aqua-verified) --- - name: golang/go@go1.26.5 From 97079aa5bbbb79e639d6b55e0d3c0bd2182403ff Mon Sep 17 00:00:00 2001 From: apostasie Date: Sat, 15 Aug 2026 20:17:13 -0700 Subject: [PATCH 3/5] Hold the CopyFrom fast path to the reader's xattr and dirent-name rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two hostile-image checks lived only in the reader. Xattrs: the reader rejects an undefined name index (7 upward, which used to map to the empty prefix and let an image spell "security.capability" in full under it, colliding with the properly prefixed entry) and a key listed twice (which was taken last-wins, so the parser's iteration order picked which copy a policy decision saw). parseXattrsFromBuf, the copyFromImage fast path, still did both — so an image the reader refused to Stat would still be copied by CopyFrom(MetadataOnly), spoofed capability included. Route its names through xattrIndex.prefix(), propagate long-prefix lookup failures instead of swallowing them, and reject duplicates; the parser now returns an error and copyFromImage fails the copy on it. Dirent names: checkDirentName rejects separators and NULs but let a zero-length name through. No writer emits one, and as a path element it names the directory itself, so a lookup for it silently succeeds; ReadDir handed it out as an entry, and the fast path — which strips trailing NULs first, so an all-NUL entry became empty — skipped it silently. Reject it in checkDirentName, which covers all four parse sites, and move the fast path's check ahead of its "."/".." filter so the empty case is an error there too. The tampered image in the new test shows why this is corruption and not cosmetics: alongside the empty entry the neighbouring name reads back as "..a". Tests: TestCopyFromImageXattrParity (undefined index, duplicate key — both fail on the previous parser) and TestEmptyDirentNameIsRejected (reader and CopyFrom; fails on the previous reader with the entry list ["" "..a" "b"]). Signed-off-by: apostasie Co-Authored-By: Claude Fable 5 --- erofs.go | 5 ++ mkfs_image.go | 85 +++++++++++++++++------ mkfs_untrusted_test.go | 153 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 22 deletions(-) diff --git a/erofs.go b/erofs.go index 8ee130a..cad307d 100644 --- a/erofs.go +++ b/erofs.go @@ -990,7 +990,12 @@ func validPath(name string) bool { // // "." and "..", the directory's own self and parent entries, are legitimate // on disk; callers filter them by name rather than treating them as errors. +// An empty name is not: no writer emits one, and as a path element it names +// the directory itself, so a lookup for it would silently succeed. func checkDirentName(name []byte) error { + if len(name) == 0 { + return fmt.Errorf("empty dirent name: %w", ErrInvalid) + } if bytes.ContainsRune(name, '/') { return fmt.Errorf("dirent name %q contains a path separator: %w", name, ErrInvalid) } diff --git a/mkfs_image.go b/mkfs_image.go index 7f6c282..2f880cd 100644 --- a/mkfs_image.go +++ b/mkfs_image.go @@ -279,7 +279,11 @@ func (fsys *Writer) copyFromImage(img *image) error { xattrAddr := inodeAddr + int64(icSize) xb := at(xattrAddr) if len(xb) >= xattrSize { - xattrs = parseXattrsFromBuf(xb[:xattrSize], at, sharedXattrOff, img.getLongPrefix) + var err error + xattrs, err = parseXattrsFromBuf(xb[:xattrSize], at, sharedXattrOff, img.getLongPrefix) + if err != nil { + return fmt.Errorf("xattrs for nid %d: %w", cur.nid, err) + } } } @@ -493,18 +497,21 @@ func (fsys *Writer) parseDirBlock(data []byte, dirSize, blockSize int, parentPat nameBytes = nameBytes[:len(nameBytes)-1] } name := string(nameBytes) - if name == "." || name == ".." || name == "" { - continue - } // A name is one path element. Without this a nested dirent named // "y/../../../.wh..wh..opq" builds a childPath that path.Dir // collapses to "/", so the merge-mode whiteout handler wipes every // entry from every prior layer; "x/../../.wh.secret" deletes an // arbitrary path, and a plain "etc/passwd" overwrites a file in a - // directory the source never named. + // directory the source never named. Checked before the "."/".." + // filter so an empty name — which the NUL-stripping above can + // produce from an all-NUL entry — is an error here as it is in + // the reader, not something skipped over. if err := checkDirentName(nameBytes); err != nil { return fmt.Errorf("in %s: %w", parentPath, err) } + if name == "." || name == ".." { + continue + } childPath := parentPath + "/" + name if parentPath == "/" { @@ -646,9 +653,17 @@ func (fsys *Writer) parseChunks(data []byte, chunkFmt uint16, fileSize uint64, b // parseXattrsFromBuf parses xattr entries from an in-memory buffer. // at provides on-demand access to the shared xattr block at sharedOff. // longPrefix resolves long xattr prefix indexes (NameIndex with high bit set). -func parseXattrsFromBuf(buf []byte, at func(int64) []byte, sharedOff int64, longPrefix func(uint8) (string, error)) map[string]string { +// +// The same rules as the reader's loadXattrs apply, because this is the +// other door an image's attributes come in through: an unknown name index +// is an error rather than the empty prefix (which let a hostile image spell +// "security.capability" in full under an undefined index and collide with +// the properly prefixed entry), and a name listed twice is an error rather +// than last-wins (which let the parser's iteration order pick which copy a +// policy decision saw). +func parseXattrsFromBuf(buf []byte, at func(int64) []byte, sharedOff int64, longPrefix func(uint8) (string, error)) (map[string]string, error) { if len(buf) < disk.SizeXattrBodyHeader { - return nil + return nil, nil } var xh disk.XattrHeader @@ -656,6 +671,14 @@ func parseXattrsFromBuf(buf []byte, at func(int64) []byte, sharedOff int64, long pos := disk.SizeXattrBodyHeader xattrs := make(map[string]string) + set := func(name, value string) error { + if _, dup := xattrs[name]; dup { + return fmt.Errorf("duplicate xattr %q: %w", name, ErrInvalid) + } + xattrs[name] = value + + return nil + } // Resolve shared xattr references. for i := 0; i < int(xh.SharedCount) && pos+4 <= len(buf); i++ { @@ -676,9 +699,14 @@ func parseXattrsFromBuf(buf []byte, at func(int64) []byte, sharedOff int64, long continue } sb := sharedBlock[disk.SizeXattrEntry:] - name := xattrName(xe, sb[:xe.NameLen], longPrefix) + name, err := xattrName(xe, sb[:xe.NameLen], longPrefix) + if err != nil { + return nil, fmt.Errorf("shared xattr %d: %w", idx, err) + } value := string(sb[xe.NameLen : int(xe.NameLen)+int(xe.ValueLen)]) - xattrs[name] = value + if err := set(name, value); err != nil { + return nil, err + } } // Parse inline xattr entries. @@ -692,12 +720,17 @@ func parseXattrsFromBuf(buf []byte, at func(int64) []byte, sharedOff int64, long break } - name := xattrName(xe, buf[pos:pos+int(xe.NameLen)], longPrefix) + name, err := xattrName(xe, buf[pos:pos+int(xe.NameLen)], longPrefix) + if err != nil { + return nil, fmt.Errorf("inline xattr: %w", err) + } pos += int(xe.NameLen) value := string(buf[pos : pos+int(xe.ValueLen)]) pos += int(xe.ValueLen) - xattrs[name] = value + if err := set(name, value); err != nil { + return nil, err + } // Round up to 4-byte boundary. if rem := pos % 4; rem != 0 { @@ -705,24 +738,32 @@ func parseXattrsFromBuf(buf []byte, at func(int64) []byte, sharedOff int64, long } } if len(xattrs) == 0 { - return nil + return nil, nil } - return xattrs + return xattrs, nil } // xattrName builds the full xattr name from an entry and its raw name bytes. -// longPrefix resolves long prefix indexes when the high bit of NameIndex is set. -func xattrName(xe disk.XattrEntry, rawName []byte, longPrefix func(uint8) (string, error)) string { +// longPrefix resolves long prefix indexes when the high bit of NameIndex is +// set. An index the format does not define is an error, as in the reader. +func xattrName(xe disk.XattrEntry, rawName []byte, longPrefix func(uint8) (string, error)) (string, error) { var prefix string if xe.NameIndex&0x80 != 0 { // Long prefix: high bit set, low 7 bits index the prefix table. - if longPrefix != nil { - if p, err := longPrefix(xe.NameIndex & 0x7F); err == nil { - prefix = p - } + if longPrefix == nil { + return "", fmt.Errorf("long xattr prefix %d without a prefix table: %w", xe.NameIndex&0x7F, ErrInvalid) + } + p, err := longPrefix(xe.NameIndex & 0x7F) + if err != nil { + return "", err + } + prefix = p + } else { + p, err := xattrIndex(xe.NameIndex).prefix() + if err != nil { + return "", err } - } else if xe.NameIndex != 0 { - prefix = xattrIndex(xe.NameIndex).String() + prefix = p } - return prefix + string(rawName) + return prefix + string(rawName), nil } diff --git a/mkfs_untrusted_test.go b/mkfs_untrusted_test.go index bb59e6b..d7e57d8 100644 --- a/mkfs_untrusted_test.go +++ b/mkfs_untrusted_test.go @@ -1212,3 +1212,156 @@ func TestXattrPrefixAndDuplicates(t *testing.T) { t.Error("a rejected duplicate overwrote the original value") } } + +// TestCopyFromImageXattrParity covers the copyFromImage fast path, the other +// door an image's attributes come in through. The reader rejects an undefined +// xattr name index and a duplicated key (TestXattrPrefixAndDuplicates); the +// fast-path parser used to map the index to the empty prefix and take the +// duplicate last-wins, so an image that the reader refused would still be +// copied — spoofed security.capability included — by CopyFrom(MetadataOnly). +func TestCopyFromImageXattrParity(t *testing.T) { + build := func(t *testing.T, xattrs map[string]string) []byte { + t.Helper() + out := &seekBuf{} + w := Create(out, WithBuildTime(1000, 0)) + f, err := w.Create("/f") + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + for k, v := range xattrs { + if err := w.Setxattr("/f", k, v); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return out.buf + } + copyMeta := func(t *testing.T, buf []byte) error { + t.Helper() + img, err := Open(bytes.NewReader(buf)) + if err != nil { + t.Fatalf("image failed to open: %v", err) + } + return Create(&seekBuf{}).CopyFrom(img, MetadataOnly()) + } + + t.Run("undefinedIndex", func(t *testing.T) { + buf := build(t, map[string]string{"security.capability": "real"}) + if err := copyMeta(t, buf); err != nil { + t.Fatalf("untampered image failed to copy: %v", err) + } + // The 4-byte entry precedes the stored name; its second byte is the + // name index. 6 is "security."; 7 upward is undefined. + i := bytes.Index(buf, []byte("capability")) + if i < 0 || buf[i-3] != 6 { + t.Fatalf("could not locate the stored xattr entry (index byte %d)", buf[i-3]) + } + buf[i-3] = 7 + err := copyMeta(t, buf) + if err == nil { + t.Fatal("CopyFrom accepted an undefined xattr name index") + } + if !errors.Is(err, ErrInvalid) { + t.Errorf("err = %v, want it to wrap ErrInvalid", err) + } + }) + + t.Run("duplicateKey", func(t *testing.T) { + // Two names under the same prefix, then rewrite the second's stored + // suffix so both spell the same full key. + buf := build(t, map[string]string{ + "user.aaaa": "first", + "user.bbbb": "second", + }) + if err := copyMeta(t, buf); err != nil { + t.Fatalf("untampered image failed to copy: %v", err) + } + i := bytes.Index(buf, []byte("bbbb")) + if i < 0 { + t.Fatal("could not locate the second stored xattr name") + } + copy(buf[i:], "aaaa") + err := copyMeta(t, buf) + if err == nil { + t.Fatal("CopyFrom accepted a duplicated xattr key") + } + if !errors.Is(err, ErrInvalid) { + t.Errorf("err = %v, want it to wrap ErrInvalid", err) + } + }) +} + +// TestEmptyDirentNameIsRejected covers a dirent whose name is zero bytes long. +// No writer emits one, and as a path element it names the directory itself, +// so a lookup for it silently succeeds; the reader used to hand it out as an +// entry, and the copyFromImage fast path silently skipped over it — neither +// treated the image as the corrupt thing it is. +func TestEmptyDirentNameIsRejected(t *testing.T) { + out := &seekBuf{} + w := Create(out, WithBuildTime(1000, 0)) + for _, n := range []string{"/a", "/b"} { + f, err := w.Create(n) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + buf := out.buf + + img0, err := Open(bytes.NewReader(buf)) + if err != nil { + t.Fatal(err) + } + aNid, _, _, err := img0.(*image).resolve("a", "a", false) + if err != nil { + t.Fatal(err) + } + + // Find a's dirent by its nid, then give it b's NameOff — the next dirent + // is 12 bytes on — so a's name spans zero bytes. Nothing else changes. + var want [8]byte + binary.LittleEndian.PutUint64(want[:], aNid) + off := -1 + for o := 0; o+disk.SizeDirent*2 <= len(buf); o += 4 { + if bytes.Equal(buf[o:o+8], want[:]) { + off = o + break + } + } + if off < 0 { + t.Fatal("could not locate the dirent for /a") + } + nextNameOff := binary.LittleEndian.Uint16(buf[off+disk.SizeDirent+8:]) + binary.LittleEndian.PutUint16(buf[off+8:], nextNameOff) + + img, err := Open(bytes.NewReader(buf)) + if err != nil { + t.Fatalf("tampered image failed to open: %v", err) + } + if ents, err := fs.ReadDir(img, "."); err == nil { + names := make([]string, 0, len(ents)) + for _, e := range ents { + names = append(names, e.Name()) + } + t.Errorf("ReadDir returned %q, want an error", names) + } else if !errors.Is(err, ErrInvalid) { + t.Errorf("ReadDir err = %v, want it to wrap ErrInvalid", err) + } + + err = Create(&seekBuf{}).CopyFrom(img, MetadataOnly()) + if err == nil { + t.Error("CopyFrom accepted an empty dirent name") + } else if !errors.Is(err, ErrInvalid) { + t.Errorf("CopyFrom err = %v, want it to wrap ErrInvalid", err) + } +} From d779268464ad25aa45bf1f7ecb63960f16bd1c25 Mon Sep 17 00:00:00 2001 From: apostasie Date: Sat, 15 Aug 2026 20:30:35 -0700 Subject: [PATCH 4/5] renovate Signed-off-by: apostasie --- renovate.json5 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/renovate.json5 b/renovate.json5 index 07f85cd..873a9d9 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -13,6 +13,11 @@ commitBody: "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>", // The update-aqua-checksum workflow pushes a fix-up commit onto Renovate's // branches; without this, Renovate treats the branch as human-modified and - // stops rebasing it. - gitIgnoredAuthors: ["41898282+github-actions[bot]@users.noreply.github.com"], + // stops rebasing it. Two identities: the org's limen-ci-forkcloser App (the + // credential the workflow prefers, so that CI re-runs on the fix-up), and + // the default GITHUB_TOKEN it falls back to. + gitIgnoredAuthors: [ + "317468017+limen-ci-forkcloser[bot]@users.noreply.github.com", + "41898282+github-actions[bot]@users.noreply.github.com", + ], } From 48eed0c901afd87a092e00718cc3c9d1bbbc18ac Mon Sep 17 00:00:00 2001 From: apostasie Date: Sat, 15 Aug 2026 21:08:52 -0700 Subject: [PATCH 5/5] Bustage fix Signed-off-by: apostasie --- .github/scripts/build-erofs-utils.sh | 58 +++++++++++++++++++--------- .github/workflows/ci.yaml | 44 ++++++++++++--------- Justfile | 20 +++++++++- 3 files changed, 83 insertions(+), 39 deletions(-) diff --git a/.github/scripts/build-erofs-utils.sh b/.github/scripts/build-erofs-utils.sh index 327eea1..c67dcb2 100755 --- a/.github/scripts/build-erofs-utils.sh +++ b/.github/scripts/build-erofs-utils.sh @@ -16,11 +16,23 @@ # unpinned, knowingly: the build-dependency packages from the runner's own # apt/brew repositories. # +# Where it lands — and why that matters: `just` runs every recipe under limen's +# HERMETIC PATH (aqua's bin plus the base system dirs; see .limen/just/main.just), +# so a `make install` into /usr/local is invisible to `just test` and every +# image-backed test skips itself while the log says the build succeeded. The +# binary therefore installs into the project's own tool dir, build/erofs-utils/, +# whose bin/ the root Justfile prepends to PATH — the one declared exception to +# the hermetic list, and the same location on every OS (the windows cross-build +# is copied there by CI). No sudo, no system prefix. +# # Usage: -# build-erofs-utils.sh native build and `make install` for this host -# (linux or macOS); mkfs.erofs lands on PATH +# build-erofs-utils.sh native build and install for this host (linux or +# macOS) into build/erofs-utils/bin/mkfs.erofs # build-erofs-utils.sh windows cross-compile a static mkfs.erofs.exe with -# MinGW-w64 (linux host); prints its path +# MinGW-w64 (linux host) into +# build/erofs-utils/bin/mkfs.erofs.exe +# Set SKIP_DEPS=1 to skip the apt/brew build-dependency step (a developer +# machine that already has autotools and lz4). set -euo pipefail EROFS_UTILS_VERSION="1.9.3" @@ -33,6 +45,7 @@ repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" patches="$repo/.github/workflows/patches/erofs-utils" headers="$repo/.github/workflows/mingw-compat-headers" work="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/erofs-utils-build" +prefix="$repo/build/erofs-utils" die() { echo "build-erofs-utils: $*" >&2 @@ -73,16 +86,18 @@ nproc_portable() { } build_native() { - case "$(uname -s)" in - Linux) - sudo apt-get update -qq - sudo apt-get install -y -qq autoconf automake libtool pkg-config libz-dev liblz4-dev uuid-dev - ;; - Darwin) - brew install autoconf automake libtool pkg-config lz4 - ;; - *) die "native build supports linux and macOS only (got $(uname -s))" ;; - esac + if [ -z "${SKIP_DEPS:-}" ]; then + case "$(uname -s)" in + Linux) + sudo apt-get update -qq + sudo apt-get install -y -qq autoconf automake libtool pkg-config libz-dev liblz4-dev uuid-dev + ;; + Darwin) + brew install autoconf automake libtool pkg-config lz4 + ;; + *) die "native build supports linux and macOS only (got $(uname -s))" ;; + esac + fi local src src=$(unpack_erofs_utils) ( @@ -92,17 +107,20 @@ build_native() { # to 16K only when the build CPU is aarch64), so the same source # yields a different mkfs per runner. Pin it: the 16384 leg of # TestReadReferenceImage skips itself otherwise. - MAX_BLOCK_SIZE=16384 ./configure --enable-lz4 + MAX_BLOCK_SIZE=16384 ./configure --enable-lz4 --prefix="$prefix" make -j"$(nproc_portable)" - sudo make install + make install ) - mkfs.erofs -V + "$prefix/bin/mkfs.erofs" -V + echo "installed $prefix/bin/mkfs.erofs" } build_windows() { [ "$(uname -s)" = "Linux" ] || die "the windows cross-build needs a linux host" - sudo apt-get update -qq - sudo apt-get install -y -qq autoconf automake libtool pkg-config mingw-w64 + if [ -z "${SKIP_DEPS:-}" ]; then + sudo apt-get update -qq + sudo apt-get install -y -qq autoconf automake libtool pkg-config mingw-w64 + fi mkdir -p "$work" # lz4, static, into the mingw sysroot — the only library the cross build @@ -158,7 +176,9 @@ build_windows() { make -j"$(nproc_portable)" -C mkfs CPPFLAGS="-D_GNU_SOURCE -include posix_compat.h" LIBS="-llz4" "${MINGW_HOST}-strip" mkfs/mkfs.erofs.exe ) - echo "$src/mkfs/mkfs.erofs.exe" + mkdir -p "$prefix/bin" + cp "$src/mkfs/mkfs.erofs.exe" "$prefix/bin/mkfs.erofs.exe" + echo "installed $prefix/bin/mkfs.erofs.exe" } case "${1:-}" in diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 77d0bb5..0dede4f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,15 +86,22 @@ jobs: # INTERIM: the image-backed tests shell out to mkfs.erofs, which the # pinned toolchain cannot carry (no release binary to pin), so it is # built from source here — checksum-verified, patched, one script for - # every job. Not on windows: no native build; the windows-2025 leg with - # an image is `verify-windows-image` below, fed by a cross-compile. The - # windows legs of this matrix run `just test` as-is, and `mkfs-info` - # says so in the log: image tests skip themselves there. + # every job — into build/erofs-utils/bin, the one directory the root + # Justfile adds to limen's hermetic PATH (a /usr/local install would be + # invisible to `just test`). Not on windows: no native build there; the + # windows-2025 leg WITH an image is `verify-windows-image` below, fed by + # a cross-compile. The windows legs of this matrix run `just test` + # as-is and their image tests skip themselves — `mkfs-info` says so. - name: Build erofs-utils (interim) if: runner.os != 'Windows' run: .github/scripts/build-erofs-utils.sh native - name: Test + env: + # The legs that just built mkfs.erofs must FAIL if `just test` + # cannot see it — a silent fall-back to "every image test skipped" + # is a coverage regression, not a pass. Windows legs: not required. + EROFS_REQUIRE_MKFS: ${{ runner.os != 'Windows' && '1' || '' }} run: just test # INTERIM: cross-compile a static mkfs.erofs.exe so the windows leg below @@ -112,15 +119,12 @@ jobs: persist-credentials: false - name: Cross-compile mkfs.erofs for windows - run: | - exe=$(.github/scripts/build-erofs-utils.sh windows | tail -n 1) - mkdir -p build/mkfs-windows - cp "$exe" build/mkfs-windows/mkfs.erofs.exe + run: .github/scripts/build-erofs-utils.sh windows - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: mkfs-erofs-windows - path: build/mkfs-windows/mkfs.erofs.exe + path: build/erofs-utils/bin/mkfs.erofs.exe if-no-files-found: error # INTERIM: the windows-2025 leg WITH an image. Same recipe as every other @@ -140,14 +144,12 @@ jobs: persist-credentials: false fetch-depth: 0 + # Into the same project-local tool dir the native builds use: the root + # Justfile's PATH addition finds it, no GITHUB_PATH plumbing needed. - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: mkfs-erofs-windows - path: build/mkfs-windows - - - name: Put mkfs.erofs on PATH - run: | - echo "$GITHUB_WORKSPACE/build/mkfs-windows" >> "$GITHUB_PATH" + path: build/erofs-utils/bin - name: Install aqua (pinned, checksum-verified) uses: ./.github/actions/setup-aqua @@ -157,11 +159,11 @@ jobs: - name: Test env: + # This leg exists for the image tests: fail if mkfs.erofs is not seen. + EROFS_REQUIRE_MKFS: "1" # mkfs.erofs uses TMPDIR for its temporaries; windows does not set it. TMPDIR: ${{ runner.temp }} - run: | - mkfs.erofs -V - just test + run: just test # INTERIM in its plumbing, permanent in intent: a short fuzz of every Fuzz* # target, once, on linux, with mkfs.erofs available (several targets round- @@ -200,8 +202,14 @@ jobs: restore-keys: | fuzz-corpus-${{ runner.os }}- + # Several targets round-trip through mkfs.erofs; without it they skip + # and the fuzz run is quietly worth less. Same guard as the test legs. - name: Fuzz - run: just do test go fuzz + env: + EROFS_REQUIRE_MKFS: "1" + run: | + just mkfs-info + just do test go fuzz # Surface crashers as artifacts: the log names the target, but the # input itself is what reproduces the bug locally. diff --git a/Justfile b/Justfile index 094114a..4a9bcf3 100644 --- a/Justfile +++ b/Justfile @@ -3,6 +3,14 @@ # The import must be kept: it mounts every shared limen task under `just do ...`. import '.limen/just/main.just' +# One declared exception to limen's hermetic PATH: build/erofs-utils/bin, where +# .github/scripts/build-erofs-utils.sh installs mkfs.erofs (a C tool with no +# release binary aqua could pin, so the tests that read real images cannot get +# it any other way). Same expression as main.just's, with that one directory in +# front — a project's root Justfile may re-export a shared variable, and the +# importing file wins. Nothing else is added: /usr/local and homebrew stay out. +export PATH := if os() == 'windows' { justfile_directory() / 'build' / 'erofs-utils' / 'bin' + ';' + aqua_bin + ';' + env_var('PATH') } else { justfile_directory() / 'build' / 'erofs-utils' / 'bin' + ':' + aqua_bin + ':/usr/bin:/bin:/usr/sbin:/sbin' } + # The FIRST recipe defined here becomes `just`'s default. lint: do::lint::go::default do::lint::default fix: do::fix::go::default do::fix::default @@ -12,13 +20,21 @@ bench: do::test::go::bench # The tests that read a real image shell out to mkfs.erofs and skip themselves # when it is absent, so a run without erofs-utils silently covers far less than # one with it. Say which it was: a green run that skipped every image test is -# not evidence about the image reader. Diagnostic only — never fails. -[doc('Report whether mkfs.erofs is available to the image-backed tests')] +# not evidence about the image reader. Diagnostic by default; with +# EROFS_REQUIRE_MKFS=1 (set by every CI leg that just built mkfs.erofs) a +# missing binary FAILS the run — a coverage regression must be red, not a line +# in a log. Locally: `.github/scripts/build-erofs-utils.sh native` (SKIP_DEPS=1 +# if you already have autotools and lz4) puts it where this looks. +[doc('Report whether mkfs.erofs is available to the image-backed tests (EROFS_REQUIRE_MKFS=1 to fail if not)')] mkfs-info: #!/usr/bin/env bash set -euo pipefail if command -v mkfs.erofs > /dev/null 2>&1; then echo "mkfs.erofs: $(mkfs.erofs -V 2>&1 | head -n 1) — image-backed tests will run" + echo " at $(command -v mkfs.erofs)" + elif [ "${EROFS_REQUIRE_MKFS:-}" = "1" ]; then + echo "mkfs.erofs: NOT FOUND, and EROFS_REQUIRE_MKFS=1 — this leg was supposed to have it (build/erofs-utils/bin)" >&2 + exit 1 else echo "mkfs.erofs: NOT FOUND — every image-backed test will skip itself" fi