From c069bb8341f0dca6995dc5b3d154c0c64270301c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 6 Oct 2026 18:20:06 +0000 Subject: [PATCH] ci: add early disk cleanup to Test and Fuzz jobs Add 'Early disk cleanup (free cargo caches and sibling run dirs)' step at the start of both Test and Fuzz jobs, before cache restoration and heavy cargo builds. This addresses ENOSPC failures on self-hosted runners (e.g. run 37505615506 on zeus-workstation) where disk pressure on the /mnt/data volume caused failures unpacking crates into CARGO_HOME. The cleanup targets: - Stale cargo registry/src and git/checkouts under RUNNER_TEMP/cargo - Sibling run directories under BLVM_RUNNER_DATA_ROOT - Bloated runner-cache/target directories (>10GB) - Stale fuzz/target artifacts (Fuzz job only) Also improves 'Dump disk usage on failure' in Test job to report: - /mnt/data volume (where ENOSPC occurred) - RUNNER_TEMP, CARGO_HOME, BLVM_RUNNER_DATA_ROOT paths and sizes - Better diagnostic output for future disk pressure issues The host root (/) was only 48% full in the failed run, so this is a volume-level issue not requiring host disk expansion. Co-authored-by: Secure Sovereign --- .github/workflows/ci.yml | 116 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54cb9a55..b38b3765 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -275,6 +275,54 @@ jobs: - name: Strip [patch.crates-io] for crates.io-only CI uses: BTCDecoded/rust-ci/strip-patch-crates-io@main + - name: Early disk cleanup (free cargo caches and sibling run dirs) + run: | + echo "=== Pre-test disk status ===" + df -h / || true + df -h "${RUNNER_TEMP:-/tmp}" 2>/dev/null || true + # Show /mnt/data if distinct volume (common self-hosted runner layout) + df -h /mnt/data 2>/dev/null || true + + # RUNNER_TEMP is the per-runner-process temp used as CARGO_HOME by this workflow. + # Clean stale cargo registry/git before the heavy test build to avoid ENOSPC. + CARGO_TEMP="${RUNNER_TEMP}/cargo" + if [ -d "$CARGO_TEMP" ]; then + echo "=== Cleaning stale cargo registry/git in $CARGO_TEMP ===" + # Remove registry src cache (large unpacked crates) — will be re-fetched as needed + rm -rf "$CARGO_TEMP/registry/src" 2>/dev/null || true + # Remove old git checkouts (can grow unbounded across runs) + rm -rf "$CARGO_TEMP/git/checkouts" 2>/dev/null || true + # Prune registry cache entries not accessed in last 7 days + find "$CARGO_TEMP/registry/cache" -type f -atime +7 -delete 2>/dev/null || true + fi + + # Clean sibling run dirs under BLVM_RUNNER_DATA_ROOT (same pattern as Setup and late cleanup). + BUILD_ROOT="${BLVM_RUNNER_DATA_ROOT}" + RUN_ID="${{ github.run_id }}" + if [ -d "$BUILD_ROOT" ]; then + echo "=== Purging sibling run dirs under $BUILD_ROOT (keeping $RUN_ID only) ===" + find "$BUILD_ROOT" -mindepth 1 -maxdepth 1 -type d \ + -not -name "$RUN_ID" \ + -exec rm -rf {} + 2>/dev/null || true + fi + + # Clean runner-cache target dirs if they exist and are large + for CACHE_TARGET in "${{ github.workspace }}/.build/runner-cache/target" "/tmp/runner-cache/target"; do + if [ -d "$CACHE_TARGET" ]; then + SIZE=$(du -sb "$CACHE_TARGET" 2>/dev/null | cut -f1 || echo 0) + # If over 10GB, prune stale entries older than 60 minutes + if [ "$SIZE" -gt $((10 * 1024 * 1024 * 1024)) ]; then + echo "=== $CACHE_TARGET >10GB ($SIZE bytes) — pruning entries >60min old ===" + find "$CACHE_TARGET" -mindepth 1 -maxdepth 1 -type d -mmin +60 -exec rm -rf {} + 2>/dev/null || true + fi + fi + done + + echo "=== Post-cleanup disk status ===" + df -h / || true + df -h "${RUNNER_TEMP:-/tmp}" 2>/dev/null || true + df -h /mnt/data 2>/dev/null || true + - name: Clean leftover path overrides run: | if [ -d "../blvm-protocol" ]; then rm -rf ../blvm-protocol; fi @@ -347,10 +395,25 @@ jobs: - name: Dump disk usage on failure if: failure() run: | - echo "=== df -h ===" + echo "=== df -h (all relevant volumes) ===" df -h / + df -h /mnt/data 2>/dev/null || true + df -h "${RUNNER_TEMP:-/tmp}" 2>/dev/null || true + df -h "${CARGO_HOME:-$HOME/.cargo}" 2>/dev/null || true + echo "" + echo "=== Key paths ===" + echo "RUNNER_TEMP=${RUNNER_TEMP:-}" + echo "CARGO_HOME=${CARGO_HOME:-}" + echo "BLVM_RUNNER_DATA_ROOT=${BLVM_RUNNER_DATA_ROOT:-}" + echo "BLVM_BUILD_TARGET=${BLVM_BUILD_TARGET:-}" + echo "" + echo "=== Size of key directories ===" + du -sh "${RUNNER_TEMP}/cargo" 2>/dev/null || echo "RUNNER_TEMP/cargo: (not found)" + du -sh "${CARGO_HOME:-$HOME/.cargo}" 2>/dev/null || echo "CARGO_HOME: (not found)" + du -sh "${BLVM_RUNNER_DATA_ROOT}" 2>/dev/null || echo "BLVM_RUNNER_DATA_ROOT: (not found)" + du -sh "${BLVM_BUILD_TARGET}" 2>/dev/null || echo "BLVM_BUILD_TARGET: (not found)" echo "" - echo "=== Top 10 largest directories ===" + echo "=== Top 10 largest directories under / ===" du -x --max-depth=4 / 2>/dev/null \ | sort -rh \ | grep -v -e '^[0-9]*\s*/proc' -e '^[0-9]*\s*/sys' -e '^[0-9]*\s*/dev' \ @@ -789,6 +852,52 @@ jobs: - name: Strip [patch.crates-io] for crates.io-only CI uses: BTCDecoded/rust-ci/strip-patch-crates-io@main + - name: Early disk cleanup (free cargo caches and sibling run dirs) + run: | + echo "=== Pre-fuzz disk status ===" + df -h / || true + df -h "${RUNNER_TEMP:-/tmp}" 2>/dev/null || true + df -h /mnt/data 2>/dev/null || true + + # RUNNER_TEMP is the per-runner-process temp used as CARGO_HOME by this workflow. + # Clean stale cargo registry/git before the heavy fuzz build to avoid ENOSPC. + CARGO_TEMP="${RUNNER_TEMP}/cargo" + if [ -d "$CARGO_TEMP" ]; then + echo "=== Cleaning stale cargo registry/git in $CARGO_TEMP ===" + rm -rf "$CARGO_TEMP/registry/src" 2>/dev/null || true + rm -rf "$CARGO_TEMP/git/checkouts" 2>/dev/null || true + find "$CARGO_TEMP/registry/cache" -type f -atime +7 -delete 2>/dev/null || true + fi + + # Clean sibling run dirs under BLVM_RUNNER_DATA_ROOT (same pattern as Setup/Test). + BUILD_ROOT="${BLVM_RUNNER_DATA_ROOT}" + RUN_ID="${{ github.run_id }}" + if [ -d "$BUILD_ROOT" ]; then + echo "=== Purging sibling run dirs under $BUILD_ROOT (keeping $RUN_ID only) ===" + find "$BUILD_ROOT" -mindepth 1 -maxdepth 1 -type d \ + -not -name "$RUN_ID" \ + -exec rm -rf {} + 2>/dev/null || true + fi + + # Clean runner-cache target dirs if bloated + for CACHE_TARGET in "${{ github.workspace }}/.build/runner-cache/target" "/tmp/runner-cache/target"; do + if [ -d "$CACHE_TARGET" ]; then + SIZE=$(du -sb "$CACHE_TARGET" 2>/dev/null | cut -f1 || echo 0) + if [ "$SIZE" -gt $((10 * 1024 * 1024 * 1024)) ]; then + echo "=== $CACHE_TARGET >10GB ($SIZE bytes) — pruning entries >60min old ===" + find "$CACHE_TARGET" -mindepth 1 -maxdepth 1 -type d -mmin +60 -exec rm -rf {} + 2>/dev/null || true + fi + fi + done + + # Clean stale fuzz build artifacts from prior runs + rm -rf fuzz/target 2>/dev/null || true + + echo "=== Post-cleanup disk status ===" + df -h / || true + df -h "${RUNNER_TEMP:-/tmp}" 2>/dev/null || true + df -h /mnt/data 2>/dev/null || true + - name: Cache uses: BTCDecoded/rust-ci/runner-cargo-cache@main with: @@ -849,9 +958,6 @@ jobs: chmod +x init_corpus.sh ./init_corpus.sh - - name: Clean stale fuzz build artifacts - run: rm -rf fuzz/target - # cargo-fuzz injects its own sancov + ASAN flags; clear the global RUSTFLAGS. - name: Build fuzz binaries working-directory: fuzz