From c382dc40776fb3028e2879f6c04c62e4a3bf507f Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 16 Nov 2026 03:43:13 +0100 Subject: [PATCH 1/2] fix(go): give vulnerability scans bounded swap headroom Prepare and verify an owned 4 GiB swap file before symbol analysis, retain the existing deadlines and finding gate, and retire only the owned file afterward. Add refusal controls and native Linux activation proof. Part of #671; historical runner-loss attribution remains open. --- .github/tests/test-go-analysis-swap.sh | 103 ++++++++++++++++++ .github/workflows/ci.yaml | 5 + .../validate-go-project-readonly.yaml | 54 ++++++++- .github/workflows/validate-go-project.yaml | 56 +++++++++- 4 files changed, 208 insertions(+), 10 deletions(-) create mode 100644 .github/tests/test-go-analysis-swap.sh diff --git a/.github/tests/test-go-analysis-swap.sh b/.github/tests/test-go-analysis-swap.sh new file mode 100644 index 00000000..b8a4c7cf --- /dev/null +++ b/.github/tests/test-go-analysis-swap.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Exercise the production setup/cleanup shells. Linux CI also activates real swap. +set -euo pipefail +root="$(cd "$(dirname "$0")/../.." && pwd)" +workflow="$root/.github/workflows/validate-go-project.yaml" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } + +setup_count="$(yq '[.jobs.govulncheck.steps[] | select(.id == "scan-swap")] | length' "$workflow")" +[[ "$setup_count" == 1 ]] || fail 'scanner must prepare swap exactly once before scanning' +yq -r '.jobs.govulncheck.steps[] | select(.id == "scan-swap") | .run' "$workflow" >"$work/setup.sh" +yq -r '.jobs.govulncheck.steps[] | select(.id == "scan-swap-cleanup") | .run' "$workflow" >"$work/cleanup.sh" +[[ -s "$work/setup.sh" && -s "$work/cleanup.sh" ]] || fail 'missing swap shells' +[[ "$(yq -r '.jobs.govulncheck.steps[] | select(.id == "scan-swap-cleanup") | .if' "$workflow")" == *'always()'* ]] || fail 'cleanup must run after failed scans too' +ids="$(yq -r '.jobs.govulncheck.steps[].id // ""' "$workflow")" +[[ "$ids" == *'scan-swap'*'scan-clock'*'scan-timeout'*'scan-swap-cleanup'* ]] || fail 'swap must span both bounded scan attempts' + +export RUNNER_TEMP="$work/runner" GITHUB_OUTPUT="$work/output" +mkdir -p "$RUNNER_TEMP" +: >"$GITHUB_OUTPUT" +if [[ "${1:-}" == --live ]]; then + [[ "$(uname -s)" == Linux ]] || fail 'live swap proof requires Linux' + bash "$work/setup.sh" + SWAP_FILE="$(sed -n 's/^path=//p' "$GITHUB_OUTPUT")" + export SWAP_FILE + [[ -n "$SWAP_FILE" ]] || fail 'setup did not publish its owned swap path' + # Preserve cleanup even if a native assertion fails. + trap 'bash "$work/cleanup.sh"; rm -rf "$work"' EXIT + [[ "$(stat -c %a "$SWAP_FILE")" == 600 ]] || fail 'swap must be private' + bytes="$(swapon --show=NAME,SIZE --bytes --json | jq -r --arg p "$SWAP_FILE" '.swapdevices[] | select(.name == $p) | .size')" + [[ "$bytes" =~ ^[0-9]+$ && "$bytes" -ge 4294963200 ]] || fail 'native kernel did not activate four GiB of swap' + bash "$work/cleanup.sh" + [[ ! -e "$SWAP_FILE" ]] || fail 'native cleanup left its file behind' + trap 'rm -rf "$work"' EXIT + echo 'PASS: native kernel swap activation, private mode and cleanup' + exit 0 +fi + +# Only privileged Linux host operations are substituted on the macOS agent host. +# Real mktemp/files and the actual production shells execute in every case. +mkdir -p "$work/bin" +export SWAP_TEST_STATE="$work/state" SWAP_TEST_LOG="$work/calls" +cat >"$work/bin/sudo" <<'SH' +#!/usr/bin/env bash +exec "$@" +SH +cat >"$work/bin/df" <<'SH' +#!/usr/bin/env bash +printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\nfixture 20000000 0 %s 0%% /\n' "${SWAP_TEST_FREE_KIB:-16000000}" +SH +cat >"$work/bin/fallocate" <<'SH' +#!/usr/bin/env bash +printf 'allocate %s\n' "$*" >>"$SWAP_TEST_LOG" +[[ "$1" == -l && "$2" == 4294967296 && "$3" == -- && -f "$4" ]] || exit 90 +[[ "${SWAP_TEST_FAIL:-}" != allocate ]] +SH +cat >"$work/bin/mkswap" <<'SH' +#!/usr/bin/env bash +[[ "${SWAP_TEST_FAIL:-}" != format ]] +SH +cat >"$work/bin/swapon" <<'SH' +#!/usr/bin/env bash +if [[ "$1" == --show=NAME ]]; then + [[ "${SWAP_TEST_FAIL:-}" != readback ]] && cat "$SWAP_TEST_STATE" +else + [[ "${SWAP_TEST_FAIL:-}" != activate ]] || exit 1 + printf '%s\n' "${!#}" >"$SWAP_TEST_STATE" +fi +SH +cat >"$work/bin/swapoff" <<'SH' +#!/usr/bin/env bash +rm -f "$SWAP_TEST_STATE" +SH +chmod +x "$work/bin/"* +export PATH="$work/bin:$PATH" + +bash "$work/setup.sh" +SWAP_FILE="$(sed -n 's/^path=//p' "$GITHUB_OUTPUT")" +export SWAP_FILE +[[ -f "$SWAP_FILE" && "$(cat "$SWAP_TEST_STATE")" == "$SWAP_FILE" ]] || fail 'setup did not activate its owned file' +if [[ "$(uname -s)" == Darwin ]]; then mode="$(stat -f %Lp "$SWAP_FILE")"; else mode="$(stat -c %a "$SWAP_FILE")"; fi +[[ "$mode" == 600 ]] || fail 'swap file permissions are not private' +bash "$work/cleanup.sh" +[[ ! -e "$SWAP_FILE" && ! -e "$SWAP_TEST_STATE" ]] || fail 'cleanup failed to retire owned swap' +echo 'PASS: prepare, confirm and retire owned swap' + +for failure in allocate format activate readback; do + export SWAP_TEST_FAIL="$failure" + : >"$GITHUB_OUTPUT" + if bash "$work/setup.sh" >"$work/failure.log" 2>&1; then fail "$failure must fail the preflight"; fi + [[ ! -s "$GITHUB_OUTPUT" ]] || fail "$failure published unverified swap" + [[ ! -e "$SWAP_TEST_STATE" ]] || fail "$failure left active swap behind" + echo "PASS: $failure cannot publish successful preparation" +done +unset SWAP_TEST_FAIL +export SWAP_TEST_FREE_KIB=8000000 +: >"$GITHUB_OUTPUT" +: >"$SWAP_TEST_LOG" +if bash "$work/setup.sh" >"$work/failure.log" 2>&1; then fail 'insufficient disk must fail before allocation'; fi +grep -q 'Insufficient disk' "$work/failure.log" || fail 'disk refusal failed for another reason' +[[ ! -s "$SWAP_TEST_LOG" && ! -s "$GITHUB_OUTPUT" ]] || fail 'disk refusal allocated or published swap' +echo 'PASS: insufficient disk fails before allocation' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 62ce6d39..3b0a724f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3653,6 +3653,11 @@ jobs: - name: Reject memory override wiring and parser regressions run: bash .github/tests/test-go-memory-limit-controls.sh + - name: Exercise scanner swap preparation and failure controls + run: bash .github/tests/test-go-analysis-swap.sh + - name: Prove scanner swap activation and cleanup on the native kernel + run: bash .github/tests/test-go-analysis-swap.sh --live + - name: 🧪 Assert govulncheck keeps a cold-cache-safe finite timeout + GOMEMLIMIT run: bash .github/tests/test-govulncheck-timeout.sh diff --git a/.github/workflows/validate-go-project-readonly.yaml b/.github/workflows/validate-go-project-readonly.yaml index 7d904397..a19daca4 100644 --- a/.github/workflows/validate-go-project-readonly.yaml +++ b/.github/workflows/validate-go-project-readonly.yaml @@ -712,10 +712,10 @@ jobs: # whose peak grows with the module's dependency graph. On large modules (e.g. ones # importing Kubernetes / Flux / Talos clients) it can exhaust the hosted runner's RAM, # which the host resolves by killing the runner mid-scan (exit 143 / "runner has - # received a shutdown signal") — an opaque, retry-resistant failure. GOMEMLIMIT caps - # the Go runtime's heap so the GC reclaims aggressively and stays under the host - # ceiling instead of OOM-killing; timeout-minutes bounds the worst case to a clear, - # fast failure rather than a hung runner. + # received a shutdown signal") — an opaque, retry-resistant failure. GOMEMLIMIT is + # a soft GC target: live data can still exceed it. The bounded swap preparation + # below provides additional headroom without weakening symbol analysis or the + # finding gate. It is a mitigation, not attribution of historical runner losses. # # The deadline lives on the scan STEP, not only the job (#1097). A job that hits its own # timeout-minutes ends `cancelled`, and a cancelled required check is terminal: Dependabot @@ -726,7 +726,8 @@ jobs: # and only a timeout, earns exactly one retry. The job ceiling stays finite but covers two # attempts plus checkout and setup-go's post-job cache save (measured 6m07s after that # 2026-09-06 scan): 2 × 24 + 10 = 58, - # rounded to 60. Below that the job cap would pre-empt the retry and end the run + # plus two one-minute swap lifecycle steps = 60. Below that the job cap would + # pre-empt the retry and end the run # `cancelled` again. Guarded by test-govulncheck-timeout and test-govulncheck-timeout-retry. timeout-minutes: 60 env: @@ -777,6 +778,36 @@ jobs: else echo "present=false" >> "$GITHUB_OUTPUT" fi + # Successful large-module scans already approach the runner's RAM ceiling. + # GOMEMLIMIT is soft; extra swap lets live data exceed that target without + # competing as sharply with the runner agent. This does not establish the + # cause of historical runner losses or change the scanner's verdict rules. + - name: Prepare bounded swap headroom for the scanner + id: scan-swap + timeout-minutes: 1 + shell: bash + run: | + set -euo pipefail + available_kib=$(df -Pk -- "$RUNNER_TEMP" | awk 'NR == 2 { print $4 }') + if [[ ! "$available_kib" =~ ^[0-9]+$ ]] || (( available_kib < 8388608 )); then + echo '::error::Insufficient disk for 4 GiB of scanner swap plus 4 GiB of free headroom.' + exit 1 + fi + swap_file=$(mktemp "$RUNNER_TEMP/govulncheck-swap.XXXXXX") + rollback() { + status=$? + # The file may not have reached swapon yet. Keep the original failure. + sudo swapoff -- "$swap_file" 2>/dev/null || true + rm -f -- "$swap_file" + exit "$status" + } + trap rollback ERR + chmod 600 "$swap_file" + sudo fallocate -l 4294967296 -- "$swap_file" + sudo mkswap -- "$swap_file" + sudo swapon -- "$swap_file" + swapon --show=NAME --noheadings --raw | grep -Fx -- "$swap_file" + printf 'path=%s\n' "$swap_file" >> "$GITHUB_OUTPUT" # Start of the first scan attempt, so the classifier below can tell a scan that ran out # of time from one that failed on its own (see the job's timeout-minutes comment). - name: ⏱️ Record vulnerability scan start @@ -848,6 +879,19 @@ jobs: work-dir: ${{ needs.changes.outputs.module-directory }} go-version-file: ${{ needs.changes.outputs.module-directory }}/go.mod allow-file: ${{ steps.govulncheck-allow.outputs.present == 'true' && format('{0}/.govulncheck-allow.txt', needs.changes.outputs.module-directory) || '' }} + - name: Retire only this job's scanner swap + id: scan-swap-cleanup + if: ${{ always() && steps.scan-swap.outputs.path != '' }} + timeout-minutes: 1 + shell: bash + env: + SWAP_FILE: ${{ steps.scan-swap.outputs.path }} + run: | + set -euo pipefail + if [[ -e "$SWAP_FILE" ]]; then + sudo swapoff -- "$SWAP_FILE" + rm -- "$SWAP_FILE" + fi lint: name: "\U0001F9F9 Lint - mega-linter" timeout-minutes: 15 diff --git a/.github/workflows/validate-go-project.yaml b/.github/workflows/validate-go-project.yaml index b57f92c9..c2a1306f 100644 --- a/.github/workflows/validate-go-project.yaml +++ b/.github/workflows/validate-go-project.yaml @@ -769,10 +769,10 @@ jobs: # whose peak grows with the module's dependency graph. On large modules (e.g. ones # importing Kubernetes / Flux / Talos clients) it can exhaust the hosted runner's RAM, # which the host resolves by killing the runner mid-scan (exit 143 / "runner has - # received a shutdown signal") — an opaque, retry-resistant failure. GOMEMLIMIT caps - # the Go runtime's heap so the GC reclaims aggressively and stays under the host - # ceiling instead of OOM-killing; timeout-minutes bounds the worst case to a clear, - # fast failure rather than a hung runner. + # received a shutdown signal") — an opaque, retry-resistant failure. GOMEMLIMIT is + # a soft GC target: live data can still exceed it. The bounded swap preparation + # below provides additional headroom without weakening symbol analysis or the + # finding gate. It is a mitigation, not attribution of historical runner losses. # # The deadline lives on the scan STEP, not only the job (#1097). A job that hits its own # timeout-minutes ends `cancelled`, and a cancelled required check is terminal: Dependabot @@ -783,7 +783,8 @@ jobs: # and only a timeout, earns exactly one retry. The job ceiling stays finite but covers two # attempts plus checkout and setup-go's post-job cache save (measured 6m07s after that # 2026-09-06 scan): 2 × 24 + 10 = 58, - # rounded to 60. Below that the job cap would pre-empt the retry and end the run + # plus two one-minute swap lifecycle steps = 60. Below that the job cap would + # pre-empt the retry and end the run # `cancelled` again. Guarded by test-govulncheck-timeout and test-govulncheck-timeout-retry. timeout-minutes: 60 env: @@ -839,6 +840,37 @@ jobs: echo "present=false" >> "$GITHUB_OUTPUT" fi + # Successful large-module scans already approach the runner's RAM ceiling. + # GOMEMLIMIT is soft; extra swap lets live data exceed that target without + # competing as sharply with the runner agent. This does not establish the + # cause of historical runner losses or change the scanner's verdict rules. + - name: Prepare bounded swap headroom for the scanner + id: scan-swap + timeout-minutes: 1 + shell: bash + run: | + set -euo pipefail + available_kib=$(df -Pk -- "$RUNNER_TEMP" | awk 'NR == 2 { print $4 }') + if [[ ! "$available_kib" =~ ^[0-9]+$ ]] || (( available_kib < 8388608 )); then + echo '::error::Insufficient disk for 4 GiB of scanner swap plus 4 GiB of free headroom.' + exit 1 + fi + swap_file=$(mktemp "$RUNNER_TEMP/govulncheck-swap.XXXXXX") + rollback() { + status=$? + # The file may not have reached swapon yet. Keep the original failure. + sudo swapoff -- "$swap_file" 2>/dev/null || true + rm -f -- "$swap_file" + exit "$status" + } + trap rollback ERR + chmod 600 "$swap_file" + sudo fallocate -l 4294967296 -- "$swap_file" + sudo mkswap -- "$swap_file" + sudo swapon -- "$swap_file" + swapon --show=NAME --noheadings --raw | grep -Fx -- "$swap_file" + printf 'path=%s\n' "$swap_file" >> "$GITHUB_OUTPUT" + # Start of the first scan attempt, so the classifier below can tell a scan that ran out # of time from one that failed on its own (see the job's timeout-minutes comment). - name: ⏱️ Record vulnerability scan start @@ -914,6 +946,20 @@ jobs: go-version-file: ${{ needs.changes.outputs.module-directory }}/go.mod allow-file: ${{ steps.govulncheck-allow.outputs.present == 'true' && format('{0}/.govulncheck-allow.txt', needs.changes.outputs.module-directory) || '' }} + - name: Retire only this job's scanner swap + id: scan-swap-cleanup + if: ${{ always() && steps.scan-swap.outputs.path != '' }} + timeout-minutes: 1 + shell: bash + env: + SWAP_FILE: ${{ steps.scan-swap.outputs.path }} + run: | + set -euo pipefail + if [[ -e "$SWAP_FILE" ]]; then + sudo swapoff -- "$SWAP_FILE" + rm -- "$SWAP_FILE" + fi + lint: name: 🧹 Lint - mega-linter timeout-minutes: 15 From c143acd52f5183bc692722fc35d97769bfe861d4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 16 Nov 2026 04:17:56 +0100 Subject: [PATCH 2/2] test(go): read native swap sizes with supported options Use the raw byte-sized swapon table for the native kernel assertion. The runner does not support JSON output. Part of #950. --- .github/tests/test-go-analysis-swap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/tests/test-go-analysis-swap.sh b/.github/tests/test-go-analysis-swap.sh index b8a4c7cf..ab4f7d0b 100644 --- a/.github/tests/test-go-analysis-swap.sh +++ b/.github/tests/test-go-analysis-swap.sh @@ -28,7 +28,7 @@ if [[ "${1:-}" == --live ]]; then # Preserve cleanup even if a native assertion fails. trap 'bash "$work/cleanup.sh"; rm -rf "$work"' EXIT [[ "$(stat -c %a "$SWAP_FILE")" == 600 ]] || fail 'swap must be private' - bytes="$(swapon --show=NAME,SIZE --bytes --json | jq -r --arg p "$SWAP_FILE" '.swapdevices[] | select(.name == $p) | .size')" + bytes="$(swapon --show=NAME,SIZE --bytes --noheadings --raw | awk -v p="$SWAP_FILE" '$1 == p { print $2 }')" [[ "$bytes" =~ ^[0-9]+$ && "$bytes" -ge 4294963200 ]] || fail 'native kernel did not activate four GiB of swap' bash "$work/cleanup.sh" [[ ! -e "$SWAP_FILE" ]] || fail 'native cleanup left its file behind'