diff --git a/scripts/ci/auto-merge-sweep.sh b/scripts/ci/auto-merge-sweep.sh index 5b7cb79e8..b585c4c77 100644 --- a/scripts/ci/auto-merge-sweep.sh +++ b/scripts/ci/auto-merge-sweep.sh @@ -53,6 +53,76 @@ REARM_WORKFLOWS="${REARM_WORKFLOWS:-$CI_WORKFLOW}" # A PR wearing any of these is never merged automatically. HOLD_LABELS='["hold","no-automerge","do-not-merge","wip"]' +# How many attempts a single run may reach before this stops retrying it. A +# GitHub incident can last hours; without a cap the sweep would re-run the same +# doomed run every 10 minutes indefinitely and bury the real signal. +MAX_RUN_ATTEMPTS="${MAX_RUN_ATTEMPTS:-3}" + +# Did this run fail WITHOUT executing any of our own steps? +# +# A job whose only failed step is "Set up job" never ran a line of this repo's +# code — GitHub could not resolve an action, provision the runner, or start the +# container. That is not a verdict about the code; it is the same shape of noise +# as a cancellation. But it lands as conclusion=failure, and the retry policy +# below deliberately leaves genuine failures alone — so during a GitHub Actions +# incident EVERY open PR is stranded permanently, with no human in the loop to +# notice. +# +# Observed 2026-08-06 (Actions major outage, webhooks throttled to ~15%): +# "Failed to resolve action download info. Error: Service Unavailable" failed +# three jobs on PR #278; the job's step list was exactly [Set up job: failure]. +# +# Conservative by construction: it re-runs only when EVERY failed job across the +# run failed at set-up. One real step failure anywhere and this returns false. +run_failure_is_infra() { + run_failure_is_infra_id="$1" + run_failure_is_infra_steps=$(gh api \ + "repos/${REPO}/actions/runs/${run_failure_is_infra_id}/jobs" --paginate \ + --jq '[ .jobs[] + | select(.conclusion == "failure") + | [ .steps[]? | select(.conclusion == "failure") | .name ] ] + | flatten | unique | join("|")' 2>/dev/null) || return 1 + + # Empty means no failed job was visible, or the API did not answer. Either way + # we do not know, and guessing "infra" here would re-run real failures forever. + [ -n "$run_failure_is_infra_steps" ] || return 1 + [ "$run_failure_is_infra_steps" = "Set up job" ] +} + +# Is this conclusion a statement ABOUT THE CODE, or just noise? +# +# `cancelled` never is (concurrency:cancel-in-progress, or an incident killing +# the run mid-flight). `failure` usually is — unless nothing of ours ever ran. +run_conclusion_is_non_verdict() { + case "$1" in + cancelled) return 0 ;; + failure) run_failure_is_infra "$2" ;; + *) return 1 ;; + esac +} + +# Re-run a run that produced no verdict, capped by its own attempt counter. +# +# `gh run rerun` (rather than a fresh `workflow run` dispatch) is deliberate: +# re-running increments run_attempt, so the attempt counter IS the loop cap. A +# fresh dispatch would start every retry back at attempt 1 and could churn +# forever. Prints why it declined, so the sweep log always explains itself. +rerun_non_verdict_run() { + rerun_id="$1" + rerun_what="$2" + + rerun_attempt=$(gh api "repos/${REPO}/actions/runs/${rerun_id}" \ + --jq '.run_attempt // 1' 2>/dev/null || echo "$MAX_RUN_ATTEMPTS") + if [ "$rerun_attempt" -ge "$MAX_RUN_ATTEMPTS" ]; then + echo "[auto-merge] ${rerun_what} run ${rerun_id} already at attempt ${rerun_attempt}/${MAX_RUN_ATTEMPTS} — not retrying again" >&2 + return 1 + fi + + echo "[auto-merge] ${rerun_what} run ${rerun_id} produced no verdict (attempt ${rerun_attempt}) — re-running" + gh run rerun "$rerun_id" --repo "$REPO" \ + || { echo "[auto-merge] could not re-run ${rerun_id}" >&2; return 1; } +} + echo "[auto-merge] sweeping open PRs against ${BASE_BRANCH} in ${REPO}" # Never add changes to a base that is red or mid-verification. @@ -64,7 +134,7 @@ echo "[auto-merge] sweeping open PRs against ${BASE_BRANCH} in ${REPO}" # batching this script exists to prevent. base_sha=$(gh api "repos/${REPO}/commits/${BASE_BRANCH}" --jq '.sha') base_ci=$(gh run list --repo "$REPO" --workflow "$CI_WORKFLOW" --branch "$BASE_BRANCH" --limit 1 \ - --json status,conclusion,headSha --jq '.[0] // empty') + --json status,conclusion,headSha,databaseId --jq '.[0] // empty') if [ -z "$base_ci" ]; then echo "[auto-merge] no CI history for ${BASE_BRANCH} — proceeding" @@ -72,6 +142,7 @@ else base_status=$(printf '%s' "$base_ci" | jq -r '.status') base_conclusion=$(printf '%s' "$base_ci" | jq -r '.conclusion // ""') base_ci_sha=$(printf '%s' "$base_ci" | jq -r '.headSha') + base_run_id=$(printf '%s' "$base_ci" | jq -r '.databaseId') if [ "$base_ci_sha" != "$base_sha" ]; then echo "[auto-merge] ${BASE_BRANCH} is at ${base_sha:0:8} but the newest CI run is for ${base_ci_sha:0:8} — waiting for CI to catch up" @@ -82,6 +153,23 @@ else exit 0 fi if [ "$base_conclusion" != "success" ]; then + # THE DEADLOCK. The only thing that produces a new CI run on the base is a + # merge, and merges are exactly what this guard blocks — so a base run that + # ended without a verdict strands every open PR until a human notices, and + # nothing signals that they should. The sweep still exits 0, so from the + # outside the automation looks perfectly healthy while merging nothing. + # + # Observed in this repo 2026-08-07: an Actions incident left main's run + # `failure` with no failed job at all (one cancelled, rest green). It held + # 11 PRs for ~14h. The retry below is the same reasoning already applied to + # PR checks further down — this is the sibling path that was left open. + # + # A genuine failure still blocks: that IS a verdict about the code. + if run_conclusion_is_non_verdict "$base_conclusion" "$base_run_id"; then + rerun_non_verdict_run "$base_run_id" "${BASE_BRANCH}" || true + echo "[auto-merge] deferring to the next sweep to judge ${BASE_BRANCH}" + exit 0 + fi echo "[auto-merge] ${BASE_BRANCH} CI is ${base_conclusion} — refusing to merge onto a broken base" >&2 exit 0 fi @@ -136,28 +224,60 @@ for number in $(printf '%s' "$prs_json" | jq -r 'sort_by(.number) | .[].number') if [ "$verdict" != "merge" ]; then echo "[auto-merge] #${number} ${verdict} — ${title}" - # A CANCELLED check is not a verdict, it is noise: CI workflows in this - # fleet use `concurrency: cancel-in-progress`, so an unrelated newer run on - # the same ref can kill a PR's build. Nothing ever re-runs it, the PR is - # never green, and it would sit in this queue forever. Re-run it and let a - # later sweep judge the real result. Genuine failures are left alone; only a - # run with no real failure is retried. + # Two kinds of non-verdict are retried; a genuine failure is never touched. + # + # CANCELLED: CI workflows here use `concurrency: cancel-in-progress`, so an + # unrelated newer run on the same ref can kill a PR's build. Nothing else + # re-runs it, so the PR would sit in this queue forever. + # + # SET-UP-ONLY FAILURE: the run never executed a step of ours (see + # run_failure_is_infra). It lands as conclusion=failure but says nothing + # about the code. + # + # Both are re-run so a later sweep can judge the real result. if [ "$verdict" = "skip: checks not green" ]; then - retry_urls=$(printf '%s' "$pr" | jq -r ' - [ .statusCheckRollup[]? - | select(has("state") | not) - | select((.conclusion // "") == "CANCELLED") - | .detailsUrl ] as $cancelled - | [ .statusCheckRollup[]? - | select(((.conclusion // .state // "") - | test("^(FAILURE|TIMED_OUT|ACTION_REQUIRED|STARTUP_FAILURE|ERROR)$"))) ] as $failed - | if ($failed | length) == 0 then $cancelled[] else empty end + cancelled_urls=$(printf '%s' "$pr" | jq -r ' + .statusCheckRollup[]? + | select(has("state") | not) + | select((.conclusion // "") == "CANCELLED") + | .detailsUrl + ') + failed_urls=$(printf '%s' "$pr" | jq -r ' + .statusCheckRollup[]? + | select(has("state") | not) + | select(((.conclusion // "") + | test("^(FAILURE|TIMED_OUT|ACTION_REQUIRED|STARTUP_FAILURE|ERROR)$"))) + | .detailsUrl ') + + retry_urls="" + if [ -z "$failed_urls" ]; then + retry_urls="$cancelled_urls" + else + # Retry a failing PR ONLY if every failed run never reached our code. + all_infra=1 + for url in $failed_urls; do + run_id=$(printf '%s' "$url" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+' || true) + if [ -z "$run_id" ] || ! run_failure_is_infra "$run_id"; then + all_infra=0 + break + fi + done + if [ "$all_infra" = "1" ]; then + echo "[auto-merge] #${number} failures never reached our code (set-up only) — treating as infrastructure" + retry_urls=$(printf '%s\n%s' "$failed_urls" "$cancelled_urls") + fi + fi + + # Whole run, never `--failed`. A partial re-run re-runs the failed jobs + # but flips everything that was SKIPPED to CANCELLED, so the PR ends up + # non-green for a NEW reason and needs yet another retry. Observed on + # #278 on 2026-08-07. Re-running everything costs more minutes and is + # the only way to get one coherent verdict. for url in $retry_urls; do run_id=$(printf '%s' "$url" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+' || true) [ -z "$run_id" ] && continue - echo "[auto-merge] #${number} re-running cancelled run ${run_id}" - gh run rerun "$run_id" --repo "$REPO" || echo "[auto-merge] #${number} could not re-run ${run_id}" >&2 + rerun_non_verdict_run "$run_id" "#${number}" || true done fi continue diff --git a/src/__tests__/ci/auto-merge-base-guard.test.ts b/src/__tests__/ci/auto-merge-base-guard.test.ts new file mode 100644 index 000000000..973668515 --- /dev/null +++ b/src/__tests__/ci/auto-merge-base-guard.test.ts @@ -0,0 +1,156 @@ +/** + * @jest-environment node + * + * Executes the REAL scripts/ci/auto-merge-sweep.sh against a fake `gh` on PATH. + * + * This tests shipped control flow rather than a description of it: a stubbed + * re-implementation of the guard would have passed happily while the actual + * script deadlocked, which is exactly what happened on 2026-08-07 (an Actions + * incident left main `failure` with no failed job, and the sweep refused every + * merge for ~14h while still exiting 0 and looking healthy). + */ +import { spawnSync } from 'child_process' +import { mkdtempSync, writeFileSync, readFileSync, chmodSync, existsSync } from 'fs' +import { tmpdir } from 'os' +import { join, resolve } from 'path' + +const SWEEP = resolve(__dirname, '../../../scripts/ci/auto-merge-sweep.sh') + +interface Scenario { + /** The base branch's newest CI run, as `gh run list --json ...` would report it. */ + baseCi: Record + /** Failed step names the jobs API reports for that run, `|`-joined (the script's jq shape). */ + failedSteps?: string + /** run_attempt the runs API reports. */ + runAttempt?: number +} + +interface SweepResult { + /** stdout + stderr — the guard's refusal and the cap notice both go to stderr. */ + output: string + ghCalls: string[] + reruns: string[] +} + +/** + * Writes a fake `gh` that answers by argument shape and logs every invocation. + * It returns values ALREADY filtered, because the script passes `--jq` and we + * only care about what the script does with the answer. + */ +function runSweep(scenario: Scenario): SweepResult { + const dir = mkdtempSync(join(tmpdir(), 'sweep-')) + const log = join(dir, 'gh-calls.log') + const gh = join(dir, 'gh') + + writeFileSync( + gh, + `#!/usr/bin/env bash +ARGS="$*" +echo "$ARGS" >> ${JSON.stringify(log)} +case "$ARGS" in + *"/commits/main"*) echo "basesha000000" ;; + "run list"*) cat <<'JSON' +${JSON.stringify(scenario.baseCi)} +JSON + ;; + *"/jobs"*) echo ${JSON.stringify(scenario.failedSteps ?? '')} ;; + "run rerun"*) echo "rerun dispatched" ;; + "api repos/"*"/actions/runs/"*) echo ${JSON.stringify(String(scenario.runAttempt ?? 1))} ;; + "pr list"*) echo "[]" ;; + "workflow run"*) echo "dispatched" ;; + *) echo "UNHANDLED gh call: $ARGS" >&2; exit 1 ;; +esac +`, + { mode: 0o755 }, + ) + chmodSync(gh, 0o755) + + const proc = spawnSync('bash', [SWEEP], { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + GH_REPO: 'maonakamoto/evig', + BASE_BRANCH: 'main', + }, + encoding: 'utf8', + }) + + const output = `${proc.stdout ?? ''}${proc.stderr ?? ''}` + + // The sweep must always exit 0 — it is a scheduled janitor, not a gate. A + // non-zero exit here means the fake gh hit an UNHANDLED call shape, which + // would silently make every assertion below vacuous. + expect({ status: proc.status, output }).toMatchObject({ status: 0 }) + + const ghCalls = existsSync(log) + ? readFileSync(log, 'utf8').split('\n').filter(Boolean) + : [] + + return { output, ghCalls, reruns: ghCalls.filter((c) => c.startsWith('run rerun')) } +} + +/** The base run is at the branch tip and completed — only the conclusion varies. */ +const baseCi = (conclusion: string) => ({ + status: 'completed', + conclusion, + headSha: 'basesha000000', + databaseId: 31119402753, +}) + +describe('auto-merge sweep — base branch guard', () => { + it('re-runs a CANCELLED base run instead of deadlocking behind it', () => { + const { reruns, output } = runSweep({ baseCi: baseCi('cancelled') }) + + // Nothing but a merge produces a new base CI run, and merges are what the + // guard blocks — so without this the queue can never recover on its own. + expect(reruns).toHaveLength(1) + expect(reruns[0]).toContain('31119402753') + expect(output).toContain('produced no verdict') + }) + + it('re-runs a base run that FAILED before executing any of our code', () => { + // The 2026-08-07 shape: conclusion=failure, but the only failed step is the + // runner refusing to start. That is not a verdict about the code. + const { reruns } = runSweep({ + baseCi: baseCi('failure'), + failedSteps: 'Set up job', + }) + + expect(reruns).toHaveLength(1) + }) + + it('refuses to merge onto a genuinely broken base, and does NOT re-run it', () => { + const { reruns, output } = runSweep({ + baseCi: baseCi('failure'), + failedSteps: 'Verify (lint + umlauts + typecheck + build)', + }) + + expect(reruns).toHaveLength(0) + expect(output).toContain('refusing to merge onto a broken base') + }) + + it('does not re-run a real failure even when the jobs API says nothing', () => { + // Empty means "we could not tell". Guessing "infra" here would re-run + // genuine failures forever. + const { reruns } = runSweep({ baseCi: baseCi('failure'), failedSteps: '' }) + + expect(reruns).toHaveLength(0) + }) + + it('stops retrying once the run hits the attempt cap', () => { + const { reruns, output } = runSweep({ + baseCi: baseCi('cancelled'), + runAttempt: 3, + }) + + expect(reruns).toHaveLength(0) + expect(output).toContain('not retrying again') + }) + + it('proceeds to the PR loop when the base is green', () => { + const { reruns, output } = runSweep({ baseCi: baseCi('success') }) + + expect(reruns).toHaveLength(0) + expect(output).toContain('no open PRs') + }) +})