fix(ci): retry runs that failed before executing any of our code

scripts/ci/auto-merge-sweep.sh

@@ -53,6 +53,76 @@ REARM_WORKFLOWS="${REARM_WORKFLOWS:-$CI_WORKFLOW}"
 # A PR wearing any of these is never merged automatically.
 HOLD_LABELS='["hold","no-automerge","do-not-merge","wip"]'
 
+# How many attempts a single run may reach before this stops retrying it. A
+# GitHub incident can last hours; without a cap the sweep would re-run the same
+# doomed run every 10 minutes indefinitely and bury the real signal.
+MAX_RUN_ATTEMPTS="${MAX_RUN_ATTEMPTS:-3}"
+
+# Did this run fail WITHOUT executing any of our own steps?
+#
+# A job whose only failed step is "Set up job" never ran a line of this repo's
+# code — GitHub could not resolve an action, provision the runner, or start the
+# container. That is not a verdict about the code; it is the same shape of noise
+# as a cancellation. But it lands as conclusion=failure, and the retry policy
+# below deliberately leaves genuine failures alone — so during a GitHub Actions
+# incident EVERY open PR is stranded permanently, with no human in the loop to
+# notice.
+#
+# Observed 2026-08-06 (Actions major outage, webhooks throttled to ~15%):
+# "Failed to resolve action download info. Error: Service Unavailable" failed
+# three jobs on PR #278; the job's step list was exactly [Set up job: failure].
+#
+# Conservative by construction: it re-runs only when EVERY failed job across the
+# run failed at set-up. One real step failure anywhere and this returns false.
+run_failure_is_infra() {
+  run_failure_is_infra_id="$1"
+  run_failure_is_infra_steps=$(gh api \
+    "repos/${REPO}/actions/runs/${run_failure_is_infra_id}/jobs" --paginate \
+    --jq '[ .jobs[]
+            | select(.conclusion == "failure")
+            | [ .steps[]? | select(.conclusion == "failure") | .name ] ]
+          | flatten | unique | join("|")' 2>/dev/null) || return 1
+
+  # Empty means no failed job was visible, or the API did not answer. Either way
+  # we do not know, and guessing "infra" here would re-run real failures forever.
+  [ -n "$run_failure_is_infra_steps" ] || return 1
+  [ "$run_failure_is_infra_steps" = "Set up job" ]
+}
+
+# Is this conclusion a statement ABOUT THE CODE, or just noise?
+#
+# `cancelled` never is (concurrency:cancel-in-progress, or an incident killing
+# the run mid-flight). `failure` usually is — unless nothing of ours ever ran.
+run_conclusion_is_non_verdict() {
+  case "$1" in
+    cancelled) return 0 ;;
+    failure) run_failure_is_infra "$2" ;;
+    *) return 1 ;;
+  esac
+}
+
+# Re-run a run that produced no verdict, capped by its own attempt counter.
+#
+# `gh run rerun` (rather than a fresh `workflow run` dispatch) is deliberate:
+# re-running increments run_attempt, so the attempt counter IS the loop cap. A
+# fresh dispatch would start every retry back at attempt 1 and could churn
+# forever. Prints why it declined, so the sweep log always explains itself.
+rerun_non_verdict_run() {
+  rerun_id="$1"
+  rerun_what="$2"
+
+  rerun_attempt=$(gh api "repos/${REPO}/actions/runs/${rerun_id}" \
+    --jq '.run_attempt // 1' 2>/dev/null || echo "$MAX_RUN_ATTEMPTS")
+  if [ "$rerun_attempt" -ge "$MAX_RUN_ATTEMPTS" ]; then
+    echo "[auto-merge] ${rerun_what} run ${rerun_id} already at attempt ${rerun_attempt}/${MAX_RUN_ATTEMPTS} — not retrying again" >&2
+    return 1
+  fi
+
+  echo "[auto-merge] ${rerun_what} run ${rerun_id} produced no verdict (attempt ${rerun_attempt}) — re-running"
+  gh run rerun "$rerun_id" --repo "$REPO" \
+    || { echo "[auto-merge] could not re-run ${rerun_id}" >&2; return 1; }
+}
+
 echo "[auto-merge] sweeping open PRs against ${BASE_BRANCH} in ${REPO}"
 
 # Never add changes to a base that is red or mid-verification.
@@ -64,14 +134,15 @@ echo "[auto-merge] sweeping open PRs against ${BASE_BRANCH} in ${REPO}"
 # batching this script exists to prevent.
 base_sha=$(gh api "repos/${REPO}/commits/${BASE_BRANCH}" --jq '.sha')
 base_ci=$(gh run list --repo "$REPO" --workflow "$CI_WORKFLOW" --branch "$BASE_BRANCH" --limit 1 \
-  --json status,conclusion,headSha --jq '.[0] // empty')
+  --json status,conclusion,headSha,databaseId --jq '.[0] // empty')
 
 if [ -z "$base_ci" ]; then
   echo "[auto-merge] no CI history for ${BASE_BRANCH} — proceeding"
 else
   base_status=$(printf '%s' "$base_ci" | jq -r '.status')
   base_conclusion=$(printf '%s' "$base_ci" | jq -r '.conclusion // ""')
   base_ci_sha=$(printf '%s' "$base_ci" | jq -r '.headSha')
+  base_run_id=$(printf '%s' "$base_ci" | jq -r '.databaseId')
 
   if [ "$base_ci_sha" != "$base_sha" ]; then
     echo "[auto-merge] ${BASE_BRANCH} is at ${base_sha:0:8} but the newest CI run is for ${base_ci_sha:0:8} — waiting for CI to catch up"
@@ -82,6 +153,23 @@ else
     exit 0
   fi
   if [ "$base_conclusion" != "success" ]; then
+    # THE DEADLOCK. The only thing that produces a new CI run on the base is a
+    # merge, and merges are exactly what this guard blocks — so a base run that
+    # ended without a verdict strands every open PR until a human notices, and
+    # nothing signals that they should. The sweep still exits 0, so from the
+    # outside the automation looks perfectly healthy while merging nothing.
+    #
+    # Observed in this repo 2026-08-07: an Actions incident left main's run
+    # `failure` with no failed job at all (one cancelled, rest green). It held
+    # 11 PRs for ~14h. The retry below is the same reasoning already applied to
+    # PR checks further down — this is the sibling path that was left open.
+    #
+    # A genuine failure still blocks: that IS a verdict about the code.
+    if run_conclusion_is_non_verdict "$base_conclusion" "$base_run_id"; then
+      rerun_non_verdict_run "$base_run_id" "${BASE_BRANCH}" || true
+      echo "[auto-merge] deferring to the next sweep to judge ${BASE_BRANCH}"
+      exit 0
+    fi
     echo "[auto-merge] ${BASE_BRANCH} CI is ${base_conclusion} — refusing to merge onto a broken base" >&2
     exit 0
   fi
@@ -136,28 +224,60 @@ for number in $(printf '%s' "$prs_json" | jq -r 'sort_by(.number) | .[].number')
   if [ "$verdict" != "merge" ]; then
     echo "[auto-merge] #${number} ${verdict} — ${title}"
 
-    # A CANCELLED check is not a verdict, it is noise: CI workflows in this
-    # fleet use `concurrency: cancel-in-progress`, so an unrelated newer run on
-    # the same ref can kill a PR's build. Nothing ever re-runs it, the PR is
-    # never green, and it would sit in this queue forever. Re-run it and let a
-    # later sweep judge the real result. Genuine failures are left alone; only a
-    # run with no real failure is retried.
+    # Two kinds of non-verdict are retried; a genuine failure is never touched.
+    #
+    # CANCELLED: CI workflows here use `concurrency: cancel-in-progress`, so an
+    # unrelated newer run on the same ref can kill a PR's build. Nothing else
+    # re-runs it, so the PR would sit in this queue forever.
+    #
+    # SET-UP-ONLY FAILURE: the run never executed a step of ours (see
+    # run_failure_is_infra). It lands as conclusion=failure but says nothing
+    # about the code.
+    #
+    # Both are re-run so a later sweep can judge the real result.
     if [ "$verdict" = "skip: checks not green" ]; then
-      retry_urls=$(printf '%s' "$pr" | jq -r '
-        [ .statusCheckRollup[]?
-          | select(has("state") | not)
-          | select((.conclusion // "") == "CANCELLED")
-          | .detailsUrl ] as $cancelled
-        | [ .statusCheckRollup[]?
-            | select(((.conclusion // .state // "")
-                      | test("^(FAILURE|TIMED_OUT|ACTION_REQUIRED|STARTUP_FAILURE|ERROR)$"))) ] as $failed
-        | if ($failed | length) == 0 then $cancelled[] else empty end
+      cancelled_urls=$(printf '%s' "$pr" | jq -r '
+        .statusCheckRollup[]?
+        | select(has("state") | not)
+        | select((.conclusion // "") == "CANCELLED")
+        | .detailsUrl
+      ')
+      failed_urls=$(printf '%s' "$pr" | jq -r '
+        .statusCheckRollup[]?
+        | select(has("state") | not)
+        | select(((.conclusion // "")
+                  | test("^(FAILURE|TIMED_OUT|ACTION_REQUIRED|STARTUP_FAILURE|ERROR)$")))
+        | .detailsUrl
       ')
+
+      retry_urls=""
+      if [ -z "$failed_urls" ]; then
+        retry_urls="$cancelled_urls"
+      else
+        # Retry a failing PR ONLY if every failed run never reached our code.
+        all_infra=1
+        for url in $failed_urls; do
+          run_id=$(printf '%s' "$url" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+' || true)
+          if [ -z "$run_id" ] || ! run_failure_is_infra "$run_id"; then
+            all_infra=0
+            break
+          fi
+        done
+        if [ "$all_infra" = "1" ]; then
+          echo "[auto-merge] #${number} failures never reached our code (set-up only) — treating as infrastructure"
+          retry_urls=$(printf '%s\n%s' "$failed_urls" "$cancelled_urls")
+        fi
+      fi
+
+      # Whole run, never `--failed`. A partial re-run re-runs the failed jobs
+      # but flips everything that was SKIPPED to CANCELLED, so the PR ends up
+      # non-green for a NEW reason and needs yet another retry. Observed on
+      # #278 on 2026-08-07. Re-running everything costs more minutes and is
+      # the only way to get one coherent verdict.
       for url in $retry_urls; do
         run_id=$(printf '%s' "$url" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+' || true)
         [ -z "$run_id" ] && continue
-        echo "[auto-merge] #${number} re-running cancelled run ${run_id}"
-        gh run rerun "$run_id" --repo "$REPO" || echo "[auto-merge] #${number} could not re-run ${run_id}" >&2
+        rerun_non_verdict_run "$run_id" "#${number}" || true
       done
     fi
     continue

src/__tests__/ci/auto-merge-base-guard.test.ts

@@ -0,0 +1,156 @@
+/**
+ * @jest-environment node
+ *
+ * Executes the REAL scripts/ci/auto-merge-sweep.sh against a fake `gh` on PATH.
+ *
+ * This tests shipped control flow rather than a description of it: a stubbed
+ * re-implementation of the guard would have passed happily while the actual
+ * script deadlocked, which is exactly what happened on 2026-08-07 (an Actions
+ * incident left main `failure` with no failed job, and the sweep refused every
+ * merge for ~14h while still exiting 0 and looking healthy).
+ */
+import { spawnSync } from 'child_process'
+import { mkdtempSync, writeFileSync, readFileSync, chmodSync, existsSync } from 'fs'
+import { tmpdir } from 'os'
+import { join, resolve } from 'path'
+
+const SWEEP = resolve(__dirname, '../../../scripts/ci/auto-merge-sweep.sh')
+
+interface Scenario {
+  /** The base branch's newest CI run, as `gh run list --json ...` would report it. */
+  baseCi: Record<string, unknown>
+  /** Failed step names the jobs API reports for that run, `|`-joined (the script's jq shape). */
+  failedSteps?: string
+  /** run_attempt the runs API reports. */
+  runAttempt?: number
+}
+
+interface SweepResult {
+  /** stdout + stderr — the guard's refusal and the cap notice both go to stderr. */
+  output: string
+  ghCalls: string[]
+  reruns: string[]
+}
+
+/**
+ * Writes a fake `gh` that answers by argument shape and logs every invocation.
+ * It returns values ALREADY filtered, because the script passes `--jq` and we
+ * only care about what the script does with the answer.
+ */
+function runSweep(scenario: Scenario): SweepResult {
+  const dir = mkdtempSync(join(tmpdir(), 'sweep-'))
+  const log = join(dir, 'gh-calls.log')
+  const gh = join(dir, 'gh')
+
+  writeFileSync(
+    gh,
+    `#!/usr/bin/env bash
+ARGS="$*"
+echo "$ARGS" >> ${JSON.stringify(log)}
+case "$ARGS" in
+  *"/commits/main"*)        echo "basesha000000" ;;
+  "run list"*)              cat <<'JSON'
+${JSON.stringify(scenario.baseCi)}
+JSON
+                            ;;
+  *"/jobs"*)                echo ${JSON.stringify(scenario.failedSteps ?? '')} ;;
+  "run rerun"*)             echo "rerun dispatched" ;;
+  "api repos/"*"/actions/runs/"*) echo ${JSON.stringify(String(scenario.runAttempt ?? 1))} ;;
+  "pr list"*)               echo "[]" ;;
+  "workflow run"*)          echo "dispatched" ;;
+  *) echo "UNHANDLED gh call: $ARGS" >&2; exit 1 ;;
+esac
+`,
+    { mode: 0o755 },
+  )
+  chmodSync(gh, 0o755)
+
+  const proc = spawnSync('bash', [SWEEP], {
+    env: {
+      ...process.env,
+      PATH: `${dir}:${process.env.PATH}`,
+      GH_REPO: 'maonakamoto/evig',
+      BASE_BRANCH: 'main',
+    },
+    encoding: 'utf8',
+  })
+
+  const output = `${proc.stdout ?? ''}${proc.stderr ?? ''}`
+
+  // The sweep must always exit 0 — it is a scheduled janitor, not a gate. A
+  // non-zero exit here means the fake gh hit an UNHANDLED call shape, which
+  // would silently make every assertion below vacuous.
+  expect({ status: proc.status, output }).toMatchObject({ status: 0 })
+
+  const ghCalls = existsSync(log)
+    ? readFileSync(log, 'utf8').split('\n').filter(Boolean)
+    : []
+
+  return { output, ghCalls, reruns: ghCalls.filter((c) => c.startsWith('run rerun')) }
+}
+
+/** The base run is at the branch tip and completed — only the conclusion varies. */
+const baseCi = (conclusion: string) => ({
+  status: 'completed',
+  conclusion,
+  headSha: 'basesha000000',
+  databaseId: 31119402753,
+})
+
+describe('auto-merge sweep — base branch guard', () => {
+  it('re-runs a CANCELLED base run instead of deadlocking behind it', () => {
+    const { reruns, output } = runSweep({ baseCi: baseCi('cancelled') })
+
+    // Nothing but a merge produces a new base CI run, and merges are what the
+    // guard blocks — so without this the queue can never recover on its own.
+    expect(reruns).toHaveLength(1)
+    expect(reruns[0]).toContain('31119402753')
+    expect(output).toContain('produced no verdict')
+  })
+
+  it('re-runs a base run that FAILED before executing any of our code', () => {
+    // The 2026-08-07 shape: conclusion=failure, but the only failed step is the
+    // runner refusing to start. That is not a verdict about the code.
+    const { reruns } = runSweep({
+      baseCi: baseCi('failure'),
+      failedSteps: 'Set up job',
+    })
+
+    expect(reruns).toHaveLength(1)
+  })
+
+  it('refuses to merge onto a genuinely broken base, and does NOT re-run it', () => {
+    const { reruns, output } = runSweep({
+      baseCi: baseCi('failure'),
+      failedSteps: 'Verify (lint + umlauts + typecheck + build)',
+    })
+
+    expect(reruns).toHaveLength(0)
+    expect(output).toContain('refusing to merge onto a broken base')
+  })
+
+  it('does not re-run a real failure even when the jobs API says nothing', () => {
+    // Empty means "we could not tell". Guessing "infra" here would re-run
+    // genuine failures forever.
+    const { reruns } = runSweep({ baseCi: baseCi('failure'), failedSteps: '' })
+
+    expect(reruns).toHaveLength(0)
+  })
+
+  it('stops retrying once the run hits the attempt cap', () => {
+    const { reruns, output } = runSweep({
+      baseCi: baseCi('cancelled'),
+      runAttempt: 3,
+    })
+
+    expect(reruns).toHaveLength(0)
+    expect(output).toContain('not retrying again')
+  })
+
+  it('proceeds to the PR loop when the base is green', () => {
+    const { reruns, output } = runSweep({ baseCi: baseCi('success') })
+
+    expect(reruns).toHaveLength(0)
+    expect(output).toContain('no open PRs')
+  })
+})