From 875e7609c28e374615d6fb33c366f1b5532ad32c Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:50:26 +0200 Subject: [PATCH 1/2] fix(ci): retry runs that failed before executing any of our code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep retries CANCELLED checks but deliberately leaves genuine failures alone. A GitHub Actions incident lands in the gap between those two: the run fails with conclusion=failure, so it is treated as a real verdict — but it never executed a line of this repo's code. Consequence: during an incident EVERY open PR is stranded permanently, and nobody is in the merge loop to notice. Today (2026-08-06, Actions major outage, webhooks throttled to ~15%) three jobs on PR #278 died with "Failed to resolve action download info. Error: Service Unavailable" and that PR cannot become green again without a human running `gh run rerun --failed`. The discriminator is precise: a job whose ONLY failed step is "Set up job" never got as far as running our code, so it produced no verdict about it. A real failure names a real step. Verified against live runs: PR #278 outage run -> [Set up job] -> infra PR #228 (TS 7) -> [Verify (lint + umlauts + typecheck + build)] -> real PR #265 (ESLint 10) -> [Verify (lint + umlauts + typecheck + build)] -> real nonexistent run -> no data -> refuses to guess Conservative by construction: retries only when EVERY failed job in the run failed at set-up. One real step failure anywhere and the PR is left alone. Capped at MAX_RUN_ATTEMPTS (default 3) — an incident can last hours, and an uncapped retry would re-run the same doomed run every sweep forever. Co-Authored-By: Claude Opus 5 --- scripts/ci/auto-merge-sweep.sh | 107 +++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 17 deletions(-) diff --git a/scripts/ci/auto-merge-sweep.sh b/scripts/ci/auto-merge-sweep.sh index 5b7cb79e8..ab4570407 100644 --- a/scripts/ci/auto-merge-sweep.sh +++ b/scripts/ci/auto-merge-sweep.sh @@ -53,6 +53,11 @@ REARM_WORKFLOWS="${REARM_WORKFLOWS:-$CI_WORKFLOW}" # A PR wearing any of these is never merged automatically. HOLD_LABELS='["hold","no-automerge","do-not-merge","wip"]' +# How many attempts a single run may reach before this stops retrying it. A +# GitHub incident can last hours; without a cap the sweep would re-run the same +# doomed run every 10 minutes indefinitely and bury the real signal. +MAX_RUN_ATTEMPTS="${MAX_RUN_ATTEMPTS:-3}" + echo "[auto-merge] sweeping open PRs against ${BASE_BRANCH} in ${REPO}" # Never add changes to a base that is red or mid-verification. @@ -98,6 +103,37 @@ fi merged_any=0 +# Did this run fail WITHOUT executing any of our own steps? +# +# A job whose only failed step is "Set up job" never ran a line of this repo's +# code — GitHub could not resolve an action, provision the runner, or start the +# container. That is not a verdict about the code; it is the same shape of noise +# as a cancellation. But it lands as conclusion=failure, and the retry policy +# below deliberately leaves genuine failures alone — so during a GitHub Actions +# incident EVERY open PR is stranded permanently, with no human in the loop to +# notice. +# +# Observed 2026-08-06 (Actions major outage, webhooks throttled to ~15%): +# "Failed to resolve action download info. Error: Service Unavailable" failed +# three jobs on PR #278; the job's step list was exactly [Set up job: failure]. +# +# Conservative by construction: it re-runs only when EVERY failed job across the +# run failed at set-up. One real step failure anywhere and this returns false. +run_failure_is_infra() { + run_failure_is_infra_id="$1" + run_failure_is_infra_steps=$(gh api \ + "repos/${REPO}/actions/runs/${run_failure_is_infra_id}/jobs" --paginate \ + --jq '[ .jobs[] + | select(.conclusion == "failure") + | [ .steps[]? | select(.conclusion == "failure") | .name ] ] + | flatten | unique | join("|")' 2>/dev/null) || return 1 + + # Empty means no failed job was visible, or the API did not answer. Either way + # we do not know, and guessing "infra" here would re-run real failures forever. + [ -n "$run_failure_is_infra_steps" ] || return 1 + [ "$run_failure_is_infra_steps" = "Set up job" ] +} + # OLDEST FIRST. `gh pr list` returns newest-first, and this loop merges the # first eligible PR and stops — so the newest green PR wins every sweep and an # older one can wait indefinitely. Observed in maonakamoto/fleetcrown on @@ -136,28 +172,65 @@ for number in $(printf '%s' "$prs_json" | jq -r 'sort_by(.number) | .[].number') if [ "$verdict" != "merge" ]; then echo "[auto-merge] #${number} ${verdict} — ${title}" - # A CANCELLED check is not a verdict, it is noise: CI workflows in this - # fleet use `concurrency: cancel-in-progress`, so an unrelated newer run on - # the same ref can kill a PR's build. Nothing ever re-runs it, the PR is - # never green, and it would sit in this queue forever. Re-run it and let a - # later sweep judge the real result. Genuine failures are left alone; only a - # run with no real failure is retried. + # Two kinds of non-verdict are retried; a genuine failure is never touched. + # + # CANCELLED: CI workflows here use `concurrency: cancel-in-progress`, so an + # unrelated newer run on the same ref can kill a PR's build. Nothing else + # re-runs it, so the PR would sit in this queue forever. + # + # SET-UP-ONLY FAILURE: the run never executed a step of ours (see + # run_failure_is_infra). It lands as conclusion=failure but says nothing + # about the code. + # + # Both are re-run so a later sweep can judge the real result. if [ "$verdict" = "skip: checks not green" ]; then - retry_urls=$(printf '%s' "$pr" | jq -r ' - [ .statusCheckRollup[]? - | select(has("state") | not) - | select((.conclusion // "") == "CANCELLED") - | .detailsUrl ] as $cancelled - | [ .statusCheckRollup[]? - | select(((.conclusion // .state // "") - | test("^(FAILURE|TIMED_OUT|ACTION_REQUIRED|STARTUP_FAILURE|ERROR)$"))) ] as $failed - | if ($failed | length) == 0 then $cancelled[] else empty end + cancelled_urls=$(printf '%s' "$pr" | jq -r ' + .statusCheckRollup[]? + | select(has("state") | not) + | select((.conclusion // "") == "CANCELLED") + | .detailsUrl + ') + failed_urls=$(printf '%s' "$pr" | jq -r ' + .statusCheckRollup[]? + | select(has("state") | not) + | select(((.conclusion // "") + | test("^(FAILURE|TIMED_OUT|ACTION_REQUIRED|STARTUP_FAILURE|ERROR)$"))) + | .detailsUrl ') + + retry_urls="" + if [ -z "$failed_urls" ]; then + retry_urls="$cancelled_urls" + else + # Retry a failing PR ONLY if every failed run never reached our code. + all_infra=1 + for url in $failed_urls; do + run_id=$(printf '%s' "$url" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+' || true) + if [ -z "$run_id" ] || ! run_failure_is_infra "$run_id"; then + all_infra=0 + break + fi + done + if [ "$all_infra" = "1" ]; then + echo "[auto-merge] #${number} failures never reached our code (set-up only) — treating as infrastructure" + retry_urls=$(printf '%s\n%s' "$failed_urls" "$cancelled_urls") + fi + fi + for url in $retry_urls; do run_id=$(printf '%s' "$url" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+' || true) [ -z "$run_id" ] && continue - echo "[auto-merge] #${number} re-running cancelled run ${run_id}" - gh run rerun "$run_id" --repo "$REPO" || echo "[auto-merge] #${number} could not re-run ${run_id}" >&2 + # Cap attempts: during a multi-hour incident an uncapped retry would + # re-run the same doomed run every sweep, forever. + attempt=$(gh api "repos/${REPO}/actions/runs/${run_id}" --jq '.run_attempt // 1' 2>/dev/null || echo "$MAX_RUN_ATTEMPTS") + if [ "$attempt" -ge "$MAX_RUN_ATTEMPTS" ]; then + echo "[auto-merge] #${number} run ${run_id} already at attempt ${attempt} — not retrying again" + continue + fi + echo "[auto-merge] #${number} re-running run ${run_id} (attempt ${attempt})" + gh run rerun "$run_id" --repo "$REPO" --failed \ + || gh run rerun "$run_id" --repo "$REPO" \ + || echo "[auto-merge] #${number} could not re-run ${run_id}" >&2 done fi continue From da804ccf7741e5bdfa2b3e147a7cf9702a3beca0 Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:32:55 +0200 Subject: [PATCH 2/2] fix(ci): self-heal a base branch whose CI ended without a verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep already knew that a cancelled/infra-failed run is noise rather than a judgement — but only for PR checks. The base branch kept the original "refuse and exit" behaviour, which deadlocks: the only thing that produces a new CI run on main is a merge, and merges are exactly what the guard blocks. Observed here on 2026-08-07. An Actions incident left main's run `failure` with no failed job at all (Migration Drift cancelled, everything else green). Eleven PRs sat for ~14h while every sweep exited 0 and looked healthy. So: extract the "is this a verdict about the code?" test and apply it to the base run too. A genuine failure still blocks — that IS a verdict. Retries are capped by the run's own attempt counter, which is why this re-runs rather than dispatching fresh (a new dispatch resets to attempt 1 and could churn forever). Also drops `--failed` from the PR-side retry. A partial re-run flips SKIPPED jobs to CANCELLED, so the PR ends up non-green for a brand new reason and needs yet another retry — seen on #278 today. Tested by running the real script against a fake `gh` on PATH, so this covers shipped control flow rather than a re-description of it. Mutation-checked: disabling the self-heal turns 3 of the 6 tests red. Co-Authored-By: Claude Opus 5 --- scripts/ci/auto-merge-sweep.sh | 133 ++++++++++----- .../ci/auto-merge-base-guard.test.ts | 156 ++++++++++++++++++ 2 files changed, 246 insertions(+), 43 deletions(-) create mode 100644 src/__tests__/ci/auto-merge-base-guard.test.ts diff --git a/scripts/ci/auto-merge-sweep.sh b/scripts/ci/auto-merge-sweep.sh index ab4570407..b585c4c77 100644 --- a/scripts/ci/auto-merge-sweep.sh +++ b/scripts/ci/auto-merge-sweep.sh @@ -58,6 +58,71 @@ HOLD_LABELS='["hold","no-automerge","do-not-merge","wip"]' # doomed run every 10 minutes indefinitely and bury the real signal. MAX_RUN_ATTEMPTS="${MAX_RUN_ATTEMPTS:-3}" +# Did this run fail WITHOUT executing any of our own steps? +# +# A job whose only failed step is "Set up job" never ran a line of this repo's +# code — GitHub could not resolve an action, provision the runner, or start the +# container. That is not a verdict about the code; it is the same shape of noise +# as a cancellation. But it lands as conclusion=failure, and the retry policy +# below deliberately leaves genuine failures alone — so during a GitHub Actions +# incident EVERY open PR is stranded permanently, with no human in the loop to +# notice. +# +# Observed 2026-08-06 (Actions major outage, webhooks throttled to ~15%): +# "Failed to resolve action download info. Error: Service Unavailable" failed +# three jobs on PR #278; the job's step list was exactly [Set up job: failure]. +# +# Conservative by construction: it re-runs only when EVERY failed job across the +# run failed at set-up. One real step failure anywhere and this returns false. +run_failure_is_infra() { + run_failure_is_infra_id="$1" + run_failure_is_infra_steps=$(gh api \ + "repos/${REPO}/actions/runs/${run_failure_is_infra_id}/jobs" --paginate \ + --jq '[ .jobs[] + | select(.conclusion == "failure") + | [ .steps[]? | select(.conclusion == "failure") | .name ] ] + | flatten | unique | join("|")' 2>/dev/null) || return 1 + + # Empty means no failed job was visible, or the API did not answer. Either way + # we do not know, and guessing "infra" here would re-run real failures forever. + [ -n "$run_failure_is_infra_steps" ] || return 1 + [ "$run_failure_is_infra_steps" = "Set up job" ] +} + +# Is this conclusion a statement ABOUT THE CODE, or just noise? +# +# `cancelled` never is (concurrency:cancel-in-progress, or an incident killing +# the run mid-flight). `failure` usually is — unless nothing of ours ever ran. +run_conclusion_is_non_verdict() { + case "$1" in + cancelled) return 0 ;; + failure) run_failure_is_infra "$2" ;; + *) return 1 ;; + esac +} + +# Re-run a run that produced no verdict, capped by its own attempt counter. +# +# `gh run rerun` (rather than a fresh `workflow run` dispatch) is deliberate: +# re-running increments run_attempt, so the attempt counter IS the loop cap. A +# fresh dispatch would start every retry back at attempt 1 and could churn +# forever. Prints why it declined, so the sweep log always explains itself. +rerun_non_verdict_run() { + rerun_id="$1" + rerun_what="$2" + + rerun_attempt=$(gh api "repos/${REPO}/actions/runs/${rerun_id}" \ + --jq '.run_attempt // 1' 2>/dev/null || echo "$MAX_RUN_ATTEMPTS") + if [ "$rerun_attempt" -ge "$MAX_RUN_ATTEMPTS" ]; then + echo "[auto-merge] ${rerun_what} run ${rerun_id} already at attempt ${rerun_attempt}/${MAX_RUN_ATTEMPTS} — not retrying again" >&2 + return 1 + fi + + echo "[auto-merge] ${rerun_what} run ${rerun_id} produced no verdict (attempt ${rerun_attempt}) — re-running" + gh run rerun "$rerun_id" --repo "$REPO" \ + || { echo "[auto-merge] could not re-run ${rerun_id}" >&2; return 1; } +} + echo "[auto-merge] sweeping open PRs against ${BASE_BRANCH} in ${REPO}" # Never add changes to a base that is red or mid-verification. @@ -69,7 +134,7 @@ echo "[auto-merge] sweeping open PRs against ${BASE_BRANCH} in ${REPO}" # batching this script exists to prevent. base_sha=$(gh api "repos/${REPO}/commits/${BASE_BRANCH}" --jq '.sha') base_ci=$(gh run list --repo "$REPO" --workflow "$CI_WORKFLOW" --branch "$BASE_BRANCH" --limit 1 \ - --json status,conclusion,headSha --jq '.[0] // empty') + --json status,conclusion,headSha,databaseId --jq '.[0] // empty') if [ -z "$base_ci" ]; then echo "[auto-merge] no CI history for ${BASE_BRANCH} — proceeding" @@ -77,6 +142,7 @@ else base_status=$(printf '%s' "$base_ci" | jq -r '.status') base_conclusion=$(printf '%s' "$base_ci" | jq -r '.conclusion // ""') base_ci_sha=$(printf '%s' "$base_ci" | jq -r '.headSha') + base_run_id=$(printf '%s' "$base_ci" | jq -r '.databaseId') if [ "$base_ci_sha" != "$base_sha" ]; then echo "[auto-merge] ${BASE_BRANCH} is at ${base_sha:0:8} but the newest CI run is for ${base_ci_sha:0:8} — waiting for CI to catch up" @@ -87,6 +153,23 @@ else exit 0 fi if [ "$base_conclusion" != "success" ]; then + # THE DEADLOCK. The only thing that produces a new CI run on the base is a + # merge, and merges are exactly what this guard blocks — so a base run that + # ended without a verdict strands every open PR until a human notices, and + # nothing signals that they should. The sweep still exits 0, so from the + # outside the automation looks perfectly healthy while merging nothing. + # + # Observed in this repo 2026-08-07: an Actions incident left main's run + # `failure` with no failed job at all (one cancelled, rest green). It held + # 11 PRs for ~14h. The retry below is the same reasoning already applied to + # PR checks further down — this is the sibling path that was left open. + # + # A genuine failure still blocks: that IS a verdict about the code. + if run_conclusion_is_non_verdict "$base_conclusion" "$base_run_id"; then + rerun_non_verdict_run "$base_run_id" "${BASE_BRANCH}" || true + echo "[auto-merge] deferring to the next sweep to judge ${BASE_BRANCH}" + exit 0 + fi echo "[auto-merge] ${BASE_BRANCH} CI is ${base_conclusion} — refusing to merge onto a broken base" >&2 exit 0 fi @@ -103,37 +186,6 @@ fi merged_any=0 -# Did this run fail WITHOUT executing any of our own steps? -# -# A job whose only failed step is "Set up job" never ran a line of this repo's -# code — GitHub could not resolve an action, provision the runner, or start the -# container. That is not a verdict about the code; it is the same shape of noise -# as a cancellation. But it lands as conclusion=failure, and the retry policy -# below deliberately leaves genuine failures alone — so during a GitHub Actions -# incident EVERY open PR is stranded permanently, with no human in the loop to -# notice. -# -# Observed 2026-08-06 (Actions major outage, webhooks throttled to ~15%): -# "Failed to resolve action download info. Error: Service Unavailable" failed -# three jobs on PR #278; the job's step list was exactly [Set up job: failure]. -# -# Conservative by construction: it re-runs only when EVERY failed job across the -# run failed at set-up. One real step failure anywhere and this returns false. -run_failure_is_infra() { - run_failure_is_infra_id="$1" - run_failure_is_infra_steps=$(gh api \ - "repos/${REPO}/actions/runs/${run_failure_is_infra_id}/jobs" --paginate \ - --jq '[ .jobs[] - | select(.conclusion == "failure") - | [ .steps[]? | select(.conclusion == "failure") | .name ] ] - | flatten | unique | join("|")' 2>/dev/null) || return 1 - - # Empty means no failed job was visible, or the API did not answer. Either way - # we do not know, and guessing "infra" here would re-run real failures forever. - [ -n "$run_failure_is_infra_steps" ] || return 1 - [ "$run_failure_is_infra_steps" = "Set up job" ] -} - # OLDEST FIRST. `gh pr list` returns newest-first, and this loop merges the # first eligible PR and stops — so the newest green PR wins every sweep and an # older one can wait indefinitely. Observed in maonakamoto/fleetcrown on @@ -217,20 +269,15 @@ for number in $(printf '%s' "$prs_json" | jq -r 'sort_by(.number) | .[].number') fi fi + # Whole run, never `--failed`. A partial re-run re-runs the failed jobs + # but flips everything that was SKIPPED to CANCELLED, so the PR ends up + # non-green for a NEW reason and needs yet another retry. Observed on + # #278 on 2026-08-07. Re-running everything costs more minutes and is + # the only way to get one coherent verdict. for url in $retry_urls; do run_id=$(printf '%s' "$url" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+' || true) [ -z "$run_id" ] && continue - # Cap attempts: during a multi-hour incident an uncapped retry would - # re-run the same doomed run every sweep, forever. - attempt=$(gh api "repos/${REPO}/actions/runs/${run_id}" --jq '.run_attempt // 1' 2>/dev/null || echo "$MAX_RUN_ATTEMPTS") - if [ "$attempt" -ge "$MAX_RUN_ATTEMPTS" ]; then - echo "[auto-merge] #${number} run ${run_id} already at attempt ${attempt} — not retrying again" - continue - fi - echo "[auto-merge] #${number} re-running run ${run_id} (attempt ${attempt})" - gh run rerun "$run_id" --repo "$REPO" --failed \ - || gh run rerun "$run_id" --repo "$REPO" \ - || echo "[auto-merge] #${number} could not re-run ${run_id}" >&2 + rerun_non_verdict_run "$run_id" "#${number}" || true done fi continue diff --git a/src/__tests__/ci/auto-merge-base-guard.test.ts b/src/__tests__/ci/auto-merge-base-guard.test.ts new file mode 100644 index 000000000..973668515 --- /dev/null +++ b/src/__tests__/ci/auto-merge-base-guard.test.ts @@ -0,0 +1,156 @@ +/** + * @jest-environment node + * + * Executes the REAL scripts/ci/auto-merge-sweep.sh against a fake `gh` on PATH. + * + * This tests shipped control flow rather than a description of it: a stubbed + * re-implementation of the guard would have passed happily while the actual + * script deadlocked, which is exactly what happened on 2026-08-07 (an Actions + * incident left main `failure` with no failed job, and the sweep refused every + * merge for ~14h while still exiting 0 and looking healthy). + */ +import { spawnSync } from 'child_process' +import { mkdtempSync, writeFileSync, readFileSync, chmodSync, existsSync } from 'fs' +import { tmpdir } from 'os' +import { join, resolve } from 'path' + +const SWEEP = resolve(__dirname, '../../../scripts/ci/auto-merge-sweep.sh') + +interface Scenario { + /** The base branch's newest CI run, as `gh run list --json ...` would report it. */ + baseCi: Record + /** Failed step names the jobs API reports for that run, `|`-joined (the script's jq shape). */ + failedSteps?: string + /** run_attempt the runs API reports. */ + runAttempt?: number +} + +interface SweepResult { + /** stdout + stderr — the guard's refusal and the cap notice both go to stderr. */ + output: string + ghCalls: string[] + reruns: string[] +} + +/** + * Writes a fake `gh` that answers by argument shape and logs every invocation. + * It returns values ALREADY filtered, because the script passes `--jq` and we + * only care about what the script does with the answer. + */ +function runSweep(scenario: Scenario): SweepResult { + const dir = mkdtempSync(join(tmpdir(), 'sweep-')) + const log = join(dir, 'gh-calls.log') + const gh = join(dir, 'gh') + + writeFileSync( + gh, + `#!/usr/bin/env bash +ARGS="$*" +echo "$ARGS" >> ${JSON.stringify(log)} +case "$ARGS" in + *"/commits/main"*) echo "basesha000000" ;; + "run list"*) cat <<'JSON' +${JSON.stringify(scenario.baseCi)} +JSON + ;; + *"/jobs"*) echo ${JSON.stringify(scenario.failedSteps ?? '')} ;; + "run rerun"*) echo "rerun dispatched" ;; + "api repos/"*"/actions/runs/"*) echo ${JSON.stringify(String(scenario.runAttempt ?? 1))} ;; + "pr list"*) echo "[]" ;; + "workflow run"*) echo "dispatched" ;; + *) echo "UNHANDLED gh call: $ARGS" >&2; exit 1 ;; +esac +`, + { mode: 0o755 }, + ) + chmodSync(gh, 0o755) + + const proc = spawnSync('bash', [SWEEP], { + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + GH_REPO: 'maonakamoto/evig', + BASE_BRANCH: 'main', + }, + encoding: 'utf8', + }) + + const output = `${proc.stdout ?? ''}${proc.stderr ?? ''}` + + // The sweep must always exit 0 — it is a scheduled janitor, not a gate. A + // non-zero exit here means the fake gh hit an UNHANDLED call shape, which + // would silently make every assertion below vacuous. + expect({ status: proc.status, output }).toMatchObject({ status: 0 }) + + const ghCalls = existsSync(log) + ? readFileSync(log, 'utf8').split('\n').filter(Boolean) + : [] + + return { output, ghCalls, reruns: ghCalls.filter((c) => c.startsWith('run rerun')) } +} + +/** The base run is at the branch tip and completed — only the conclusion varies. */ +const baseCi = (conclusion: string) => ({ + status: 'completed', + conclusion, + headSha: 'basesha000000', + databaseId: 31119402753, +}) + +describe('auto-merge sweep — base branch guard', () => { + it('re-runs a CANCELLED base run instead of deadlocking behind it', () => { + const { reruns, output } = runSweep({ baseCi: baseCi('cancelled') }) + + // Nothing but a merge produces a new base CI run, and merges are what the + // guard blocks — so without this the queue can never recover on its own. + expect(reruns).toHaveLength(1) + expect(reruns[0]).toContain('31119402753') + expect(output).toContain('produced no verdict') + }) + + it('re-runs a base run that FAILED before executing any of our code', () => { + // The 2026-08-07 shape: conclusion=failure, but the only failed step is the + // runner refusing to start. That is not a verdict about the code. + const { reruns } = runSweep({ + baseCi: baseCi('failure'), + failedSteps: 'Set up job', + }) + + expect(reruns).toHaveLength(1) + }) + + it('refuses to merge onto a genuinely broken base, and does NOT re-run it', () => { + const { reruns, output } = runSweep({ + baseCi: baseCi('failure'), + failedSteps: 'Verify (lint + umlauts + typecheck + build)', + }) + + expect(reruns).toHaveLength(0) + expect(output).toContain('refusing to merge onto a broken base') + }) + + it('does not re-run a real failure even when the jobs API says nothing', () => { + // Empty means "we could not tell". Guessing "infra" here would re-run + // genuine failures forever. + const { reruns } = runSweep({ baseCi: baseCi('failure'), failedSteps: '' }) + + expect(reruns).toHaveLength(0) + }) + + it('stops retrying once the run hits the attempt cap', () => { + const { reruns, output } = runSweep({ + baseCi: baseCi('cancelled'), + runAttempt: 3, + }) + + expect(reruns).toHaveLength(0) + expect(output).toContain('not retrying again') + }) + + it('proceeds to the PR loop when the base is green', () => { + const { reruns, output } = runSweep({ baseCi: baseCi('success') }) + + expect(reruns).toHaveLength(0) + expect(output).toContain('no open PRs') + }) +})