From 4e1f42f3f838e14ebf83b84c2eb7ffe555a1dfaf Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 10:51:49 -0400 Subject: [PATCH 1/8] ci: wait for a newer run instead of failing on an earlier one's result The Required Merge Gate reads the newest check run of each required name on the commit. When CI runs again on the same commit (a re-run, a reopened pull request, a label), the gate starts within seconds, before the new workflows have created their own gate checks. It then found the previous run's result and, if that run had been cancelled or failed, failed at once, although the new run was still on its way. PR #229's gate failed this way three seconds after a reopen. Treat a check that concluded other than success before this gate run started as pending: it belongs to an earlier run, and a newer one may not have reported yet. The gate keeps polling and fails, with the reason named, only if no newer result arrives before its deadline. A failure after the gate run started still fails at once, and an earlier success still counts, as a label starts no new Core Checks. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/required-merge-gate.yml | 30 +++++++-- ci/checks/test_merge_gate_poller.py | 75 +++++++++++++++++++++++ 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index 2fc2fa372b..8f00cc279b 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -740,6 +740,12 @@ jobs: GATE_QUERY_BACKOFF_SECONDS="${GATE_QUERY_BACKOFF_SECONDS:-10}" GATE_QUERY_BACKOFF_CAP_SECONDS="${GATE_QUERY_BACKOFF_CAP_SECONDS:-60}" GATE_QUERY_TIMEOUT_SECONDS="${GATE_QUERY_TIMEOUT_SECONDS:-60}" + # When this gate run started, in the API's UTC format. A check that + # failed before then belongs to an earlier run on the same commit (a + # re-run, a reopen, a label); its newer run may not have created its + # check yet, so it is waited on rather than trusted. The test suite + # sets it. + GATE_RUN_STARTED_AT="${GATE_RUN_STARTED_AT:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" for gate_tunable in \ GATE_WAIT_TIMEOUT_SECONDS \ @@ -774,7 +780,7 @@ jobs: | map(select(.name == $name)) | sort_by(.started_at // .created_at // "") | last - | if . == null then "absent" else ["found", (.status // ""), (.conclusion // "")] | @tsv end + | if . == null then "absent" else ["found", (.status // ""), (.conclusion // ""), (.completed_at // "")] | @tsv end ' summarize_stderr() { @@ -784,7 +790,7 @@ jobs: # Read the newest check run named $1 on ${SHA}. # # On success sets QUERY_OBSERVATION to "absent" or to - # "foundstatusconclusion" and returns 0. Returns 1 when the + # "foundstatusconclusioncompleted_at" and returns 0. Returns 1 when the # query could not be read within GATE_QUERY_MAX_ATTEMPTS attempts or # before $2, the wall-clock deadline shared with the caller: an # unreadable query is never success and never "not found yet". @@ -873,11 +879,14 @@ jobs: # Returns 1 under three distinct error titles: a check concluded # something other than success, the check runs could not be read, or # the shared deadline passed with checks still pending. Only a - # completed check with a success conclusion is success. + # completed check with a success conclusion is success. A check that + # concluded otherwise before this gate run started is still pending: + # it is the previous run on this commit, and its newer run may not + # have reported yet. wait_for_checks() { local check_names=("$@") local pending=() - local check_name conclusion fields now observation status deadline + local check_name completed_at conclusion fields now observation status deadline rest now="$(date +%s)" deadline=$(( now + GATE_WAIT_TIMEOUT_SECONDS )) @@ -900,11 +909,22 @@ jobs: fields="${observation#found$'\t'}" status="${fields%%$'\t'*}" - conclusion="${fields#*$'\t'}" + rest="${fields#*$'\t'}" + conclusion="${rest%%$'\t'*}" + completed_at="" + if [[ "${rest}" == *$'\t'* ]]; then + completed_at="${rest#*$'\t'}" + fi echo "${check_name}: status=${status} conclusion=${conclusion:-pending}" if [[ "${status}" == "completed" ]]; then if [[ "${conclusion}" != "success" ]]; then + # ISO 8601 UTC timestamps of the same format order as strings. + if [[ -n "${completed_at}" && "${completed_at}" < "${GATE_RUN_STARTED_AT}" ]]; then + echo "${check_name}: concluded ${conclusion:-unknown} at ${completed_at}, before this gate run started at ${GATE_RUN_STARTED_AT}; waiting for a newer run" + pending+=("${check_name} (last ${conclusion:-unknown} before this gate run)") + continue + fi echo "::error title=Required Merge Gate failed::${check_name} concluded ${conclusion:-unknown}." return 1 fi diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index 4ea097df16..55d59e057d 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -474,6 +474,81 @@ def test_the_newest_check_run_wins_when_a_check_was_rerun(self) -> None: f"{ERROR_CHECK_FAILED}rpc-docs concluded failure.", ) + # -- results from an earlier run on the same commit --------------------- + + GATE_START = "2026-10-06T14:12:00Z" + BEFORE = "2026-10-05T20:47:00Z" + AFTER = "2026-10-06T14:40:00Z" + + def test_a_failure_from_before_this_gate_run_is_waited_on_not_trusted(self) -> None: + # A reopen, label or re-run starts this gate seconds after the new + # workflows, before their own gate checks exist. The newest check of + # that name is then the previous run's, which says nothing about the + # run under way. + for conclusion in ("cancelled", "failure", "timed_out"): + with self.subTest(conclusion=conclusion): + fake = FakeGh( + self.tmp(f"stale-{conclusion}"), + default=ok(body(page( + check_run("Core Checks Gate", "completed", conclusion, started_at=self.BEFORE, completed_at=self.BEFORE), + check_run("Full Validation Gate", "completed", "success", started_at=self.BEFORE, completed_at=self.BEFORE), + ))), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3", GATE_RUN_STARTED_AT=self.GATE_START) + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: " + f"Core Checks Gate (last {conclusion} before this gate run)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNREADABLE), + ) + self.assertIn("waiting for a newer run", completed.stdout) + self.assertGreater(len(fake.calls), 1, "a stale failure must be polled again, not judged once") + + def test_the_newer_run_replaces_a_stale_failure(self) -> None: + stale = check_run("Core Checks Gate", "completed", "cancelled", started_at=self.BEFORE, completed_at=self.BEFORE) + validation = check_run("Full Validation Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER) + fresh_running = check_run("Core Checks Gate", "in_progress", None, started_at=self.AFTER) + fresh_green = check_run("Core Checks Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER) + fake = FakeGh( + self.tmp("stale-then-green"), + responses=[ + ok(body(page(stale, validation))), + ok(body(page(stale, validation))), + ok(body(page(stale, fresh_running, validation))), + ok(body(page(stale, fresh_running, validation))), + ], + default=ok(body(page(stale, fresh_green, validation))), + ) + completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn("Core Checks Gate: status=completed conclusion=success", completed.stdout) + + def test_a_failure_after_this_gate_run_started_still_fails_at_once(self) -> None: + fake = FakeGh( + self.tmp("fresh-failure"), + default=ok(body(page( + check_run("Core Checks Gate", "completed", "failure", started_at=self.AFTER, completed_at=self.AFTER), + check_run("Full Validation Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER), + ))), + ) + completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) + self.assertGateFailed(completed, f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT,)) + self.assertEqual(len(fake.calls), 1, fake.calls) + + def test_a_success_from_before_this_gate_run_still_counts(self) -> None: + # A label starts the gate and Full Validation but not Core Checks, whose + # earlier green result on the same commit is still the right answer. + fake = FakeGh( + self.tmp("stale-success"), + default=ok(body(page( + check_run("Core Checks Gate", "completed", "success", started_at=self.BEFORE, completed_at=self.BEFORE), + check_run("Full Validation Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER), + ))), + ) + completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + def test_a_queued_rerun_falls_back_to_created_at_and_is_pending(self) -> None: # A re-run that has not started yet has no started_at at all; ordering # by created_at keeps it, so an old green run cannot pass the gate. From a4a5befe15c24d4696178a18407159ddb15fb753 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 14:13:45 -0400 Subject: [PATCH 2/8] ci: date the merge gate from its workflow run, not its step The gate step starts only after the jobs it needs, so a check from the current run can fail before the step's own start time and be taken for an earlier run's result, making the gate wait out its deadline instead of failing at once. Read this run attempt's run_started_at from the API instead, and only when checks are polled, so profiles that poll nothing still make no API calls. If it can't be read, fall back to the step's start with a warning. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/required-merge-gate.yml | 28 ++++++++++++++---- ci/checks/test_merge_gate_poller.py | 35 +++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index 8f00cc279b..fc7e1a9515 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -711,6 +711,8 @@ jobs: TOUCHED_OPERATOR_KEYS: ${{ needs.classify-changes.outputs.touched_operator_keys }} REPOSITORY: ${{ github.repository }} SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} run: | set -euo pipefail @@ -740,12 +742,6 @@ jobs: GATE_QUERY_BACKOFF_SECONDS="${GATE_QUERY_BACKOFF_SECONDS:-10}" GATE_QUERY_BACKOFF_CAP_SECONDS="${GATE_QUERY_BACKOFF_CAP_SECONDS:-60}" GATE_QUERY_TIMEOUT_SECONDS="${GATE_QUERY_TIMEOUT_SECONDS:-60}" - # When this gate run started, in the API's UTC format. A check that - # failed before then belongs to an earlier run on the same commit (a - # re-run, a reopen, a label); its newer run may not have created its - # check yet, so it is waited on rather than trusted. The test suite - # sets it. - GATE_RUN_STARTED_AT="${GATE_RUN_STARTED_AT:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" for gate_tunable in \ GATE_WAIT_TIMEOUT_SECONDS \ @@ -874,6 +870,25 @@ jobs: return 1 } + # When this gate run (this attempt of the workflow run) started, in + # the API's UTC format. A check that failed before then belongs to an + # earlier run on the same commit (a re-run, a reopen, a label); its + # newer run may not have created its check yet, so it is waited on + # rather than trusted. It is the workflow run's start, not this step's: + # this step starts after the jobs it needs, so a current failure can + # complete before it. Read only when checks are polled; the test suite + # sets it. + resolve_gate_run_started_at() { + if [[ -z "${GATE_RUN_STARTED_AT:-}" ]]; then + GATE_RUN_STARTED_AT="$(gh api "/repos/${REPOSITORY}/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}" 2>/dev/null | jq -r '.run_started_at // empty' 2>/dev/null || true)" + if [[ ! "${GATE_RUN_STARTED_AT}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then + GATE_RUN_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "::warning title=Required Merge Gate start time unknown::Could not read this workflow run's start time; using the step's start time (${GATE_RUN_STARTED_AT}), so a failure that completed before this step may be waited on instead of failing at once." + fi + fi + echo "Gate run started at ${GATE_RUN_STARTED_AT}" + } + # Wait until every named check on ${SHA} has concluded successfully. # # Returns 1 under three distinct error titles: a check concluded @@ -888,6 +903,7 @@ jobs: local pending=() local check_name completed_at conclusion fields now observation status deadline rest + resolve_gate_run_started_at now="$(date +%s)" deadline=$(( now + GATE_WAIT_TIMEOUT_SECONDS )) diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index 55d59e057d..18b6cd78ae 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -216,6 +216,10 @@ def gate_env(self, fake: FakeGh, profile: str, **overrides: str) -> dict[str, st "GH_TOKEN": "fake-token", "REPOSITORY": "example/repo", "SHA": "0" * 40, + "RUN_ID": "12345", + "RUN_ATTEMPT": "1", + # Tests that model the start-time lookup clear this. + "GATE_RUN_STARTED_AT": "2026-10-06T14:12:00Z", } declared = set(self.step["env"]) self.assertFalse( @@ -549,6 +553,37 @@ def test_a_success_from_before_this_gate_run_still_counts(self) -> None: completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + def test_the_gate_start_is_the_workflow_runs_not_the_steps(self) -> None: + # The step starts after the jobs it needs, so a failure from this run can + # complete before it; the run's own start time keeps it a current failure. + attempt = ok(json.dumps({"run_started_at": "2026-10-06T14:00:00Z"})) + fake = FakeGh( + self.tmp("run-start"), + responses=[attempt], + default=ok(body(page( + check_run("Core Checks Gate", "completed", "failure", started_at="2026-10-06T14:01:00Z", completed_at="2026-10-06T14:05:00Z"), + check_run("Full Validation Gate", "completed", "success", started_at="2026-10-06T14:01:00Z", completed_at="2026-10-06T14:05:00Z"), + ))), + ) + completed = self.run_gate(fake, GATE_RUN_STARTED_AT="") + self.assertGateFailed(completed, f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT,)) + self.assertIn("Gate run started at 2026-10-06T14:00:00Z", completed.stdout) + self.assertIn("/actions/runs/12345/attempts/1", " ".join(fake.calls)) + + def test_an_unreadable_run_start_falls_back_with_a_warning(self) -> None: + fake = FakeGh( + self.tmp("run-start-unreadable"), + responses=[ok("not json")], + default=ok(body(page( + check_run("Core Checks Gate", "completed", "success", started_at="1"), + check_run("Full Validation Gate", "completed", "success", started_at="1"), + ))), + ) + completed = self.run_gate(fake, GATE_RUN_STARTED_AT="") + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn("Required Merge Gate start time unknown", completed.stdout + completed.stderr) + def test_a_queued_rerun_falls_back_to_created_at_and_is_pending(self) -> None: # A re-run that has not started yet has no started_at at all; ordering # by created_at keeps it, so an old green run cannot pass the gate. From 2dff5bcea7b402d6993e9094eb24c582adca917b Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 14:35:58 -0400 Subject: [PATCH 3/8] ci: never take a cancelled check as the merge gate's verdict A newer push, reopen or label cancels the running Full Validation (cancel-in-progress), and that cancellation can complete after this gate run started, so the run-start cutoff alone took it for a current failure. A cancelled check means superseded or never run, so treat it as pending and wait for the run that replaces it; the gate still fails at its deadline if none arrives. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/required-merge-gate.yml | 9 ++++++++ ci/checks/test_merge_gate_poller.py | 25 +++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index fc7e1a9515..733e6145fa 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -934,6 +934,15 @@ jobs: echo "${check_name}: status=${status} conclusion=${conclusion:-pending}" if [[ "${status}" == "completed" ]]; then + if [[ "${conclusion}" == "cancelled" ]]; then + # A cancelled check is never a verdict: it was superseded (a + # newer push, reopen or label cancels the running workflow, + # possibly after this gate started) or it never ran. Wait for + # the run that replaces it; the deadline still fails the gate. + echo "${check_name}: cancelled; waiting for the run that replaces it" + pending+=("${check_name} (cancelled)") + continue + fi if [[ "${conclusion}" != "success" ]]; then # ISO 8601 UTC timestamps of the same format order as strings. if [[ -n "${completed_at}" && "${completed_at}" < "${GATE_RUN_STARTED_AT}" ]]; then diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index 18b6cd78ae..0b72d51804 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -306,7 +306,7 @@ def test_completed_success_on_every_required_check_passes(self) -> None: self.assertIn("Full Validation Gate: status=completed conclusion=success", completed.stdout) def test_failed_conclusion_fails_as_a_failed_check_not_as_pending(self) -> None: - for conclusion in ("failure", "cancelled", "timed_out", "action_required", "neutral", "skipped"): + for conclusion in ("failure", "timed_out", "action_required", "neutral", "skipped"): with self.subTest(conclusion=conclusion): fake = FakeGh( self.tmp(f"concluded-{conclusion}"), @@ -489,7 +489,7 @@ def test_a_failure_from_before_this_gate_run_is_waited_on_not_trusted(self) -> N # workflows, before their own gate checks exist. The newest check of # that name is then the previous run's, which says nothing about the # run under way. - for conclusion in ("cancelled", "failure", "timed_out"): + for conclusion in ("failure", "timed_out"): with self.subTest(conclusion=conclusion): fake = FakeGh( self.tmp(f"stale-{conclusion}"), @@ -553,6 +553,27 @@ def test_a_success_from_before_this_gate_run_still_counts(self) -> None: completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + def test_a_cancelled_check_is_never_a_verdict(self) -> None: + # A newer push or label cancels the running Full Validation, and that + # cancellation can complete after this gate run started. + cancelled = check_run("Full Validation Gate", "completed", "cancelled", started_at=self.BEFORE, completed_at=self.AFTER) + core = check_run("Core Checks Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER) + fake = FakeGh(self.tmp("cancelled-alone"), default=ok(body(page(core, cancelled)))) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3", GATE_RUN_STARTED_AT=self.GATE_START) + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: Full Validation Gate (cancelled)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNREADABLE), + ) + newer_green = check_run("Full Validation Gate", "completed", "success", started_at=self.AFTER, completed_at="2026-10-06T15:00:00Z") + fake = FakeGh( + self.tmp("cancelled-then-green"), + responses=[ok(body(page(core, cancelled))), ok(body(page(core, cancelled)))], + default=ok(body(page(core, cancelled, newer_green))), + ) + completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + def test_the_gate_start_is_the_workflow_runs_not_the_steps(self) -> None: # The step starts after the jobs it needs, so a failure from this run can # complete before it; the run's own start time keeps it a current failure. From 164533a5debb66e74c7e67ff4f6736130db9afa1 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 15:01:22 -0400 Subject: [PATCH 4/8] ci: let the merge gate read its run attempt's start time Getting a workflow run attempt needs Actions: read, and the workflow grants only contents and checks, so the start-time lookup always fell back to the step's own start. Grant actions: read, and pin it in the poller contract test. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/required-merge-gate.yml | 2 ++ ci/checks/test_merge_gate_poller.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index 733e6145fa..4c85ef22d9 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -37,6 +37,8 @@ on: permissions: contents: read checks: read + # The gate reads its own run attempt's start time (run_started_at). + actions: read concurrency: group: required-merge-gate-${{ github.event_name != 'pull_request' && github.run_id || github.ref }} diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index 0b72d51804..bcffee2f8c 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -574,6 +574,13 @@ def test_a_cancelled_check_is_never_a_verdict(self) -> None: completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + def test_the_workflow_token_can_read_its_run_start(self) -> None: + # Get a workflow run attempt needs Actions: read; without it the lookup + # always falls back to the step's start time. + permissions = self.workflow["permissions"] + self.assertEqual(permissions.get("actions"), "read", permissions) + self.assertEqual(permissions.get("checks"), "read", permissions) + def test_the_gate_start_is_the_workflow_runs_not_the_steps(self) -> None: # The step starts after the jobs it needs, so a failure from this run can # complete before it; the run's own start time keeps it a current failure. From 7d6f83909eea9c4604184916e83b062ff94b55a5 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 15:51:16 -0400 Subject: [PATCH 5/8] ci: start Core Checks on every event that starts the merge gate The gate now waits for a newer Core Checks Gate instead of trusting an earlier failure, but Core Checks ran only on opened, synchronize and reopened, so a label, unlabel or ready-for-review event left the gate waiting out its deadline for a run that could not start. Give Core Checks the same pull_request activity types as the gate and Full Validation, and pin that in the poller contract test. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/core-checks.yml | 9 +++++++++ ci/checks/test_merge_gate_poller.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/core-checks.yml b/.github/workflows/core-checks.yml index 30111a8636..a843f7c01c 100644 --- a/.github/workflows/core-checks.yml +++ b/.github/workflows/core-checks.yml @@ -6,6 +6,15 @@ name: Core Checks on: pull_request: + # The same activity types as the Required Merge Gate, which waits for a + # newer Core Checks Gate whenever it reruns on a commit. + types: + - opened + - synchronize + - reopened + - ready_for_review + - labeled + - unlabeled branches: - main - 1.0.0 diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index bcffee2f8c..506524b1e8 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -169,6 +169,12 @@ def calls(self) -> list[str]: return self.calls_path.read_text(encoding="utf8").splitlines() +def load_yaml(path: Path) -> dict: + import yaml + from test_classifier_ci_contract import WorkflowLoader # noqa: E402 keeps "on" a string key + return yaml.load(path.read_text(encoding="utf8"), Loader=WorkflowLoader) + + class MergeGatePollerTest(unittest.TestCase): maxDiff = None @@ -574,6 +580,15 @@ def test_a_cancelled_check_is_never_a_verdict(self) -> None: completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + def test_every_gate_event_can_start_the_checks_it_waits_for(self) -> None: + # The gate waits for a newer run instead of trusting an earlier failure, + # so each event that starts the gate must also start the gated workflows. + gate_types = set(self.workflow["on"]["pull_request"]["types"]) + for name in ("core-checks.yml", "ci.yml"): + other = load_yaml(REPO_ROOT / ".github" / "workflows" / name) + types = set(other["on"]["pull_request"].get("types", ["opened", "synchronize", "reopened"])) + self.assertEqual(gate_types - types, set(), f"{name} misses events that start the Required Merge Gate") + def test_the_workflow_token_can_read_its_run_start(self) -> None: # Get a workflow run attempt needs Actions: read; without it the lookup # always falls back to the step's start time. From 8fc28a8fa86c114c137ae4eb85cce7a78e6c7824 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 16:48:16 -0400 Subject: [PATCH 6/8] ci: tell the merge gate's checks apart by the event that started them The gate waited on a check that failed before its own run started, taking it for an earlier run's result. A time cutoff cannot tell the two apart: Core Checks, Full Validation and the gate are separate workflows, so this event's check can fail before the gate's run starts, an earlier event's can fail or be cancelled after it, and re-running the gate alone moves its start but starts nothing else. Date each failed check by its event instead. Every workflow run an event starts is created at that event, and a re-run keeps the run, its check suite and its created_at. The gate maps a failed check's check suite to its workflow run (GET /actions/runs?check_suite_id=) and compares that run's created_at with its own run's (GET /actions/runs/{run_id}), allowing 60 seconds between one event's runs. - An earlier event's failure is waited on until a newer run reports. - A failure from this event or a later one fails at once, including after a gate-only re-run. - A success counts whichever event produced it, and a cancellation is never a verdict; neither needs a lookup. - If either run can't be read after the usual retries, the gate fails closed under its own error title. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/core-checks.yml | 4 +- .github/workflows/required-merge-gate.yml | 252 ++++++--- ci/checks/test_merge_gate_poller.py | 638 +++++++++++++++++----- 3 files changed, 686 insertions(+), 208 deletions(-) diff --git a/.github/workflows/core-checks.yml b/.github/workflows/core-checks.yml index a843f7c01c..8a429e6241 100644 --- a/.github/workflows/core-checks.yml +++ b/.github/workflows/core-checks.yml @@ -6,8 +6,8 @@ name: Core Checks on: pull_request: - # The same activity types as the Required Merge Gate, which waits for a - # newer Core Checks Gate whenever it reruns on a commit. + # The same activity types as the Required Merge Gate, which waits for + # its own event's Core Checks Gate instead of an earlier event's failure. types: - opened - synchronize diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index 4c85ef22d9..d2832794ed 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -37,7 +37,8 @@ on: permissions: contents: read checks: read - # The gate reads its own run attempt's start time (run_started_at). + # The gate reads workflow runs, its own and those behind the checks it + # waits for, to tell which event produced a failed check. actions: read concurrency: @@ -714,7 +715,6 @@ jobs: REPOSITORY: ${{ github.repository }} SHA: ${{ github.event.pull_request.head.sha || github.sha }} RUN_ID: ${{ github.run_id }} - RUN_ATTEMPT: ${{ github.run_attempt }} run: | set -euo pipefail @@ -768,7 +768,10 @@ jobs: # started_at (created_at while a run has not started) wins; # * "absent" means the query succeeded and reported no run with that # name, which is a different fact from a query that could not be - # read at all. + # read at all; + # * the check suite travels with the result: it names the workflow + # run, and so the event, that produced the check (see + # date_check_event below). # $name is a jq variable bound by --arg below, not a shell expansion. # shellcheck disable=SC2016 check_run_filter=' @@ -778,32 +781,40 @@ jobs: | map(select(.name == $name)) | sort_by(.started_at // .created_at // "") | last - | if . == null then "absent" else ["found", (.status // ""), (.conclusion // ""), (.completed_at // "")] | @tsv end + | if . == null then "absent" else ["found", (.status // ""), (.conclusion // ""), (((.check_suite // {}).id // "") | tostring)] | @tsv end ' + check_run_pattern=$'^(absent|found\t[a-z_]+\t[^\t]*\t[^\t]*)$' summarize_stderr() { tr '\n' ' ' < "$1" | cut -c1-300 } - # Read the newest check run named $1 on ${SHA}. + # GET the API path $4 with `gh api --paginate`, pass the slurped + # pages through the jq program $6 (arguments from $7 on go to jq), + # and trust the result only if it matches the bash regex $5. $1 + # names the check for the log and $2 what is read. # - # On success sets QUERY_OBSERVATION to "absent" or to - # "foundstatusconclusioncompleted_at" and returns 0. Returns 1 when the - # query could not be read within GATE_QUERY_MAX_ATTEMPTS attempts or - # before $2, the wall-clock deadline shared with the caller: an - # unreadable query is never success and never "not found yet". + # On success sets API_RESULT and returns 0. Returns 1 when the path + # could not be read within GATE_QUERY_MAX_ATTEMPTS attempts or + # before $3, the wall-clock deadline shared with the caller: an + # unreadable response is never success and never "not found yet". # # The result travels in a variable rather than on stdout because this # function also logs there, and errexit is not in force for a function # called from a condition, so every failure is handled explicitly. - query_check_run() { + read_api() { local check_name="$1" - local deadline="$2" + local what="$2" + local deadline="$3" + local path="$4" + local pattern="$5" + local filter="$6" + shift 6 local attempt=1 local backoff="${GATE_QUERY_BACKOFF_SECONDS}" - local body fields observation reason status stderr_path now rc + local body reason result stderr_path now rc - QUERY_OBSERVATION="" + API_RESULT="" while (( attempt <= GATE_QUERY_MAX_ATTEMPTS )); do stderr_path="$(mktemp)" @@ -812,39 +823,27 @@ jobs: timeout "${GATE_QUERY_TIMEOUT_SECONDS}" gh api \ -H "Accept: application/vnd.github+json" \ --paginate \ - "/repos/${REPOSITORY}/commits/${SHA}/check-runs?per_page=100" \ + "${path}" \ 2>"${stderr_path}" )" || rc=$? if (( rc != 0 )); then - reason="check-runs query failed (exit ${rc}): $(summarize_stderr "${stderr_path}")" + reason="${what} query failed (exit ${rc}): $(summarize_stderr "${stderr_path}")" else rc=0 - observation="$( + result="$( printf '%s' "${body}" \ - | jq -e -r -s --arg name "${check_name}" "${check_run_filter}" 2>"${stderr_path}" + | jq -e -r -s "$@" "${filter}" 2>"${stderr_path}" )" || rc=$? if (( rc != 0 )); then - reason="check-runs response could not be read (jq exit ${rc}): $(summarize_stderr "${stderr_path}")" + reason="${what} response could not be read (jq exit ${rc}): $(summarize_stderr "${stderr_path}")" + elif [[ "${result}" =~ ${pattern} ]]; then + rm -f "${stderr_path}" + API_RESULT="${result}" + return 0 else - case "${observation}" in - absent) - rm -f "${stderr_path}" - QUERY_OBSERVATION="${observation}" - return 0 - ;; - found$'\t'*) - fields="${observation#found$'\t'}" - status="${fields%%$'\t'*}" - if [[ "${status}" =~ ^[a-z_]+$ ]]; then - rm -f "${stderr_path}" - QUERY_OBSERVATION="${observation}" - return 0 - fi - ;; - esac - reason="check-runs response was malformed: ${observation}" + reason="${what} response was malformed: ${result}" fi fi @@ -858,7 +857,7 @@ jobs: now="$(date +%s)" if (( now + backoff > deadline )); then - echo "${check_name}: wait deadline reached while the check-runs query was unreadable" + echo "${check_name}: wait deadline reached while the ${what} query was unreadable" break fi @@ -872,40 +871,159 @@ jobs: return 1 } - # When this gate run (this attempt of the workflow run) started, in - # the API's UTC format. A check that failed before then belongs to an - # earlier run on the same commit (a re-run, a reopen, a label); its - # newer run may not have created its check yet, so it is waited on - # rather than trusted. It is the workflow run's start, not this step's: - # this step starts after the jobs it needs, so a current failure can - # complete before it. Read only when checks are polled; the test suite - # sets it. - resolve_gate_run_started_at() { - if [[ -z "${GATE_RUN_STARTED_AT:-}" ]]; then - GATE_RUN_STARTED_AT="$(gh api "/repos/${REPOSITORY}/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}" 2>/dev/null | jq -r '.run_started_at // empty' 2>/dev/null || true)" - if [[ ! "${GATE_RUN_STARTED_AT}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then - GATE_RUN_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "::warning title=Required Merge Gate start time unknown::Could not read this workflow run's start time; using the step's start time (${GATE_RUN_STARTED_AT}), so a failure that completed before this step may be waited on instead of failing at once." + # Read the newest check run named $1 on ${SHA}. + # + # On success sets QUERY_OBSERVATION to "absent" or to + # "foundstatusconclusioncheck suite id" and returns 0. + # Returns 1 when the query could not be read (see read_api; $2 is + # the deadline). + query_check_run() { + QUERY_OBSERVATION="" + read_api "$1" "check-runs" "$2" \ + "/repos/${REPOSITORY}/commits/${SHA}/check-runs?per_page=100" \ + "${check_run_pattern}" "${check_run_filter}" --arg name "$1" || return 1 + QUERY_OBSERVATION="${API_RESULT}" + } + + # Which event a check belongs to. + # + # When CI runs again on a commit (a reopen, a label, a re-run), this + # gate starts within seconds, but Core Checks and Full Validation + # create their gate checks only at their last job, minutes later. + # Until then the newest check of that name is an earlier event's, + # and its failure says nothing about the run under way. A cutoff + # in time cannot tell that apart from a failure of this event: + # Core Checks, Full Validation and this gate are separate workflows, + # scheduled independently, so a check this event started can fail + # before this gate's run even starts; an earlier event's check can be + # cancelled, or fail, after it started; and re-running this gate + # alone starts it again but starts nothing else. + # + # What the checks of one event share is the event. Every workflow + # run a push, pull request event or merge group starts is created at + # that event, with the same created_at to within a second or two, + # and a re-run keeps the workflow run: its id, its check suite and + # its created_at stay, and only run_attempt and run_started_at move + # (GitHub's REST API, checked on this repository's PR #229). So: + # + # * a check run's check suite names the workflow run that produced + # it (GET /actions/runs?check_suite_id=), whose created_at dates + # the event that started it; once read, a check suite's date is + # cached, as it cannot change; + # * this gate's own workflow run's created_at (GET + # /actions/runs/{run_id}, never the attempt's) dates the event it + # judges, so a gate-only re-run still judges its original event; + # * a check whose workflow run was created more than + # EVENT_SKEW_SECONDS before this gate's belongs to an earlier event; + # any other belongs to this event or a later one and is current. + # + # Only a check that concluded neither success nor cancelled is + # dated: a success on this commit passes whichever event produced + # it, and a cancellation is never a verdict. + # + # Workflow runs started by one event are created this close together. + # A documented allowance, not a tunable: a larger value takes more of + # an earlier event's failures as current (they fail at once rather + # than wait), a smaller one risks taking a current failure as earlier + # (it is waited on until the deadline). + EVENT_SKEW_SECONDS=60 + + # GET /actions/runs/{run_id}: this gate's own workflow run. + # $run_id is bound by --argjson below. + # shellcheck disable=SC2016 + gate_run_filter=' + if type != "array" or length != 1 then error("expected one workflow run") else .[0] end + | if .id != $run_id then error("not workflow run \($run_id)") else . end + | .created_at + | [(fromdateiso8601 | tostring), .] + | @tsv + ' + # GET /actions/runs?check_suite_id={id}: the one workflow run that + # owns a check suite. A run of another suite is never taken for it, + # even if the filter were ignored. $suite is bound by --argjson. + # shellcheck disable=SC2016 + suite_run_filter=' + if type != "array" or length == 0 then error("empty workflow-runs response") else . end + | map(if (.workflow_runs | type) == "array" then .workflow_runs[] else error("malformed workflow-runs page") end) + | map(select(.check_suite_id == $suite)) + | if length != 1 then error("\(length) workflow runs own check suite \($suite)") else .[0] end + | .created_at + | [(fromdateiso8601 | tostring), .] + | @tsv + ' + # "" + created_at_pattern=$'^[0-9]+\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + + GATE_RUN_CREATED="" + declare -A SUITE_RUN_CREATED=() + + # Date the event of check $1, from check suite $2 (see above). + # + # Sets CHECK_EVENT to "earlier" or "current" and CHECK_RUN_CREATED_AT + # to its workflow run's created_at, and returns 0. Returns 1 when the + # check run names no check suite, or this gate's run or the check's + # run could not be read by $3, the shared deadline: an event that + # cannot be dated is never success, never earlier and never current. + date_check_event() { + local check_name="$1" + local check_suite_id="$2" + local deadline="$3" + local check_created gate_created + + CHECK_EVENT="" + CHECK_RUN_CREATED_AT="" + + if [[ -z "${GATE_RUN_CREATED}" ]]; then + if [[ ! "${RUN_ID}" =~ ^[1-9][0-9]*$ ]]; then + echo "${check_name}: this gate's workflow run id is not a number: '${RUN_ID}'" + return 1 fi + read_api "${check_name}" "workflow-run" "${deadline}" \ + "/repos/${REPOSITORY}/actions/runs/${RUN_ID}" \ + "${created_at_pattern}" "${gate_run_filter}" --argjson run_id "${RUN_ID}" || return 1 + GATE_RUN_CREATED="${API_RESULT}" + echo "This gate's workflow run was created at ${GATE_RUN_CREATED#*$'\t'}, by the event it judges" + fi + + if [[ ! "${check_suite_id}" =~ ^[1-9][0-9]*$ ]]; then + echo "${check_name}: the check run names no check suite: '${check_suite_id}'" + return 1 + fi + if [[ -z "${SUITE_RUN_CREATED[${check_suite_id}]:-}" ]]; then + read_api "${check_name}" "workflow-run" "${deadline}" \ + "/repos/${REPOSITORY}/actions/runs?check_suite_id=${check_suite_id}&exclude_pull_requests=true&per_page=100" \ + "${created_at_pattern}" "${suite_run_filter}" --argjson suite "${check_suite_id}" || return 1 + SUITE_RUN_CREATED[${check_suite_id}]="${API_RESULT}" + fi + + check_created="${SUITE_RUN_CREATED[${check_suite_id}]}" + gate_created="${GATE_RUN_CREATED}" + CHECK_RUN_CREATED_AT="${check_created#*$'\t'}" + check_created="${check_created%%$'\t'*}" + gate_created="${gate_created%%$'\t'*}" + if (( check_created < gate_created - EVENT_SKEW_SECONDS )); then + CHECK_EVENT="earlier" + else + CHECK_EVENT="current" fi - echo "Gate run started at ${GATE_RUN_STARTED_AT}" } # Wait until every named check on ${SHA} has concluded successfully. # - # Returns 1 under three distinct error titles: a check concluded - # something other than success, the check runs could not be read, or + # Returns 1 under four distinct error titles: a check concluded + # something other than success, the check runs could not be read, a + # check that concluded otherwise could not be traced to its event, or # the shared deadline passed with checks still pending. Only a # completed check with a success conclusion is success. A check that - # concluded otherwise before this gate run started is still pending: - # it is the previous run on this commit, and its newer run may not - # have reported yet. + # concluded otherwise is a verdict only if it belongs to the event + # this gate judges, or a later one; an earlier event's is waited on, + # as its newer run may not have reported yet, and a cancelled one is + # always waited on. wait_for_checks() { local check_names=("$@") local pending=() - local check_name completed_at conclusion fields now observation status deadline rest + local check_name check_suite_id conclusion fields now observation status deadline rest - resolve_gate_run_started_at now="$(date +%s)" deadline=$(( now + GATE_WAIT_TIMEOUT_SECONDS )) @@ -929,10 +1047,7 @@ jobs: status="${fields%%$'\t'*}" rest="${fields#*$'\t'}" conclusion="${rest%%$'\t'*}" - completed_at="" - if [[ "${rest}" == *$'\t'* ]]; then - completed_at="${rest#*$'\t'}" - fi + check_suite_id="${rest#*$'\t'}" echo "${check_name}: status=${status} conclusion=${conclusion:-pending}" if [[ "${status}" == "completed" ]]; then @@ -946,10 +1061,13 @@ jobs: continue fi if [[ "${conclusion}" != "success" ]]; then - # ISO 8601 UTC timestamps of the same format order as strings. - if [[ -n "${completed_at}" && "${completed_at}" < "${GATE_RUN_STARTED_AT}" ]]; then - echo "${check_name}: concluded ${conclusion:-unknown} at ${completed_at}, before this gate run started at ${GATE_RUN_STARTED_AT}; waiting for a newer run" - pending+=("${check_name} (last ${conclusion:-unknown} before this gate run)") + if ! date_check_event "${check_name}" "${check_suite_id}" "${deadline}"; then + echo "::error title=Required Merge Gate could not correlate check runs::${check_name} on ${SHA} concluded ${conclusion:-unknown}, but the workflow runs that tell whether it belongs to this event could not be read; failing closed with no verdict for: ${check_names[*]}" + return 1 + fi + if [[ "${CHECK_EVENT}" == "earlier" ]]; then + echo "${check_name}: concluded ${conclusion:-unknown} in a workflow run created at ${CHECK_RUN_CREATED_AT}, by an earlier event than this gate's (${GATE_RUN_CREATED#*$'\t'}); waiting for a newer run" + pending+=("${check_name} (last ${conclusion:-unknown} from an earlier event)") continue fi echo "::error title=Required Merge Gate failed::${check_name} concluded ${conclusion:-unknown}." diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index 506524b1e8..c0a880daf8 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -17,9 +17,16 @@ test, so the production defaults stay the values CI actually uses while the suite still runs in seconds. +The stub also answers the workflow-run lookups the gate makes to tell which +event produced a failed check (its own run, and the run that owns each failed +check's check suite), so those are exercised by the same real script. + Not provable here: whether the hosted ``Required Merge Gate`` check is attached to a branch ruleset, and whether the real API agrees with the shapes -modelled below. +modelled below. The workflow-run shapes and their re-run semantics (a re-run +keeps the run's ``check_suite_id`` and ``created_at`` and moves only +``run_attempt`` and ``run_started_at``) were read from GitHub's REST API +description and checked against this repository's PR #229 on 2026-10-06. """ from __future__ import annotations @@ -31,6 +38,7 @@ import tempfile import time import unittest +from datetime import datetime, timedelta, timezone from pathlib import Path CHECKS_DIR = Path(__file__).resolve().parent @@ -85,16 +93,90 @@ ERROR_CHECK_FAILED = "::error title=Required Merge Gate failed::" ERROR_UNREADABLE = "::error title=Required Merge Gate could not read check runs::" +ERROR_UNCORRELATED = "::error title=Required Merge Gate could not correlate check runs::" ERROR_TIMED_OUT = "::error title=Required Merge Gate timed out::" WARNING_RETRYING = "::warning title=Required Merge Gate retrying::" +# The documented allowance between the workflow runs one event creates. +EVENT_SKEW = re.compile(r"^\s*EVENT_SKEW_SECONDS=(?P\d+)\s*$", re.MULTILINE) +DOCUMENTED_EVENT_SKEW_SECONDS = 60 + +# -- the events a commit has seen ------------------------------------------- +# +# Every workflow run an event starts is created at that event, and a re-run +# keeps the run's created_at. The gate's own run (RUN_ID) was created by +# THIS_EVENT unless a test says otherwise. Check suites, one per workflow run, +# are numbered by event: 1xx earlier, 2xx this one, 3xx a later one. +GATE_RUN_ID = 12345 +EARLIER_EVENT = "2026-10-05T20:17:51Z" +THIS_EVENT = "2026-10-06T14:12:00Z" +LATER_EVENT = "2026-10-06T14:40:00Z" +CORE_EARLIER, FULL_EARLIER = 101, 102 +CORE_THIS, FULL_THIS, RPC_DOCS_THIS = 201, 202, 203 +CORE_LATER = 301 +SUITE_CREATED_AT = { + CORE_EARLIER: EARLIER_EVENT, + FULL_EARLIER: EARLIER_EVENT, + CORE_THIS: THIS_EVENT, + FULL_THIS: THIS_EVENT, + RPC_DOCS_THIS: THIS_EVENT, + CORE_LATER: LATER_EVENT, +} + +GATE_RUN_PATH = re.escape(f"/actions/runs/{GATE_RUN_ID}") + "$" + + +def suite_lookup(suite: int) -> str: + """The route for the lookup of the workflow run that owns ``suite``.""" + return re.escape(f"/actions/runs?check_suite_id={suite}&exclude_pull_requests=true&per_page=100") + "$" -def check_run(name: str, status: str, conclusion: str | None = None, **stamps: str) -> dict: + +def check_run( + name: str, status: str, conclusion: str | None = None, *, suite: int | None = None, **stamps: str +) -> dict: run = {"name": name, "status": status, "conclusion": conclusion} + if suite is not None: + run["check_suite"] = {"id": suite} run.update(stamps) return run +def workflow_run(run_id: int, suite: int, created_at: str, **fields) -> dict: + run = { + "id": run_id, + "check_suite_id": suite, + "created_at": created_at, + "run_started_at": created_at, + "run_attempt": 1, + } + run.update(fields) + return run + + +def workflow_runs(*runs: dict) -> str: + return json.dumps({"total_count": len(runs), "workflow_runs": list(runs)}) + + +def event_routes( + gate_run: dict | None = None, suites: dict[int, str] | None = None +) -> dict[str, tuple]: + """Answers for the gate's own run and for the run behind each check suite.""" + routes: dict[str, tuple] = { + GATE_RUN_PATH: ok(json.dumps(gate_run or workflow_run(GATE_RUN_ID, 999, THIS_EVENT))), + } + for suite, created_at in (SUITE_CREATED_AT if suites is None else suites).items(): + routes[suite_lookup(suite)] = ok(workflow_runs(workflow_run(50_000 + suite, suite, created_at))) + return routes + + +def iso(moment: datetime) -> str: + return moment.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_iso(stamp: str) -> datetime: + return datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + + def page(*runs: dict) -> dict: return {"total_count": len(runs), "check_runs": list(runs)} @@ -109,65 +191,93 @@ def ok(payload: str) -> tuple[int, str]: class FakeGh: - """A ``gh`` on PATH that scripts one response per call. - - ``responses`` is consumed in order; once exhausted, ``default`` answers - every further call. Each entry is ``(exit_code, stdout)``, optionally with - a third element written to stderr the way ``gh`` reports an HTTP error. - ``delay`` makes every call take that many seconds, so a test can show the - wait deadline counts time spent in the API and not only in ``sleep``. + """A ``gh`` on PATH that scripts its answers. + + Check-runs queries are answered in order from ``responses``; once those are + exhausted, ``default`` answers every further one. ``routes`` answers any + other call whose arguments match its regular expression, with one reply or + a list of replies whose last one repeats. A call nothing answers fails as + an unscripted call, which ``run_gate`` refuses. Each reply is + ``(exit_code, stdout)``, optionally with a third element written to stderr + the way ``gh`` reports an HTTP error. ``delay`` makes every call take that + many seconds, so a test can show the wait deadline counts time spent in the + API and not only in ``sleep``. """ + STUB = """#!/usr/bin/env python3 +import json, re, sys, time +from pathlib import Path + +state = Path(__file__).resolve().parent +config = json.loads((state / "config.json").read_text(encoding="utf8")) +args = " ".join(sys.argv[1:]) +with open(state / "calls", "a", encoding="utf8") as log: + log.write(args + "\\n") +counts_path = state / "counts.json" +counts = json.loads(counts_path.read_text(encoding="utf8")) if counts_path.exists() else {} +key, replies, fallback = None, [], None +for index, (pattern, route_replies) in enumerate(config["routes"]): + if re.search(pattern, args): + key, replies, fallback = f"route-{index}", route_replies, route_replies[-1] + break +if key is None and "/check-runs" in args: + key, replies, fallback = "check-runs", config["responses"], config["default"] +served = counts.get(key, 0) if key else 0 +reply = replies[served] if served < len(replies) else fallback +if reply is None: + sys.stderr.write(f"fake gh: unscripted call: {args}\\n") + sys.exit(97) +counts[key] = served + 1 +counts_path.write_text(json.dumps(counts), encoding="utf8") +time.sleep(config["delay"]) +sys.stdout.write(reply[1]) +sys.stderr.write(reply[2] if len(reply) > 2 and reply[2] else "") +sys.exit(reply[0]) +""" + def __init__( self, directory: Path, *, responses: list[tuple] | None = None, default: tuple | None = None, + routes: dict[str, tuple | list[tuple]] | None = None, delay: int = 0, ) -> None: directory.mkdir(parents=True, exist_ok=True) self.directory = directory self.calls_path = directory / "calls" - replies = directory / "replies" - replies.mkdir() - for name, reply in [(str(i), r) for i, r in enumerate(responses or [], start=1)]: - self._write_reply(replies, name, reply) - if default is not None: - self._write_reply(replies, "default", default) + config = { + "responses": list(responses or []), + "default": default, + "routes": [ + [pattern, list(reply) if isinstance(reply, list) else [reply]] + for pattern, reply in (routes or {}).items() + ], + "delay": delay, + } + (directory / "config.json").write_text(json.dumps(config), encoding="utf8") script = directory / "gh" - script.write_text( - "#!/usr/bin/env bash\n" - f'printf "%s\\n" "$*" >> {self.calls_path!s}\n' - f'call="$(wc -l < {self.calls_path!s})"\n' - f'reply="{replies!s}/${{call}}"\n' - f'[[ -f "${{reply}}" ]] || reply="{replies!s}/default"\n' - 'if [[ ! -f "${reply}" ]]; then\n' - ' echo "fake gh: unscripted call: $*" >&2\n' - " exit 97\n" - "fi\n" - f"{f'sleep {delay}' if delay else ''}\n" - 'code="$(head -n 1 "${reply}")"\n' - '[[ -f "${reply}.err" ]] && cat "${reply}.err" >&2\n' - 'tail -n +2 "${reply}"\n' - 'exit "${code}"\n', - encoding="utf8", - ) + script.write_text(self.STUB, encoding="utf8") script.chmod(0o755) - @staticmethod - def _write_reply(replies: Path, name: str, reply: tuple) -> None: - code, payload = reply[0], reply[1] - (replies / name).write_text(f"{code}\n{payload}", encoding="utf8") - if len(reply) > 2 and reply[2]: - (replies / f"{name}.err").write_text(reply[2], encoding="utf8") - @property def calls(self) -> list[str]: if not self.calls_path.exists(): return [] return self.calls_path.read_text(encoding="utf8").splitlines() + def calls_matching(self, pattern: str) -> list[str]: + return [call for call in self.calls if re.search(pattern, call)] + + @property + def check_run_calls(self) -> list[str]: + return [call for call in self.calls if "/check-runs" in call] + + @property + def workflow_run_calls(self) -> list[str]: + return [call for call in self.calls if "/actions/runs" in call] + def load_yaml(path: Path) -> dict: import yaml @@ -198,6 +308,9 @@ def setUp(self) -> None: match.group("name"): int(match.group("default")) for match in TUNABLE.finditer(self.script) } + skews = EVENT_SKEW.findall(self.script) + self.assertEqual(len(skews), 1, "expected the gate to declare EVENT_SKEW_SECONDS once") + self.event_skew = int(skews[0]) # -- helpers ----------------------------------------------------------- @@ -222,10 +335,7 @@ def gate_env(self, fake: FakeGh, profile: str, **overrides: str) -> dict[str, st "GH_TOKEN": "fake-token", "REPOSITORY": "example/repo", "SHA": "0" * 40, - "RUN_ID": "12345", - "RUN_ATTEMPT": "1", - # Tests that model the start-time lookup clear this. - "GATE_RUN_STARTED_AT": "2026-10-06T14:12:00Z", + "RUN_ID": str(GATE_RUN_ID), } declared = set(self.step["env"]) self.assertFalse( @@ -312,27 +422,28 @@ def test_completed_success_on_every_required_check_passes(self) -> None: self.assertIn("Full Validation Gate: status=completed conclusion=success", completed.stdout) def test_failed_conclusion_fails_as_a_failed_check_not_as_pending(self) -> None: - for conclusion in ("failure", "timed_out", "action_required", "neutral", "skipped"): + for conclusion in ("failure", "timed_out", "action_required", "neutral", "skipped", "stale"): with self.subTest(conclusion=conclusion): fake = FakeGh( self.tmp(f"concluded-{conclusion}"), default=ok( body( page( - check_run("Core Checks Gate", "completed", conclusion, started_at="1"), - check_run("Full Validation Gate", "completed", "success", started_at="1"), + check_run("Core Checks Gate", "completed", conclusion, suite=CORE_THIS, started_at="1"), + check_run("Full Validation Gate", "completed", "success", suite=FULL_THIS, started_at="1"), ) ) ), + routes=event_routes(), ) completed = self.run_gate(fake) self.assertGateFailed( completed, f"{ERROR_CHECK_FAILED}Core Checks Gate concluded {conclusion}.", - forbidden=(ERROR_TIMED_OUT, ERROR_UNREADABLE), + forbidden=(ERROR_TIMED_OUT, ERROR_UNREADABLE, ERROR_UNCORRELATED), ) # A verdict is reached on the first poll, not after a wait. - self.assertEqual(len(fake.calls), 1, fake.calls) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) def test_a_check_still_running_is_pending_and_the_timeout_names_it(self) -> None: fake = FakeGh( @@ -477,50 +588,107 @@ def test_the_newest_check_run_wins_when_a_check_was_rerun(self) -> None: # ... and the reverse: a green first attempt must not mask a red re-run. older_green = check_run("rpc-docs", "completed", "success", started_at="2026-09-17T00:00:00Z") - newer_failed = check_run("rpc-docs", "completed", "failure", started_at="2026-09-18T00:00:00Z") - fake = FakeGh(self.tmp("rerun-red"), default=ok(body(page(older_green, newer_failed)))) + newer_failed = check_run( + "rpc-docs", "completed", "failure", suite=RPC_DOCS_THIS, started_at="2026-09-18T00:00:00Z" + ) + fake = FakeGh( + self.tmp("rerun-red"), default=ok(body(page(older_green, newer_failed))), routes=event_routes() + ) self.assertGateFailed( self.run_gate(fake, profile="rpc-docs"), f"{ERROR_CHECK_FAILED}rpc-docs concluded failure.", ) - # -- results from an earlier run on the same commit --------------------- - - GATE_START = "2026-10-06T14:12:00Z" - BEFORE = "2026-10-05T20:47:00Z" - AFTER = "2026-10-06T14:40:00Z" + # -- which event a failed check belongs to ------------------------------- + # + # A reopen, a label or a re-run starts this gate within seconds, while the + # workflows it waits for create their gate checks only at their last job. + # Until then the newest check of a name can be an earlier event's, whose + # failure says nothing about the run under way. + + def failed_core(self, suite: int, **stamps: str) -> dict: + stamps.setdefault("started_at", "2026-10-06T14:20:00Z") + stamps.setdefault("completed_at", "2026-10-06T14:26:19Z") + return check_run("Core Checks Gate", "completed", "failure", suite=suite, **stamps) + + def green_validation(self) -> dict: + return check_run( + "Full Validation Gate", "completed", "success", suite=FULL_THIS, + started_at="2026-10-06T14:30:00Z", completed_at="2026-10-06T14:31:00Z", + ) - def test_a_failure_from_before_this_gate_run_is_waited_on_not_trusted(self) -> None: - # A reopen, label or re-run starts this gate seconds after the new - # workflows, before their own gate checks exist. The newest check of - # that name is then the previous run's, which says nothing about the - # run under way. + def test_a_failure_from_this_event_fails_at_once(self) -> None: + # Core Checks, Full Validation and this gate are separate workflows the + # same event starts, so a free runner can let Core Checks fail before + # this gate's step, or even its run, has started. for conclusion in ("failure", "timed_out"): with self.subTest(conclusion=conclusion): fake = FakeGh( - self.tmp(f"stale-{conclusion}"), + self.tmp(f"this-event-{conclusion}"), default=ok(body(page( - check_run("Core Checks Gate", "completed", conclusion, started_at=self.BEFORE, completed_at=self.BEFORE), - check_run("Full Validation Gate", "completed", "success", started_at=self.BEFORE, completed_at=self.BEFORE), + check_run( + "Core Checks Gate", "completed", conclusion, suite=CORE_THIS, + started_at="2026-10-06T14:12:01Z", completed_at="2026-10-06T14:12:02Z", + ), + check_run("Full Validation Gate", "in_progress", None, suite=FULL_THIS, started_at="2026-10-06T14:12:01Z"), ))), + routes=event_routes(), ) - completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3", GATE_RUN_STARTED_AT=self.GATE_START) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded {conclusion}.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNREADABLE, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + self.assertIn(f"This gate's workflow run was created at {THIS_EVENT}", completed.stdout) + # The workflow run, not one of its attempts: an attempt has its own created_at. + self.assertEqual(len(fake.calls_matching(GATE_RUN_PATH)), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_THIS))), 1, fake.calls) + + def test_an_earlier_events_failure_is_waited_on_and_the_timeout_names_it(self) -> None: + # It ended after this gate's event, as a failure that outlives a newer + # push or label can: when a check ended says nothing about its event. + for conclusion in ("failure", "timed_out", "action_required"): + with self.subTest(conclusion=conclusion): + fake = FakeGh( + self.tmp(f"earlier-{conclusion}"), + default=ok(body(page( + check_run( + "Core Checks Gate", "completed", conclusion, suite=CORE_EARLIER, + started_at=EARLIER_EVENT, completed_at=LATER_EVENT, + ), + self.green_validation(), + ))), + routes=event_routes(), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3") self.assertGateFailed( completed, f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: " - f"Core Checks Gate (last {conclusion} before this gate run)", - forbidden=(ERROR_CHECK_FAILED, ERROR_UNREADABLE), + f"Core Checks Gate (last {conclusion} from an earlier event)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNREADABLE, ERROR_UNCORRELATED), ) - self.assertIn("waiting for a newer run", completed.stdout) - self.assertGreater(len(fake.calls), 1, "a stale failure must be polled again, not judged once") - - def test_the_newer_run_replaces_a_stale_failure(self) -> None: - stale = check_run("Core Checks Gate", "completed", "cancelled", started_at=self.BEFORE, completed_at=self.BEFORE) - validation = check_run("Full Validation Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER) - fresh_running = check_run("Core Checks Gate", "in_progress", None, started_at=self.AFTER) - fresh_green = check_run("Core Checks Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER) + self.assertIn( + f"Core Checks Gate: concluded {conclusion} in a workflow run created at {EARLIER_EVENT}, " + f"by an earlier event than this gate's ({THIS_EVENT}); waiting for a newer run", + completed.stdout, + ) + self.assertGreater(len(fake.check_run_calls), 2, "an earlier event's failure must be polled again") + # Dated once: the workflow run behind a check suite cannot change. + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_EARLIER))), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(GATE_RUN_PATH)), 1, fake.calls) + + def test_the_newer_run_replaces_an_earlier_events_failure(self) -> None: + stale = self.failed_core(CORE_EARLIER, started_at=EARLIER_EVENT, completed_at=EARLIER_EVENT) + fresh_running = check_run("Core Checks Gate", "in_progress", None, suite=CORE_THIS, started_at="2026-10-06T14:30:00Z") + fresh_green = check_run( + "Core Checks Gate", "completed", "success", suite=CORE_THIS, + started_at="2026-10-06T14:30:00Z", completed_at="2026-10-06T14:44:00Z", + ) + validation = self.green_validation() fake = FakeGh( - self.tmp("stale-then-green"), + self.tmp("earlier-then-green"), responses=[ ok(body(page(stale, validation))), ok(body(page(stale, validation))), @@ -528,105 +696,297 @@ def test_the_newer_run_replaces_a_stale_failure(self) -> None: ok(body(page(stale, fresh_running, validation))), ], default=ok(body(page(stale, fresh_green, validation))), + routes=event_routes(), ) - completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) + completed = self.run_gate(fake) self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn("waiting for a newer run", completed.stdout) + self.assertIn("Core Checks Gate: status=in_progress conclusion=pending", completed.stdout) self.assertIn("Core Checks Gate: status=completed conclusion=success", completed.stdout) - def test_a_failure_after_this_gate_run_started_still_fails_at_once(self) -> None: + def test_a_rerun_of_only_this_gate_fails_at_once_on_this_events_failure(self) -> None: + # Re-running this gate alone starts no Core Checks or Full Validation, + # so no newer run is coming. The re-run keeps its workflow run and that + # run's created_at, and moves only run_attempt and run_started_at, so a + # failure that ended hours before the re-run is still this event's. + rerun = workflow_run(GATE_RUN_ID, 999, THIS_EVENT, run_attempt=4, run_started_at="2026-10-06T18:02:04Z") + fake = FakeGh( + self.tmp("gate-only-rerun"), + default=ok(body(page(self.failed_core(CORE_THIS), self.green_validation()))), + routes=event_routes(gate_run=rerun), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + + def test_a_rerun_of_an_earlier_events_workflow_is_still_that_events(self) -> None: + # Re-running an earlier event's Core Checks keeps its workflow run's + # created_at, so its failure, however recent, is still waited on: the + # check this event started will be newer. + rerun = workflow_run( + 50_101, CORE_EARLIER, EARLIER_EVENT, run_attempt=2, run_started_at="2026-10-06T15:31:19Z" + ) + routes = event_routes() + routes[suite_lookup(CORE_EARLIER)] = ok(workflow_runs(rerun)) fake = FakeGh( - self.tmp("fresh-failure"), + self.tmp("earlier-workflow-rerun"), default=ok(body(page( - check_run("Core Checks Gate", "completed", "failure", started_at=self.AFTER, completed_at=self.AFTER), - check_run("Full Validation Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER), + self.failed_core(CORE_EARLIER, started_at="2026-10-06T15:31:30Z", completed_at="2026-10-06T15:45:16Z"), + self.green_validation(), ))), + routes=routes, + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="2") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 2s and are still pending: " + "Core Checks Gate (last failure from an earlier event)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED), ) - completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) - self.assertGateFailed(completed, f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", - forbidden=(ERROR_TIMED_OUT,)) - self.assertEqual(len(fake.calls), 1, fake.calls) - def test_a_success_from_before_this_gate_run_still_counts(self) -> None: - # A label starts the gate and Full Validation but not Core Checks, whose - # earlier green result on the same commit is still the right answer. + def test_a_label_event_fails_at_once_on_its_own_core_checks_failure(self) -> None: + # A label on a commit whose Core Checks had passed starts this gate, + # Core Checks and Full Validation again. This event's Core Checks + # failed before this gate first polled: it is the newest Core Checks + # Gate and the verdict, while Full Validation is still running. + earlier_green = check_run( + "Core Checks Gate", "completed", "success", suite=CORE_EARLIER, + started_at=EARLIER_EVENT, completed_at=EARLIER_EVENT, + ) + this_failed = self.failed_core(CORE_THIS, started_at="2026-10-06T14:12:40Z", completed_at="2026-10-06T14:13:10Z") + validation_running = check_run("Full Validation Gate", "in_progress", None, suite=FULL_THIS, started_at="2026-10-06T14:12:05Z") fake = FakeGh( - self.tmp("stale-success"), + self.tmp("label-this-event"), + default=ok(body(page(earlier_green, this_failed, validation_running))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + + def test_a_later_events_failure_is_current(self) -> None: + # Nothing this gate could wait for is newer than a later event's run. + fake = FakeGh( + self.tmp("later-event"), + default=ok(body(page(self.failed_core(CORE_LATER, started_at=LATER_EVENT), self.green_validation()))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + + def test_the_event_skew_allowance_is_a_documented_constant(self) -> None: + self.assertEqual(self.event_skew, DOCUMENTED_EVENT_SKEW_SECONDS) + self.assertNotIn("EVENT_SKEW_SECONDS:-", self.script, "the allowance must not be overridable") + + def test_the_event_skew_boundary(self) -> None: + # Workflow runs one event starts are created within the allowance of + # each other; a run created further before this gate's is an earlier + # event's. The documented value, not the script's, sets the boundary. + gate = parse_iso(THIS_EVENT) + skew = timedelta(seconds=DOCUMENTED_EVENT_SKEW_SECONDS) + one = timedelta(seconds=1) + cases = { + "the same second": (gate, "current"), + "the allowance before": (gate - skew, "current"), + "one second more": (gate - skew - one, "earlier"), + "after this gate's run": (gate + skew + one, "current"), + } + for label, (created, expected) in cases.items(): + with self.subTest(case=label, created=iso(created)): + fake = FakeGh( + self.tmp(f"skew-{label}"), + default=ok(body(page(self.failed_core(777), self.green_validation()))), + routes=event_routes(suites={777: iso(created)}), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="2") + if expected == "current": + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + else: + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 2s and are still pending: " + "Core Checks Gate (last failure from an earlier event)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED), + ) + + def test_a_success_from_an_earlier_event_still_counts(self) -> None: + # A success on this commit is a verdict whichever event produced it, + # so it is never dated: no workflow run is read for it. + fake = FakeGh( + self.tmp("earlier-success"), default=ok(body(page( - check_run("Core Checks Gate", "completed", "success", started_at=self.BEFORE, completed_at=self.BEFORE), - check_run("Full Validation Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER), + check_run( + "Core Checks Gate", "completed", "success", suite=CORE_EARLIER, + started_at=EARLIER_EVENT, completed_at=EARLIER_EVENT, + ), + self.green_validation(), ))), ) - completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) + completed = self.run_gate(fake) self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertEqual(fake.workflow_run_calls, []) def test_a_cancelled_check_is_never_a_verdict(self) -> None: - # A newer push or label cancels the running Full Validation, and that - # cancellation can complete after this gate run started. - cancelled = check_run("Full Validation Gate", "completed", "cancelled", started_at=self.BEFORE, completed_at=self.AFTER) - core = check_run("Core Checks Gate", "completed", "success", started_at=self.AFTER, completed_at=self.AFTER) - fake = FakeGh(self.tmp("cancelled-alone"), default=ok(body(page(core, cancelled)))) - completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3", GATE_RUN_STARTED_AT=self.GATE_START) - self.assertGateFailed( - completed, - f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: Full Validation Gate (cancelled)", - forbidden=(ERROR_CHECK_FAILED, ERROR_UNREADABLE), + # A newer push, reopen or label cancels the running Full Validation, + # and that cancellation can complete after this gate run started. + # Whichever event's it is, it is waited on and never dated. + core = check_run( + "Core Checks Gate", "completed", "success", suite=CORE_THIS, + started_at="2026-10-06T14:12:05Z", completed_at="2026-10-06T14:26:00Z", + ) + for label, suite in {"earlier": FULL_EARLIER, "this": FULL_THIS}.items(): + with self.subTest(event=label): + cancelled = check_run( + "Full Validation Gate", "completed", "cancelled", suite=suite, + started_at=EARLIER_EVENT, completed_at=LATER_EVENT, + ) + fake = FakeGh(self.tmp(f"cancelled-{label}"), default=ok(body(page(core, cancelled)))) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: " + "Full Validation Gate (cancelled)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNREADABLE, ERROR_UNCORRELATED), + ) + self.assertEqual(fake.workflow_run_calls, []) + + cancelled = check_run( + "Full Validation Gate", "completed", "cancelled", suite=FULL_THIS, + started_at="2026-10-06T14:12:05Z", completed_at="2026-10-06T14:13:00Z", + ) + newer_green = check_run( + "Full Validation Gate", "completed", "success", suite=FULL_THIS, + started_at="2026-10-06T14:20:00Z", completed_at="2026-10-06T15:00:00Z", ) - newer_green = check_run("Full Validation Gate", "completed", "success", started_at=self.AFTER, completed_at="2026-10-06T15:00:00Z") fake = FakeGh( self.tmp("cancelled-then-green"), responses=[ok(body(page(core, cancelled))), ok(body(page(core, cancelled)))], default=ok(body(page(core, cancelled, newer_green))), ) - completed = self.run_gate(fake, GATE_RUN_STARTED_AT=self.GATE_START) + completed = self.run_gate(fake) self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + def test_a_failure_that_cannot_be_dated_fails_closed_with_a_distinct_error(self) -> None: + # Not knowing a failure's event is never success, never a wait for a + # run that may not come, and never reported as the check's own verdict. + gate_run = ok(json.dumps(workflow_run(GATE_RUN_ID, 999, THIS_EVENT))) + core_run = workflow_run(60_201, CORE_THIS, THIS_EVENT) + server_error = (1, "", "gh: Server Error (HTTP 502)\n") + forbidden = (1, "", "gh: Resource not accessible by integration (HTTP 403)\n") + failed = self.failed_core(CORE_THIS) + suite_route = suite_lookup(CORE_THIS) + # case: (routes, check run, env overrides, the route read until it gave up) + cases = { + "the check's run cannot be read": ({GATE_RUN_PATH: gate_run, suite_route: server_error}, failed, {}, suite_route), + "the token cannot read workflow runs": ({GATE_RUN_PATH: gate_run, suite_route: forbidden}, failed, {}, suite_route), + "no run owns the check suite": ({GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs())}, failed, {}, suite_route), + "only another suite's run": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, 555, THIS_EVENT)))}, + failed, {}, suite_route, + ), + "two runs claim the check suite": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(core_run, workflow_run(2, CORE_THIS, EARLIER_EVENT)))}, + failed, {}, suite_route, + ), + "the check's run has no created_at": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, None)))}, + failed, {}, suite_route, + ), + "the check's run has fractional seconds": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, "2026-10-06T14:12:00.5Z")))}, + failed, {}, suite_route, + ), + "the response is not json": ({GATE_RUN_PATH: gate_run, suite_route: ok("502")}, failed, {}, suite_route), + "this gate's run cannot be read": ({GATE_RUN_PATH: server_error}, failed, {}, GATE_RUN_PATH), + "this gate's run is another run": ( + {GATE_RUN_PATH: ok(json.dumps(workflow_run(GATE_RUN_ID + 1, 999, THIS_EVENT)))}, failed, {}, GATE_RUN_PATH, + ), + "this gate's run has no created_at": ( + {GATE_RUN_PATH: ok(json.dumps(workflow_run(GATE_RUN_ID, 999, "")))}, failed, {}, GATE_RUN_PATH, + ), + "this gate's run id is missing": ({}, failed, {"RUN_ID": ""}, None), + "the check run names no check suite": ( + {GATE_RUN_PATH: gate_run}, + check_run("Core Checks Gate", "completed", "failure", started_at="2026-10-06T14:20:00Z"), + {}, None, + ), + } + for index, (label, (routes, core, env, gave_up)) in enumerate(cases.items()): + with self.subTest(case=label): + fake = FakeGh( + self.tmp(f"undatable-{index}"), + default=ok(body(page(core, self.green_validation()))), + routes=routes, + ) + completed = self.run_gate(fake, GATE_QUERY_MAX_ATTEMPTS="2", **env) + self.assertGateFailed( + completed, + f"{ERROR_UNCORRELATED}Core Checks Gate on {'0' * 40} concluded failure, but the workflow " + "runs that tell whether it belongs to this event could not be read; failing closed with " + "no verdict for: Core Checks Gate Full Validation Gate", + forbidden=(ERROR_CHECK_FAILED, ERROR_TIMED_OUT, ERROR_UNREADABLE), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + if gave_up is not None: + # Bounded by the same retries as the check-runs query. + self.assertEqual(len(fake.calls_matching(gave_up)), 2, fake.calls) + self.assertIn(f"{WARNING_RETRYING}Core Checks Gate: workflow-run ", completed.stdout) + + def test_a_transient_lookup_failure_is_retried(self) -> None: + routes = event_routes() + routes[suite_lookup(CORE_THIS)] = [ + (1, "", "gh: Server Error (HTTP 502)\n"), + ok(workflow_runs(workflow_run(60_201, CORE_THIS, THIS_EVENT))), + ] + fake = FakeGh( + self.tmp("lookup-transient"), + default=ok(body(page(self.failed_core(CORE_THIS), self.green_validation()))), + routes=routes, + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_UNCORRELATED, ERROR_TIMED_OUT), + ) + self.assertIn(f"{WARNING_RETRYING}Core Checks Gate: workflow-run query failed (exit 1)", completed.stdout) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_THIS))), 2, fake.calls) + def test_every_gate_event_can_start_the_checks_it_waits_for(self) -> None: - # The gate waits for a newer run instead of trusting an earlier failure, - # so each event that starts the gate must also start the gated workflows. + # The gate waits for this event's run instead of trusting an earlier + # event's failure, so each event that starts the gate must also start + # the gated workflows. gate_types = set(self.workflow["on"]["pull_request"]["types"]) for name in ("core-checks.yml", "ci.yml"): other = load_yaml(REPO_ROOT / ".github" / "workflows" / name) types = set(other["on"]["pull_request"].get("types", ["opened", "synchronize", "reopened"])) self.assertEqual(gate_types - types, set(), f"{name} misses events that start the Required Merge Gate") - def test_the_workflow_token_can_read_its_run_start(self) -> None: - # Get a workflow run attempt needs Actions: read; without it the lookup - # always falls back to the step's start time. + def test_the_workflow_token_can_read_workflow_runs(self) -> None: + # Getting or listing workflow runs needs Actions: read; without it every + # failure would fail closed as undatable. permissions = self.workflow["permissions"] self.assertEqual(permissions.get("actions"), "read", permissions) self.assertEqual(permissions.get("checks"), "read", permissions) - def test_the_gate_start_is_the_workflow_runs_not_the_steps(self) -> None: - # The step starts after the jobs it needs, so a failure from this run can - # complete before it; the run's own start time keeps it a current failure. - attempt = ok(json.dumps({"run_started_at": "2026-10-06T14:00:00Z"})) - fake = FakeGh( - self.tmp("run-start"), - responses=[attempt], - default=ok(body(page( - check_run("Core Checks Gate", "completed", "failure", started_at="2026-10-06T14:01:00Z", completed_at="2026-10-06T14:05:00Z"), - check_run("Full Validation Gate", "completed", "success", started_at="2026-10-06T14:01:00Z", completed_at="2026-10-06T14:05:00Z"), - ))), - ) - completed = self.run_gate(fake, GATE_RUN_STARTED_AT="") - self.assertGateFailed(completed, f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", - forbidden=(ERROR_TIMED_OUT,)) - self.assertIn("Gate run started at 2026-10-06T14:00:00Z", completed.stdout) - self.assertIn("/actions/runs/12345/attempts/1", " ".join(fake.calls)) - - def test_an_unreadable_run_start_falls_back_with_a_warning(self) -> None: - fake = FakeGh( - self.tmp("run-start-unreadable"), - responses=[ok("not json")], - default=ok(body(page( - check_run("Core Checks Gate", "completed", "success", started_at="1"), - check_run("Full Validation Gate", "completed", "success", started_at="1"), - ))), - ) - completed = self.run_gate(fake, GATE_RUN_STARTED_AT="") - self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) - self.assertIn("Required Merge Gate start time unknown", completed.stdout + completed.stderr) - def test_a_queued_rerun_falls_back_to_created_at_and_is_pending(self) -> None: # A re-run that has not started yet has no started_at at all; ordering # by created_at keeps it, so an old green run cannot pass the gate. From 780afd46318e8869b928f297b1d37411e189da85 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Wed, 21 Oct 2026 12:13:01 -0400 Subject: [PATCH 7/8] ci: let the newest event's check decide, not the newest-started one The gate judged only the newest check run of each required name by started_at. A manual re-run of an earlier event's workflow starts later than this event's check, so it hid this event's result: a failed old re-run was taken as an earlier event's failure and waited on, poll after poll, until the deadline, even with this event's success already on the commit. Read every check run of the name instead. When they come from more than one workflow run and not all of them succeeded, map each check suite to its workflow run (cached per suite) and let the newest workflow run's check decide, by created_at, then run_number for runs created in the same second, then started_at for its attempts. - An earlier event's re-run never outranks this or a later event's result, whether it failed, passed, was cancelled or is still running. - Attempts within one workflow run still rank by started_at alone, and a check whose runs all succeeded passes, both without a lookup. - If the runs to rank can't be read, or a check run names no check suite, the gate fails closed under its "could not correlate" title. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/required-merge-gate.yml | 315 +++++++++++++++------- ci/checks/test_merge_gate_poller.py | 259 ++++++++++++++++-- 2 files changed, 457 insertions(+), 117 deletions(-) diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index d2832794ed..999cf9f8db 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -764,14 +764,17 @@ jobs: # # * a page without a check_runs array is malformed and must never be # mistaken for "this commit has no check runs"; - # * a re-run check appears more than once, so the newest run by - # started_at (created_at while a run has not started) wins; + # * every check run with the name is reported, one per line, oldest + # first by started_at (created_at while a run has not started): a + # re-run check appears more than once, and so does a check that + # more than one event ran (judge_check below picks the one that + # decides); # * "absent" means the query succeeded and reported no run with that # name, which is a different fact from a query that could not be # read at all; - # * the check suite travels with the result: it names the workflow - # run, and so the event, that produced the check (see - # date_check_event below). + # * each check run's check suite travels with it: it names the + # workflow run, and so the event, that produced the check (see + # date_check_suite below). # $name is a jq variable bound by --arg below, not a shell expansion. # shellcheck disable=SC2016 check_run_filter=' @@ -780,10 +783,13 @@ jobs: | add | map(select(.name == $name)) | sort_by(.started_at // .created_at // "") - | last - | if . == null then "absent" else ["found", (.status // ""), (.conclusion // ""), (((.check_suite // {}).id // "") | tostring)] | @tsv end + | if length == 0 then "absent" + else .[] | ["found", (.status // ""), (.conclusion // ""), (((.check_suite // {}).id // "") | tostring), (.started_at // .created_at // "")] | @tsv + end ' - check_run_pattern=$'^(absent|found\t[a-z_]+\t[^\t]*\t[^\t]*)$' + # "foundstatusconclusioncheck suite idstarted_at" + check_run_line=$'found\t[a-z_]+\t[^\t\n]*\t[^\t\n]*\t[^\t\n]*' + check_run_pattern="^(absent|${check_run_line}("$'\n'"${check_run_line})*)\$" summarize_stderr() { tr '\n' ' ' < "$1" | cut -c1-300 @@ -871,12 +877,11 @@ jobs: return 1 } - # Read the newest check run named $1 on ${SHA}. + # Read every check run named $1 on ${SHA}. # - # On success sets QUERY_OBSERVATION to "absent" or to - # "foundstatusconclusioncheck suite id" and returns 0. - # Returns 1 when the query could not be read (see read_api; $2 is - # the deadline). + # On success sets QUERY_OBSERVATION to "absent" or to one line per + # check run (see check_run_pattern) and returns 0. Returns 1 when the + # query could not be read (see read_api; $2 is the deadline). query_check_run() { QUERY_OBSERVATION="" read_api "$1" "check-runs" "$2" \ @@ -908,8 +913,8 @@ jobs: # # * a check run's check suite names the workflow run that produced # it (GET /actions/runs?check_suite_id=), whose created_at dates - # the event that started it; once read, a check suite's date is - # cached, as it cannot change; + # the event that started it; once read, a check suite's workflow + # run is cached, as it cannot change; # * this gate's own workflow run's created_at (GET # /actions/runs/{run_id}, never the attempt's) dates the event it # judges, so a gate-only re-run still judges its original event; @@ -918,8 +923,9 @@ jobs: # any other belongs to this event or a later one and is current. # # Only a check that concluded neither success nor cancelled is - # dated: a success on this commit passes whichever event produced - # it, and a cancellation is never a verdict. + # dated against this gate: a success on this commit passes whichever + # event produced it, and a cancellation is never a verdict. Which of + # a check's runs is judged at all is judge_check's to say, below. # # Workflow runs started by one event are created this close together. # A documented allowance, not a tunable: a larger value takes more of @@ -938,6 +944,8 @@ jobs: | [(fromdateiso8601 | tostring), .] | @tsv ' + # "" + created_at_pattern=$'^[0-9]+\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' # GET /actions/runs?check_suite_id={id}: the one workflow run that # owns a check suite. A run of another suite is never taken for it, # even if the filter were ignored. $suite is bound by --argjson. @@ -947,15 +955,50 @@ jobs: | map(if (.workflow_runs | type) == "array" then .workflow_runs[] else error("malformed workflow-runs page") end) | map(select(.check_suite_id == $suite)) | if length != 1 then error("\(length) workflow runs own check suite \($suite)") else .[0] end - | .created_at - | [(fromdateiso8601 | tostring), .] + | [(.created_at | fromdateiso8601 | tostring), (.run_number | tostring), .created_at] | @tsv ' - # "" - created_at_pattern=$'^[0-9]+\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + # "" + suite_run_pattern=$'^[0-9]+\t[1-9][0-9]*\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' GATE_RUN_CREATED="" - declare -A SUITE_RUN_CREATED=() + declare -A SUITE_RUN=() + + # Read this gate's own workflow run once, into GATE_RUN_CREATED + # (""). Returns 1 if it could not be read by + # $2, the shared deadline ($1 names the check for the log). + date_gate_run() { + if [[ -n "${GATE_RUN_CREATED}" ]]; then + return 0 + fi + if [[ ! "${RUN_ID}" =~ ^[1-9][0-9]*$ ]]; then + echo "$1: this gate's workflow run id is not a number: '${RUN_ID}'" + return 1 + fi + read_api "$1" "workflow-run" "$2" \ + "/repos/${REPOSITORY}/actions/runs/${RUN_ID}" \ + "${created_at_pattern}" "${gate_run_filter}" --argjson run_id "${RUN_ID}" || return 1 + GATE_RUN_CREATED="${API_RESULT}" + echo "This gate's workflow run was created at ${GATE_RUN_CREATED#*$'\t'}, by the event it judges" + } + + # Read the workflow run that owns check suite $2 once, into + # SUITE_RUN[$2] (see suite_run_pattern). Returns 1 if $2 is not a + # check suite id, or its workflow run could not be read by $3, the + # shared deadline ($1 names the check for the log). + date_check_suite() { + if [[ ! "$2" =~ ^[1-9][0-9]*$ ]]; then + echo "$1: the check run names no check suite: '$2'" + return 1 + fi + if [[ -n "${SUITE_RUN[$2]:-}" ]]; then + return 0 + fi + read_api "$1" "workflow-run" "$3" \ + "/repos/${REPOSITORY}/actions/runs?check_suite_id=$2&exclude_pull_requests=true&per_page=100" \ + "${suite_run_pattern}" "${suite_run_filter}" --argjson suite "$2" || return 1 + SUITE_RUN[$2]="${API_RESULT}" + } # Date the event of check $1, from check suite $2 (see above). # @@ -973,34 +1016,13 @@ jobs: CHECK_EVENT="" CHECK_RUN_CREATED_AT="" - if [[ -z "${GATE_RUN_CREATED}" ]]; then - if [[ ! "${RUN_ID}" =~ ^[1-9][0-9]*$ ]]; then - echo "${check_name}: this gate's workflow run id is not a number: '${RUN_ID}'" - return 1 - fi - read_api "${check_name}" "workflow-run" "${deadline}" \ - "/repos/${REPOSITORY}/actions/runs/${RUN_ID}" \ - "${created_at_pattern}" "${gate_run_filter}" --argjson run_id "${RUN_ID}" || return 1 - GATE_RUN_CREATED="${API_RESULT}" - echo "This gate's workflow run was created at ${GATE_RUN_CREATED#*$'\t'}, by the event it judges" - fi - - if [[ ! "${check_suite_id}" =~ ^[1-9][0-9]*$ ]]; then - echo "${check_name}: the check run names no check suite: '${check_suite_id}'" - return 1 - fi - if [[ -z "${SUITE_RUN_CREATED[${check_suite_id}]:-}" ]]; then - read_api "${check_name}" "workflow-run" "${deadline}" \ - "/repos/${REPOSITORY}/actions/runs?check_suite_id=${check_suite_id}&exclude_pull_requests=true&per_page=100" \ - "${created_at_pattern}" "${suite_run_filter}" --argjson suite "${check_suite_id}" || return 1 - SUITE_RUN_CREATED[${check_suite_id}]="${API_RESULT}" - fi + date_gate_run "${check_name}" "${deadline}" || return 1 + date_check_suite "${check_name}" "${check_suite_id}" "${deadline}" || return 1 - check_created="${SUITE_RUN_CREATED[${check_suite_id}]}" - gate_created="${GATE_RUN_CREATED}" - CHECK_RUN_CREATED_AT="${check_created#*$'\t'}" + check_created="${SUITE_RUN[${check_suite_id}]}" + CHECK_RUN_CREATED_AT="${check_created##*$'\t'}" check_created="${check_created%%$'\t'*}" - gate_created="${gate_created%%$'\t'*}" + gate_created="${GATE_RUN_CREATED%%$'\t'*}" if (( check_created < gate_created - EVENT_SKEW_SECONDS )); then CHECK_EVENT="earlier" else @@ -1008,21 +1030,146 @@ jobs: fi } + # Judge check $1 from its check runs in QUERY_OBSERVATION, by $2, the + # shared deadline. + # + # A check has a run per event that ran CI on this commit, each in + # that event's workflow run, and a re-run adds another to the same + # workflow run. The check run of the newest workflow run decides: by + # its created_at, which dates its event, then its run_number, which + # orders one workflow's runs created in the same second; and within + # that workflow run the newest attempt, by started_at. So a manual + # re-run of an earlier event's workflow, however recent, never + # outranks this event's (or a later event's) result, and an earlier + # event's result is judged only while no later one has reported. + # Workflow runs are read only to rank check runs from more than one + # of them, and only when not all of those succeeded: a check whose + # runs all succeeded passes whichever decides. + # + # Sets CHECK_VERDICT to "success", "failed" or "pending" and returns 0; + # a failed check's conclusion is in CHECK_CONCLUSION, and a pending + # one is named in CHECK_PENDING the way the timeout reports it. + # Returns 1, with the reason in CHECK_UNCORRELATED, when the workflow + # runs the verdict depends on could not be read. + judge_check() { + local check_name="$1" + local deadline="$2" + local lines=() statuses=() conclusions=() suites=() stamps=() + local line fields index decide ranked run run_number + local status conclusion all_success=1 + local -A distinct=() + + CHECK_VERDICT="" + CHECK_CONCLUSION="" + CHECK_PENDING="${check_name}" + CHECK_UNCORRELATED="" + + if [[ "${QUERY_OBSERVATION}" == "absent" ]]; then + echo "${check_name}: query succeeded, no such check run on ${SHA} yet" + CHECK_VERDICT="pending" + return 0 + fi + + mapfile -t lines <<< "${QUERY_OBSERVATION}" + for line in "${lines[@]}"; do + fields="${line#found$'\t'}" + statuses+=("${fields%%$'\t'*}") + fields="${fields#*$'\t'}" + conclusions+=("${fields%%$'\t'*}") + fields="${fields#*$'\t'}" + suites+=("${fields%%$'\t'*}") + stamps+=("${fields#*$'\t'}") + if [[ "${statuses[-1]}" != "completed" || "${conclusions[-1]}" != "success" ]]; then + all_success=0 + fi + done + + # The newest by started_at, which is right within one workflow run. + decide=$(( ${#lines[@]} - 1 )) + if (( ${#lines[@]} > 1 && ! all_success )); then + for index in "${!lines[@]}"; do + if [[ ! "${suites[index]}" =~ ^[1-9][0-9]*$ ]]; then + echo "${check_name}: a check run names no check suite: '${suites[index]}'" + CHECK_UNCORRELATED="${check_name} on ${SHA} has ${#lines[@]} check runs, but not every one names the workflow run that tells which event is newest" + return 1 + fi + distinct[${suites[index]}]=1 + done + if (( ${#distinct[@]} > 1 )); then + ranked="" + for index in "${!lines[@]}"; do + if ! date_check_suite "${check_name}" "${suites[index]}" "${deadline}"; then + CHECK_UNCORRELATED="${check_name} on ${SHA} has check runs from ${#distinct[@]} workflow runs, but the workflow runs that tell which event is newest could not be read" + return 1 + fi + run="${SUITE_RUN[${suites[index]}]}" + run_number="${run#*$'\t'}" + run_number="${run_number%%$'\t'*}" + ranked+="${run%%$'\t'*}"$'\t'"${run_number}"$'\t'"${stamps[index]}"$'\t'"${index}"$'\n' + done + decide="$(printf '%s' "${ranked}" | LC_ALL=C sort -t $'\t' -k1,1n -k2,2n -k3,3 -k4,4n | tail -n 1)" + decide="${decide##*$'\t'}" + if [[ ! "${decide}" =~ ^[0-9]+$ ]] || (( decide >= ${#lines[@]} )); then + CHECK_UNCORRELATED="${check_name} on ${SHA} has check runs from ${#distinct[@]} workflow runs, which could not be ranked" + return 1 + fi + run="${SUITE_RUN[${suites[decide]}]}" + echo "${check_name}: ${#lines[@]} check runs from ${#distinct[@]} workflow runs; the newest workflow run's, created at ${run##*$'\t'}, decides" + fi + fi + + status="${statuses[decide]}" + conclusion="${conclusions[decide]}" + echo "${check_name}: status=${status} conclusion=${conclusion:-pending}" + + if [[ "${status}" != "completed" ]]; then + CHECK_VERDICT="pending" + return 0 + fi + if [[ "${conclusion}" == "success" ]]; then + CHECK_VERDICT="success" + return 0 + fi + if [[ "${conclusion}" == "cancelled" ]]; then + # A cancelled check is never a verdict: it was superseded (a + # newer push, reopen or label cancels the running workflow, + # possibly after this gate started) or it never ran. Wait for + # the run that replaces it; the deadline still fails the gate. + echo "${check_name}: cancelled; waiting for the run that replaces it" + CHECK_PENDING="${check_name} (cancelled)" + CHECK_VERDICT="pending" + return 0 + fi + + if ! date_check_event "${check_name}" "${suites[decide]}" "${deadline}"; then + CHECK_UNCORRELATED="${check_name} on ${SHA} concluded ${conclusion:-unknown}, but the workflow runs that tell whether it belongs to this event could not be read" + return 1 + fi + if [[ "${CHECK_EVENT}" == "earlier" ]]; then + echo "${check_name}: concluded ${conclusion:-unknown} in a workflow run created at ${CHECK_RUN_CREATED_AT}, by an earlier event than this gate's (${GATE_RUN_CREATED#*$'\t'}); waiting for a newer run" + CHECK_PENDING="${check_name} (last ${conclusion:-unknown} from an earlier event)" + CHECK_VERDICT="pending" + return 0 + fi + CHECK_CONCLUSION="${conclusion}" + CHECK_VERDICT="failed" + } + # Wait until every named check on ${SHA} has concluded successfully. # # Returns 1 under four distinct error titles: a check concluded # something other than success, the check runs could not be read, a - # check that concluded otherwise could not be traced to its event, or - # the shared deadline passed with checks still pending. Only a - # completed check with a success conclusion is success. A check that - # concluded otherwise is a verdict only if it belongs to the event - # this gate judges, or a later one; an earlier event's is waited on, - # as its newer run may not have reported yet, and a cancelled one is - # always waited on. + # check's runs could not be traced to their events, or the shared + # deadline passed with checks still pending. Only a completed check + # with a success conclusion is success. A check that concluded + # otherwise is a verdict only if it belongs to the event this gate + # judges, or a later one; an earlier event's is waited on, as its + # newer run may not have reported yet, and a cancelled one is always + # waited on. wait_for_checks() { local check_names=("$@") local pending=() - local check_name check_suite_id conclusion fields now observation status deadline rest + local check_name now deadline now="$(date +%s)" deadline=$(( now + GATE_WAIT_TIMEOUT_SECONDS )) @@ -1035,48 +1182,26 @@ jobs: echo "::error title=Required Merge Gate could not read check runs::Check runs for ${check_name} on ${SHA} could not be read after ${GATE_QUERY_MAX_ATTEMPTS} attempts; failing closed with no verdict for: ${check_names[*]}" return 1 fi - observation="${QUERY_OBSERVATION}" - - if [[ "${observation}" == "absent" ]]; then - echo "${check_name}: query succeeded, no such check run on ${SHA} yet" - pending+=("${check_name}") - continue + if ! judge_check "${check_name}" "${deadline}"; then + echo "::error title=Required Merge Gate could not correlate check runs::${CHECK_UNCORRELATED}; failing closed with no verdict for: ${check_names[*]}" + return 1 fi - fields="${observation#found$'\t'}" - status="${fields%%$'\t'*}" - rest="${fields#*$'\t'}" - conclusion="${rest%%$'\t'*}" - check_suite_id="${rest#*$'\t'}" - echo "${check_name}: status=${status} conclusion=${conclusion:-pending}" - - if [[ "${status}" == "completed" ]]; then - if [[ "${conclusion}" == "cancelled" ]]; then - # A cancelled check is never a verdict: it was superseded (a - # newer push, reopen or label cancels the running workflow, - # possibly after this gate started) or it never ran. Wait for - # the run that replaces it; the deadline still fails the gate. - echo "${check_name}: cancelled; waiting for the run that replaces it" - pending+=("${check_name} (cancelled)") - continue - fi - if [[ "${conclusion}" != "success" ]]; then - if ! date_check_event "${check_name}" "${check_suite_id}" "${deadline}"; then - echo "::error title=Required Merge Gate could not correlate check runs::${check_name} on ${SHA} concluded ${conclusion:-unknown}, but the workflow runs that tell whether it belongs to this event could not be read; failing closed with no verdict for: ${check_names[*]}" - return 1 - fi - if [[ "${CHECK_EVENT}" == "earlier" ]]; then - echo "${check_name}: concluded ${conclusion:-unknown} in a workflow run created at ${CHECK_RUN_CREATED_AT}, by an earlier event than this gate's (${GATE_RUN_CREATED#*$'\t'}); waiting for a newer run" - pending+=("${check_name} (last ${conclusion:-unknown} from an earlier event)") - continue - fi - echo "::error title=Required Merge Gate failed::${check_name} concluded ${conclusion:-unknown}." + case "${CHECK_VERDICT}" in + success) + ;; + pending) + pending+=("${CHECK_PENDING}") + ;; + failed) + echo "::error title=Required Merge Gate failed::${check_name} concluded ${CHECK_CONCLUSION:-unknown}." return 1 - fi - continue - fi - - pending+=("${check_name}") + ;; + *) + echo "::error title=Required Merge Gate failed::${check_name} has no verdict ('${CHECK_VERDICT}')." + return 1 + ;; + esac done if (( ${#pending[@]} == 0 )); then diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index c0a880daf8..9551ff0f1d 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -17,9 +17,10 @@ test, so the production defaults stay the values CI actually uses while the suite still runs in seconds. -The stub also answers the workflow-run lookups the gate makes to tell which -event produced a failed check (its own run, and the run that owns each failed -check's check suite), so those are exercised by the same real script. +The stub also answers the workflow-run lookups the gate makes to rank a +check's runs by the event that produced them and to tell which event produced +a failed one (its own run, and the run that owns each such check suite), so +those are exercised by the same real script. Not provable here: whether the hosted ``Required Merge Gate`` check is attached to a branch ruleset, and whether the real API agrees with the shapes @@ -106,21 +107,24 @@ # Every workflow run an event starts is created at that event, and a re-run # keeps the run's created_at. The gate's own run (RUN_ID) was created by # THIS_EVENT unless a test says otherwise. Check suites, one per workflow run, -# are numbered by event: 1xx earlier, 2xx this one, 3xx a later one. +# are numbered by event: 1xx earlier, 2xx this one, 3xx a later one. Each +# workflow's run_number grows by one per run. GATE_RUN_ID = 12345 EARLIER_EVENT = "2026-10-05T20:17:51Z" THIS_EVENT = "2026-10-06T14:12:00Z" LATER_EVENT = "2026-10-06T14:40:00Z" +CORE_WORKFLOW, FULL_WORKFLOW, RPC_DOCS_WORKFLOW = 7001, 7002, 7003 CORE_EARLIER, FULL_EARLIER = 101, 102 CORE_THIS, FULL_THIS, RPC_DOCS_THIS = 201, 202, 203 CORE_LATER = 301 -SUITE_CREATED_AT = { - CORE_EARLIER: EARLIER_EVENT, - FULL_EARLIER: EARLIER_EVENT, - CORE_THIS: THIS_EVENT, - FULL_THIS: THIS_EVENT, - RPC_DOCS_THIS: THIS_EVENT, - CORE_LATER: LATER_EVENT, +# check suite: (workflow id, run_number, created_at) of the run that owns it +SUITE_RUNS = { + CORE_EARLIER: (CORE_WORKFLOW, 452, EARLIER_EVENT), + FULL_EARLIER: (FULL_WORKFLOW, 445, EARLIER_EVENT), + CORE_THIS: (CORE_WORKFLOW, 457, THIS_EVENT), + FULL_THIS: (FULL_WORKFLOW, 450, THIS_EVENT), + RPC_DOCS_THIS: (RPC_DOCS_WORKFLOW, 30, THIS_EVENT), + CORE_LATER: (CORE_WORKFLOW, 460, LATER_EVENT), } GATE_RUN_PATH = re.escape(f"/actions/runs/{GATE_RUN_ID}") + "$" @@ -141,9 +145,13 @@ def check_run( return run -def workflow_run(run_id: int, suite: int, created_at: str, **fields) -> dict: +def workflow_run( + run_id: int, suite: int, created_at: str | None, *, workflow: int = CORE_WORKFLOW, number: int | None = 1, **fields +) -> dict: run = { "id": run_id, + "workflow_id": workflow, + "run_number": number, "check_suite_id": suite, "created_at": created_at, "run_started_at": created_at, @@ -157,15 +165,20 @@ def workflow_runs(*runs: dict) -> str: return json.dumps({"total_count": len(runs), "workflow_runs": list(runs)}) +def suite_run(suite: int, owner: tuple[int, int, str]) -> dict: + workflow, number, created_at = owner + return workflow_run(50_000 + suite, suite, created_at, workflow=workflow, number=number) + + def event_routes( - gate_run: dict | None = None, suites: dict[int, str] | None = None + gate_run: dict | None = None, suites: dict[int, tuple[int, int, str]] | None = None ) -> dict[str, tuple]: """Answers for the gate's own run and for the run behind each check suite.""" routes: dict[str, tuple] = { GATE_RUN_PATH: ok(json.dumps(gate_run or workflow_run(GATE_RUN_ID, 999, THIS_EVENT))), } - for suite, created_at in (SUITE_CREATED_AT if suites is None else suites).items(): - routes[suite_lookup(suite)] = ok(workflow_runs(workflow_run(50_000 + suite, suite, created_at))) + for suite, owner in (SUITE_RUNS if suites is None else suites).items(): + routes[suite_lookup(suite)] = ok(workflow_runs(suite_run(suite, owner))) return routes @@ -573,8 +586,14 @@ def test_non_numeric_tunable_fails_closed_before_polling(self) -> None: # -- selection --------------------------------------------------------- def test_the_newest_check_run_wins_when_a_check_was_rerun(self) -> None: - older_failed = check_run("rpc-docs", "completed", "failure", started_at="2026-09-17T00:00:00Z") - newer_green = check_run("rpc-docs", "completed", "success", started_at="2026-09-18T00:00:00Z") + # A re-run adds a check run to the same workflow run, and so to the + # same check suite: its attempts rank by started_at alone. + older_failed = check_run( + "rpc-docs", "completed", "failure", suite=RPC_DOCS_THIS, started_at="2026-09-17T00:00:00Z" + ) + newer_green = check_run( + "rpc-docs", "completed", "success", suite=RPC_DOCS_THIS, started_at="2026-09-18T00:00:00Z" + ) for label, runs in { "newest last": [older_failed, newer_green], @@ -585,9 +604,12 @@ def test_the_newest_check_run_wins_when_a_check_was_rerun(self) -> None: completed = self.run_gate(fake, profile="rpc-docs") self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) self.assertIn("rpc-docs: status=completed conclusion=success", completed.stdout) + self.assertEqual(fake.workflow_run_calls, []) # ... and the reverse: a green first attempt must not mask a red re-run. - older_green = check_run("rpc-docs", "completed", "success", started_at="2026-09-17T00:00:00Z") + older_green = check_run( + "rpc-docs", "completed", "success", suite=RPC_DOCS_THIS, started_at="2026-09-17T00:00:00Z" + ) newer_failed = check_run( "rpc-docs", "completed", "failure", suite=RPC_DOCS_THIS, started_at="2026-09-18T00:00:00Z" ) @@ -728,7 +750,7 @@ def test_a_rerun_of_an_earlier_events_workflow_is_still_that_events(self) -> Non # created_at, so its failure, however recent, is still waited on: the # check this event started will be newer. rerun = workflow_run( - 50_101, CORE_EARLIER, EARLIER_EVENT, run_attempt=2, run_started_at="2026-10-06T15:31:19Z" + 50_101, CORE_EARLIER, EARLIER_EVENT, number=452, run_attempt=2, run_started_at="2026-10-06T15:31:19Z" ) routes = event_routes() routes[suite_lookup(CORE_EARLIER)] = ok(workflow_runs(rerun)) @@ -787,6 +809,195 @@ def test_a_later_events_failure_is_current(self) -> None: ) self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + # -- which of a check's runs decides ------------------------------------- + # + # A check has a run per event that ran CI on the commit, and a re-run adds + # another to the same workflow run. The newest workflow run's decides, at + # its latest attempt, so a manual re-run of an earlier event's workflow, + # which starts later, never outranks this event's result. + + def earlier_rerun(self, status: str, conclusion: str | None) -> dict: + if status == "queued": + return check_run("Core Checks Gate", status, conclusion, suite=CORE_EARLIER, created_at="2026-10-06T15:31:30Z") + stamps = {"started_at": "2026-10-06T15:31:30Z"} + if status == "completed": + stamps["completed_at"] = "2026-10-06T15:45:16Z" + return check_run("Core Checks Gate", status, conclusion, suite=CORE_EARLIER, **stamps) + + def this_events_core(self, status: str, conclusion: str | None) -> dict: + stamps = {"started_at": "2026-10-06T14:30:00Z"} + if status == "completed": + stamps["completed_at"] = "2026-10-06T14:44:00Z" + return check_run("Core Checks Gate", status, conclusion, suite=CORE_THIS, **stamps) + + def test_an_earlier_events_rerun_does_not_hide_this_events_success(self) -> None: + for status, conclusion in ( + ("completed", "failure"), + ("completed", "timed_out"), + ("completed", "cancelled"), + ("in_progress", None), + ("queued", None), + ): + with self.subTest(rerun=conclusion or status): + fake = FakeGh( + self.tmp(f"earlier-rerun-{conclusion or status}"), + default=ok(body(page( + self.this_events_core("completed", "success"), + self.earlier_rerun(status, conclusion), + self.green_validation(), + ))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn( + "Core Checks Gate: 2 check runs from 2 workflow runs; the newest workflow run's, " + f"created at {THIS_EVENT}, decides", + completed.stdout, + ) + self.assertIn("Core Checks Gate: status=completed conclusion=success", completed.stdout) + # A verdict on the first poll, from one lookup per check suite. + self.assertEqual(len(fake.check_run_calls), 2, fake.calls) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_EARLIER))), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_THIS))), 1, fake.calls) + self.assertEqual(fake.calls_matching(GATE_RUN_PATH), [], "a success is never dated") + + def test_an_earlier_events_rerun_does_not_hide_this_events_failure(self) -> None: + fake = FakeGh( + self.tmp("earlier-rerun-green"), + default=ok(body(page( + self.this_events_core("completed", "failure"), + self.earlier_rerun("completed", "success"), + self.green_validation(), + ))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + + def test_an_earlier_events_rerun_failure_waits_for_this_events_result(self) -> None: + # The re-run failed before this event's check existed, and stays the + # newest by started_at once it does: only this event's result decides. + rerun_failed = self.earlier_rerun("completed", "failure") + validation = self.green_validation() + fake = FakeGh( + self.tmp("earlier-rerun-then-green"), + responses=[ + ok(body(page(rerun_failed, validation))), + ok(body(page(rerun_failed, validation))), + ok(body(page(rerun_failed, self.this_events_core("in_progress", None), validation))), + ok(body(page(rerun_failed, self.this_events_core("in_progress", None), validation))), + ], + default=ok(body(page(rerun_failed, self.this_events_core("completed", "success"), validation))), + routes=event_routes(), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn("by an earlier event than this gate's", completed.stdout) + self.assertIn("Core Checks Gate: status=in_progress conclusion=pending", completed.stdout) + self.assertGreater(len(fake.check_run_calls), 4, fake.calls) + # Cached per check suite across polls: a suite's workflow run cannot change. + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_EARLIER))), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(suite_lookup(CORE_THIS))), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(GATE_RUN_PATH)), 1, fake.calls) + + def test_check_runs_that_all_succeeded_are_never_ranked(self) -> None: + fake = FakeGh( + self.tmp("all-runs-green"), + default=ok(body(page( + self.this_events_core("completed", "success"), + self.earlier_rerun("completed", "success"), + self.green_validation(), + ))), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertEqual(fake.workflow_run_calls, []) + + def test_runs_created_in_the_same_second_rank_by_run_number(self) -> None: + # Two labels applied at once start two runs of a workflow in the same + # second; the later one has the higher run_number, whichever check + # started last. + first, second = 401, 402 + suites = { + first: (CORE_WORKFLOW, 456, THIS_EVENT), + second: (CORE_WORKFLOW, 457, THIS_EVENT), + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + for label, (newer, older) in { + "the newer run failed": ("failure", "success"), + "the newer run passed": ("success", "failure"), + }.items(): + with self.subTest(case=label): + fake = FakeGh( + self.tmp(f"same-second-{newer}"), + default=ok(body(page( + check_run("Core Checks Gate", "completed", newer, suite=second, started_at="2026-10-06T14:20:00Z"), + check_run("Core Checks Gate", "completed", older, suite=first, started_at="2026-10-06T15:31:30Z"), + self.green_validation(), + ))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake) + if newer == "failure": + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + else: + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertEqual(len(fake.check_run_calls), 1 if newer == "failure" else 2, fake.calls) + + def test_check_runs_that_cannot_be_ranked_fail_closed(self) -> None: + # Which of several workflow runs is newest must be read, never guessed. + server_error = (1, "", "gh: Server Error (HTTP 502)\n") + earlier_failed = self.failed_core(CORE_EARLIER, started_at=EARLIER_EVENT, completed_at=EARLIER_EVENT) + this_running = self.this_events_core("in_progress", None) + unranked = ( + f"{ERROR_UNCORRELATED}Core Checks Gate on {'0' * 40} has check runs from 2 workflow runs, but the " + "workflow runs that tell which event is newest could not be read; failing closed with no verdict " + "for: Core Checks Gate Full Validation Gate" + ) + no_number = workflow_run(50_201, CORE_THIS, THIS_EVENT, number=None) + cases = { + "a check suite's run cannot be read": ( + {suite_lookup(CORE_THIS): server_error}, this_running, unranked, suite_lookup(CORE_THIS), + ), + "a check suite's run has no run_number": ( + {suite_lookup(CORE_THIS): ok(workflow_runs(no_number))}, this_running, unranked, suite_lookup(CORE_THIS), + ), + "a check run names no check suite": ( + {}, + check_run("Core Checks Gate", "in_progress", None, started_at="2026-10-06T14:30:00Z"), + f"{ERROR_UNCORRELATED}Core Checks Gate on {'0' * 40} has 2 check runs, but not every one names " + "the workflow run that tells which event is newest; failing closed with no verdict for: " + "Core Checks Gate Full Validation Gate", + None, + ), + } + for index, (label, (overrides, other, expected, gave_up)) in enumerate(cases.items()): + with self.subTest(case=label): + routes = event_routes() + routes.update(overrides) + fake = FakeGh( + self.tmp(f"unranked-{index}"), + default=ok(body(page(earlier_failed, other, self.green_validation()))), + routes=routes, + ) + completed = self.run_gate(fake, GATE_QUERY_MAX_ATTEMPTS="2") + self.assertGateFailed( + completed, expected, forbidden=(ERROR_CHECK_FAILED, ERROR_TIMED_OUT, ERROR_UNREADABLE) + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + if gave_up is not None: + self.assertEqual(len(fake.calls_matching(gave_up)), 2, fake.calls) + def test_the_event_skew_allowance_is_a_documented_constant(self) -> None: self.assertEqual(self.event_skew, DOCUMENTED_EVENT_SKEW_SECONDS) self.assertNotIn("EVENT_SKEW_SECONDS:-", self.script, "the allowance must not be overridable") @@ -809,7 +1020,7 @@ def test_the_event_skew_boundary(self) -> None: fake = FakeGh( self.tmp(f"skew-{label}"), default=ok(body(page(self.failed_core(777), self.green_validation()))), - routes=event_routes(suites={777: iso(created)}), + routes=event_routes(suites={777: (CORE_WORKFLOW, 1, iso(created))}), ) completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="2") if expected == "current": @@ -910,6 +1121,10 @@ def test_a_failure_that_cannot_be_dated_fails_closed_with_a_distinct_error(self) {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, None)))}, failed, {}, suite_route, ), + "the check's run has no run_number": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, THIS_EVENT, number=None)))}, + failed, {}, suite_route, + ), "the check's run has fractional seconds": ( {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, "2026-10-06T14:12:00.5Z")))}, failed, {}, suite_route, @@ -991,10 +1206,10 @@ def test_a_queued_rerun_falls_back_to_created_at_and_is_pending(self) -> None: # A re-run that has not started yet has no started_at at all; ordering # by created_at keeps it, so an old green run cannot pass the gate. old_green = check_run( - "rpc-docs", "completed", "success", + "rpc-docs", "completed", "success", suite=RPC_DOCS_THIS, started_at="2026-09-17T00:00:00Z", created_at="2026-09-17T00:00:00Z", ) - queued_rerun = check_run("rpc-docs", "queued", None, created_at="2026-09-18T00:00:00Z") + queued_rerun = check_run("rpc-docs", "queued", None, suite=RPC_DOCS_THIS, created_at="2026-09-18T00:00:00Z") fake = FakeGh(self.tmp("queued-rerun"), default=ok(body(page(old_green, queued_rerun)))) completed = self.run_gate(fake, profile="rpc-docs", GATE_WAIT_TIMEOUT_SECONDS="3") self.assertIn("rpc-docs: status=queued conclusion=pending", completed.stdout) From e92b46b91911da3ef80b9e74c684bbb3687a9547 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Wed, 21 Oct 2026 12:34:52 -0400 Subject: [PATCH 8/8] ci: tell close events apart by a newer run of the check's workflow The gate took any failure whose workflow run was created within 60 s of its own as this event's. Two events that close together (a label, then an unlabel or a reopen) made the second gate fail at once on the first event's result. That is the usual case, not an edge one: the newer event cancels the earlier event's Core Checks and Full Validation, and their if: always() gate jobs then conclude failure (7 of 8 cancelled Core Checks runs sampled here did), after the newer event has started. GitHub's REST API names no event on a workflow run, check suite or check run, so there is no marker to compare. It does give each workflow's runs a run_number that grows with every new run and is kept by re-runs. Before failing at once, the gate now lists the check's workflow's runs on the commit (GET /actions/workflows/{id}/runs ?head_sha=): if one has a higher run_number, an event since has started the run that replaces this one, and the failure is waited on. That tells events apart at any spacing, even within the same second. Tighten the allowance from 60 s to 10 s. Across 325 pull request events on this repository (June to October 2026), one event's runs were created within 4 s of each other, and the gate's up to 3 s after another's, so exact equality would misdate a current failure; the closest two events on one commit were 28 s apart. - A current failure with no newer run of its workflow still fails at once, including after a gate-only re-run. - A current failure whose workflow already has a newer run, from a later event, is waited on, and that run's result decides. - A newer run, once listed, is cached; a listing that can't be read, or lacks the check's own run, fails closed. - Left open: two events within 10 s where the earlier one's check fails and the later one's run is not yet listed when the gate judges it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/required-merge-gate.yml | 156 +++++++++++--- ci/checks/test_merge_gate_poller.py | 235 +++++++++++++++++++++- 2 files changed, 350 insertions(+), 41 deletions(-) diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index 999cf9f8db..5dc513e2af 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -904,12 +904,22 @@ jobs: # cancelled, or fail, after it started; and re-running this gate # alone starts it again but starts nothing else. # - # What the checks of one event share is the event. Every workflow - # run a push, pull request event or merge group starts is created at - # that event, with the same created_at to within a second or two, - # and a re-run keeps the workflow run: its id, its check suite and - # its created_at stay, and only run_attempt and run_started_at move - # (GitHub's REST API, checked on this repository's PR #229). So: + # What the checks of one event share is the event, but GitHub's REST + # API names it nowhere: a workflow run, check suite or check run + # carries no delivery id or activity type. What it does give: + # + # * every workflow run a push, pull request event or merge group + # starts is created at that event, within seconds of the others + # (see EVENT_SKEW_SECONDS for how close), and a re-run keeps the + # workflow run: its id, run_number, check suite and created_at + # stay, and only run_attempt and run_started_at move (checked on + # this repository's PR #229); + # * a workflow's run_number grows with each new run and is kept by a + # re-run (GitHub's contexts reference), so of two runs of one + # workflow on this commit the higher run_number is the later + # event's, even when both were created in the same second. + # + # So: # # * a check run's check suite names the workflow run that produced # it (GET /actions/runs?check_suite_id=), whose created_at dates @@ -920,7 +930,25 @@ jobs: # judges, so a gate-only re-run still judges its original event; # * a check whose workflow run was created more than # EVENT_SKEW_SECONDS before this gate's belongs to an earlier event; - # any other belongs to this event or a later one and is current. + # * any other belongs to this event or a later one, unless a newer + # run of its workflow is on this commit (GET + # /actions/workflows/{id}/runs?head_sha=): then an event since, + # however soon after, has started the run that replaces it. + # + # That newer run is the usual case, not an edge one. A newer event + # cancels an earlier event's Core Checks and Full Validation runs + # (their concurrency groups cancel in progress), whose gate jobs + # still run (if: always()) and conclude failure: of 8 such cancelled + # Core Checks runs sampled here, 7 did. So the earlier event's + # failure lands after the newer event, and the newer run already + # exists, having caused the cancellation. + # + # What is left: two events whose runs are created within + # EVENT_SKEW_SECONDS of each other, the earlier one's check failing, + # and the later one's run of that workflow not yet listed when this + # gate judges it. That failure is taken as current and fails at once. + # And a current failure whose workflow has a newer run, from an event + # after this gate's, is waited on: that run's result decides. # # Only a check that concluded neither success nor cancelled is # dated against this gate: a success on this commit passes whichever @@ -928,11 +956,14 @@ jobs: # a check's runs is judged at all is judge_check's to say, below. # # Workflow runs started by one event are created this close together. - # A documented allowance, not a tunable: a larger value takes more of - # an earlier event's failures as current (they fail at once rather - # than wait), a smaller one risks taking a current failure as earlier - # (it is waited on until the deadline). - EVENT_SKEW_SECONDS=60 + # Across 325 pull request events on this repository (June to October + # 2026), one event's runs were created within 4 s of each other, and + # this gate's run up to 3 s after another's; the closest two events + # on one commit were 28 s apart. A documented allowance, not a + # tunable: a smaller value risks taking a current failure as earlier + # (it is waited on until the deadline), a larger one widens the + # window left above, where only the newer run tells events apart. + EVENT_SKEW_SECONDS=10 # GET /actions/runs/{run_id}: this gate's own workflow run. # $run_id is bound by --argjson below. @@ -955,14 +986,34 @@ jobs: | map(if (.workflow_runs | type) == "array" then .workflow_runs[] else error("malformed workflow-runs page") end) | map(select(.check_suite_id == $suite)) | if length != 1 then error("\(length) workflow runs own check suite \($suite)") else .[0] end - | [(.created_at | fromdateiso8601 | tostring), (.run_number | tostring), .created_at] + | [(.created_at | fromdateiso8601 | tostring), (.run_number | tostring), (.workflow_id | tostring), .created_at] + | @tsv + ' + # "" + suite_run_pattern=$'^[0-9]+\t[1-9][0-9]*\t[1-9][0-9]*\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + # GET /actions/workflows/{id}/runs?head_sha=: a workflow's runs on + # this commit, and of them the one with the highest run_number. Runs + # of another workflow or commit are never counted, even if the + # filters were ignored, and a listing that lacks the failed check's + # own run is not trusted. $workflow and $suite are bound by + # --argjson, $sha by --arg. + # shellcheck disable=SC2016 + newest_run_filter=' + if type != "array" or length == 0 then error("empty workflow-runs response") else . end + | map(if (.workflow_runs | type) == "array" then .workflow_runs[] else error("malformed workflow-runs page") end) + | map(select(.workflow_id == $workflow and .head_sha == $sha)) + | if any(.[]; .check_suite_id == $suite) then . else error("check suite \($suite) is not among the runs of workflow \($workflow) on \($sha)") end + | if all(.[]; (.run_number | type) == "number") then . else error("a run of workflow \($workflow) has no run_number") end + | max_by(.run_number) + | [(.run_number | tostring), .created_at] | @tsv ' - # "" - suite_run_pattern=$'^[0-9]+\t[1-9][0-9]*\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + # "" + newest_run_pattern=$'^[1-9][0-9]*\t[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' GATE_RUN_CREATED="" declare -A SUITE_RUN=() + declare -A SUITE_NEWER_RUN=() # Read this gate's own workflow run once, into GATE_RUN_CREATED # (""). Returns 1 if it could not be read by @@ -1002,29 +1053,57 @@ jobs: # Date the event of check $1, from check suite $2 (see above). # - # Sets CHECK_EVENT to "earlier" or "current" and CHECK_RUN_CREATED_AT - # to its workflow run's created_at, and returns 0. Returns 1 when the - # check run names no check suite, or this gate's run or the check's - # run could not be read by $3, the shared deadline: an event that - # cannot be dated is never success, never earlier and never current. + # Sets CHECK_EVENT to "earlier", "superseded" (a newer run of its + # workflow is on this commit, named in CHECK_NEWER_RUN as + # "") or "current", and CHECK_RUN_NUMBER + # and CHECK_RUN_CREATED_AT to its own workflow run's, and returns 0. + # Returns 1 when the check run names no check suite, or a workflow + # run or listing it needs could not be read by $3, the shared + # deadline: an event that cannot be dated is never success, never + # earlier and never current. date_check_event() { local check_name="$1" local check_suite_id="$2" local deadline="$3" - local check_created gate_created + local check_created gate_created workflow_id newest_number CHECK_EVENT="" + CHECK_RUN_NUMBER="" CHECK_RUN_CREATED_AT="" + CHECK_NEWER_RUN="" date_gate_run "${check_name}" "${deadline}" || return 1 date_check_suite "${check_name}" "${check_suite_id}" "${deadline}" || return 1 check_created="${SUITE_RUN[${check_suite_id}]}" CHECK_RUN_CREATED_AT="${check_created##*$'\t'}" + CHECK_RUN_NUMBER="${check_created#*$'\t'}" + workflow_id="${CHECK_RUN_NUMBER#*$'\t'}" + CHECK_RUN_NUMBER="${CHECK_RUN_NUMBER%%$'\t'*}" + workflow_id="${workflow_id%%$'\t'*}" check_created="${check_created%%$'\t'*}" gate_created="${GATE_RUN_CREATED%%$'\t'*}" if (( check_created < gate_created - EVENT_SKEW_SECONDS )); then CHECK_EVENT="earlier" + return 0 + fi + + # A newer run, once listed, stays on the commit, so it is cached; + # its absence is read when the verdict hangs on it, and a failure + # with no newer run is a verdict that ends the wait. + if [[ -z "${SUITE_NEWER_RUN[${check_suite_id}]:-}" ]]; then + read_api "${check_name}" "workflow-runs" "${deadline}" \ + "/repos/${REPOSITORY}/actions/workflows/${workflow_id}/runs?head_sha=${SHA}&exclude_pull_requests=true&per_page=100" \ + "${newest_run_pattern}" "${newest_run_filter}" \ + --argjson workflow "${workflow_id}" --argjson suite "${check_suite_id}" --arg sha "${SHA}" || return 1 + newest_number="${API_RESULT%%$'\t'*}" + if (( newest_number > CHECK_RUN_NUMBER )); then + SUITE_NEWER_RUN[${check_suite_id}]="${API_RESULT}" + fi + fi + if [[ -n "${SUITE_NEWER_RUN[${check_suite_id}]:-}" ]]; then + CHECK_NEWER_RUN="${SUITE_NEWER_RUN[${check_suite_id}]}" + CHECK_EVENT="superseded" else CHECK_EVENT="current" fi @@ -1145,14 +1224,26 @@ jobs: CHECK_UNCORRELATED="${check_name} on ${SHA} concluded ${conclusion:-unknown}, but the workflow runs that tell whether it belongs to this event could not be read" return 1 fi - if [[ "${CHECK_EVENT}" == "earlier" ]]; then - echo "${check_name}: concluded ${conclusion:-unknown} in a workflow run created at ${CHECK_RUN_CREATED_AT}, by an earlier event than this gate's (${GATE_RUN_CREATED#*$'\t'}); waiting for a newer run" - CHECK_PENDING="${check_name} (last ${conclusion:-unknown} from an earlier event)" - CHECK_VERDICT="pending" - return 0 - fi - CHECK_CONCLUSION="${conclusion}" - CHECK_VERDICT="failed" + case "${CHECK_EVENT}" in + earlier) + echo "${check_name}: concluded ${conclusion:-unknown} in a workflow run created at ${CHECK_RUN_CREATED_AT}, by an earlier event than this gate's (${GATE_RUN_CREATED#*$'\t'}); waiting for a newer run" + CHECK_PENDING="${check_name} (last ${conclusion:-unknown} from an earlier event)" + CHECK_VERDICT="pending" + ;; + superseded) + echo "${check_name}: concluded ${conclusion:-unknown} in run #${CHECK_RUN_NUMBER} of its workflow, created at ${CHECK_RUN_CREATED_AT}, but run #${CHECK_NEWER_RUN%%$'\t'*}, created at ${CHECK_NEWER_RUN#*$'\t'}, has since started on ${SHA}; waiting for its result" + CHECK_PENDING="${check_name} (last ${conclusion:-unknown} superseded by a newer run)" + CHECK_VERDICT="pending" + ;; + current) + CHECK_CONCLUSION="${conclusion}" + CHECK_VERDICT="failed" + ;; + *) + CHECK_UNCORRELATED="${check_name} on ${SHA} concluded ${conclusion:-unknown} in an event that could not be told ('${CHECK_EVENT}')" + return 1 + ;; + esac } # Wait until every named check on ${SHA} has concluded successfully. @@ -1163,9 +1254,10 @@ jobs: # deadline passed with checks still pending. Only a completed check # with a success conclusion is success. A check that concluded # otherwise is a verdict only if it belongs to the event this gate - # judges, or a later one; an earlier event's is waited on, as its - # newer run may not have reported yet, and a cancelled one is always - # waited on. + # judges, or a later one, and no newer run of its workflow is on this + # commit; an earlier event's, or a superseded one, is waited on, as + # its newer run may not have reported yet, and a cancelled one is + # always waited on. wait_for_checks() { local check_names=("$@") local pending=() diff --git a/ci/checks/test_merge_gate_poller.py b/ci/checks/test_merge_gate_poller.py index 9551ff0f1d..1d1e6cb2b8 100755 --- a/ci/checks/test_merge_gate_poller.py +++ b/ci/checks/test_merge_gate_poller.py @@ -98,9 +98,12 @@ ERROR_TIMED_OUT = "::error title=Required Merge Gate timed out::" WARNING_RETRYING = "::warning title=Required Merge Gate retrying::" -# The documented allowance between the workflow runs one event creates. +# The documented allowance between the workflow runs one event creates: 325 +# pull request events on this repository (June to October 2026) spread their +# runs over at most 4 s, and the closest two events on one commit were 28 s +# apart. EVENT_SKEW = re.compile(r"^\s*EVENT_SKEW_SECONDS=(?P\d+)\s*$", re.MULTILINE) -DOCUMENTED_EVENT_SKEW_SECONDS = 60 +DOCUMENTED_EVENT_SKEW_SECONDS = 10 # -- the events a commit has seen ------------------------------------------- # @@ -117,16 +120,19 @@ CORE_EARLIER, FULL_EARLIER = 101, 102 CORE_THIS, FULL_THIS, RPC_DOCS_THIS = 201, 202, 203 CORE_LATER = 301 -# check suite: (workflow id, run_number, created_at) of the run that owns it +# check suite: (workflow id, run_number, created_at) of the run that owns it. +# These are the workflow runs on the commit unless a test says otherwise; the +# later event's run is on it only in the tests that ask for it. SUITE_RUNS = { CORE_EARLIER: (CORE_WORKFLOW, 452, EARLIER_EVENT), FULL_EARLIER: (FULL_WORKFLOW, 445, EARLIER_EVENT), CORE_THIS: (CORE_WORKFLOW, 457, THIS_EVENT), FULL_THIS: (FULL_WORKFLOW, 450, THIS_EVENT), RPC_DOCS_THIS: (RPC_DOCS_WORKFLOW, 30, THIS_EVENT), - CORE_LATER: (CORE_WORKFLOW, 460, LATER_EVENT), } +LATER_RUNS = {CORE_LATER: (CORE_WORKFLOW, 460, LATER_EVENT)} +SHA = "0" * 40 GATE_RUN_PATH = re.escape(f"/actions/runs/{GATE_RUN_ID}") + "$" @@ -135,6 +141,11 @@ def suite_lookup(suite: int) -> str: return re.escape(f"/actions/runs?check_suite_id={suite}&exclude_pull_requests=true&per_page=100") + "$" +def commit_runs_lookup(workflow: int) -> str: + """The route for the listing of ``workflow``'s runs on the commit.""" + return re.escape(f"/actions/workflows/{workflow}/runs?head_sha={SHA}&exclude_pull_requests=true&per_page=100") + "$" + + def check_run( name: str, status: str, conclusion: str | None = None, *, suite: int | None = None, **stamps: str ) -> dict: @@ -153,6 +164,7 @@ def workflow_run( "workflow_id": workflow, "run_number": number, "check_suite_id": suite, + "head_sha": SHA, "created_at": created_at, "run_started_at": created_at, "run_attempt": 1, @@ -173,12 +185,17 @@ def suite_run(suite: int, owner: tuple[int, int, str]) -> dict: def event_routes( gate_run: dict | None = None, suites: dict[int, tuple[int, int, str]] | None = None ) -> dict[str, tuple]: - """Answers for the gate's own run and for the run behind each check suite.""" + """Answers for the gate's own run, for the run behind each check suite, and + for each workflow's runs on the commit: exactly the runs in ``suites``.""" routes: dict[str, tuple] = { GATE_RUN_PATH: ok(json.dumps(gate_run or workflow_run(GATE_RUN_ID, 999, THIS_EVENT))), } + by_workflow: dict[int, list[dict]] = {} for suite, owner in (SUITE_RUNS if suites is None else suites).items(): routes[suite_lookup(suite)] = ok(workflow_runs(suite_run(suite, owner))) + by_workflow.setdefault(owner[0], []).append(suite_run(suite, owner)) + for workflow, runs in by_workflow.items(): + routes[commit_runs_lookup(workflow)] = ok(workflow_runs(*runs)) return routes @@ -347,7 +364,7 @@ def gate_env(self, fake: FakeGh, profile: str, **overrides: str) -> dict[str, st "SOURCE_VALIDATION_REQUIRED": "true", "GH_TOKEN": "fake-token", "REPOSITORY": "example/repo", - "SHA": "0" * 40, + "SHA": SHA, "RUN_ID": str(GATE_RUN_ID), } declared = set(self.step["env"]) @@ -799,7 +816,7 @@ def test_a_later_events_failure_is_current(self) -> None: fake = FakeGh( self.tmp("later-event"), default=ok(body(page(self.failed_core(CORE_LATER, started_at=LATER_EVENT), self.green_validation()))), - routes=event_routes(), + routes=event_routes(suites={**SUITE_RUNS, **LATER_RUNS}), ) completed = self.run_gate(fake) self.assertGateFailed( @@ -1038,6 +1055,180 @@ def test_the_event_skew_boundary(self) -> None: forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED), ) + # -- two events on one commit, close together ---------------------------- + # + # No marker names a workflow run's event, so events are told apart by when + # their runs were created and, closer together than the allowance, by a + # newer run of the same workflow on the commit. A newer event cancels the + # earlier event's run, whose always() gate job then concludes failure, + # after the newer run exists. + + def earlier_events_failure(self, suite: int) -> dict: + # Concluded after this event, as a cancelled run's gate job does. + return check_run( + "Core Checks Gate", "completed", "failure", suite=suite, + started_at="2026-10-06T14:12:40Z", completed_at="2026-10-06T14:12:45Z", + ) + + def test_an_earlier_event_seconds_before_waits_for_this_events_run(self) -> None: + # 0 s is two labels applied at once. Inside the allowance only the + # newer run of the workflow tells the earlier event's failure apart. + gate = parse_iso(THIS_EVENT) + first = 401 + for seconds in (0, 2, DOCUMENTED_EVENT_SKEW_SECONDS): + with self.subTest(seconds_apart=seconds): + created = iso(gate - timedelta(seconds=seconds)) + suites = { + first: (CORE_WORKFLOW, 456, created), + CORE_THIS: SUITE_RUNS[CORE_THIS], + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + stale = self.earlier_events_failure(first) + validation = self.green_validation() + running = self.this_events_core("in_progress", None) + fake = FakeGh( + self.tmp(f"apart-{seconds}"), + responses=[ok(body(page(stale, validation)))] * 4 + [ok(body(page(stale, running, validation)))] * 2, + default=ok(body(page(stale, self.this_events_core("completed", "success"), validation))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertIn( + f"Core Checks Gate: concluded failure in run #456 of its workflow, created at {created}, " + f"but run #457, created at {THIS_EVENT}, has since started on {SHA}; waiting for its result", + completed.stdout, + ) + self.assertIn("Core Checks Gate: status=in_progress conclusion=pending", completed.stdout) + # Listed once: a newer run, once seen, stays on the commit. + self.assertEqual(len(fake.calls_matching(commit_runs_lookup(CORE_WORKFLOW))), 1, fake.calls) + + def test_a_superseded_failure_is_named_by_the_timeout(self) -> None: + gate = parse_iso(THIS_EVENT) + suites = { + 401: (CORE_WORKFLOW, 456, iso(gate - timedelta(seconds=2))), + CORE_THIS: SUITE_RUNS[CORE_THIS], + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + fake = FakeGh( + self.tmp("superseded-timeout"), + default=ok(body(page(self.earlier_events_failure(401), self.green_validation()))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="3") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 3s and are still pending: " + "Core Checks Gate (last failure superseded by a newer run)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED, ERROR_UNREADABLE), + ) + + def test_an_earlier_event_30_seconds_before_is_told_apart_by_time(self) -> None: + # Outside the allowance the failure is an earlier event's whether or + # not this event's run of the workflow is listed yet, and no listing + # is read for it. + gate = parse_iso(THIS_EVENT) + first = 401 + for label, listed in {"this event's run listed": True, "not listed yet": False}.items(): + with self.subTest(case=label): + suites = { + first: (CORE_WORKFLOW, 456, iso(gate - timedelta(seconds=30))), + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + if listed: + suites[CORE_THIS] = SUITE_RUNS[CORE_THIS] + fake = FakeGh( + self.tmp(f"apart-30-{listed}"), + default=ok(body(page(self.earlier_events_failure(first), self.green_validation()))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake, GATE_WAIT_TIMEOUT_SECONDS="2") + self.assertGateFailed( + completed, + f"{ERROR_TIMED_OUT}Required checks did not complete within 2s and are still pending: " + "Core Checks Gate (last failure from an earlier event)", + forbidden=(ERROR_CHECK_FAILED, ERROR_UNCORRELATED), + ) + self.assertEqual(fake.calls_matching(commit_runs_lookup(CORE_WORKFLOW)), [], fake.calls) + + def test_a_current_failure_with_no_newer_run_fails_at_once(self) -> None: + # One event's runs are created seconds apart, this gate's up to 3 s + # after another's: such a failure is this event's, and with no newer + # run of its workflow on the commit it fails at once, also when only + # this gate is re-run, hours later. + gate = parse_iso(THIS_EVENT) + rerun = workflow_run(GATE_RUN_ID, 999, THIS_EVENT, run_attempt=3, run_started_at="2026-10-06T19:40:00Z") + for label, gate_run in {"first attempt": None, "gate-only re-run": rerun}.items(): + for seconds in (3, DOCUMENTED_EVENT_SKEW_SECONDS): + with self.subTest(gate=label, seconds_before=seconds): + suites = { + CORE_EARLIER: SUITE_RUNS[CORE_EARLIER], + 401: (CORE_WORKFLOW, 457, iso(gate - timedelta(seconds=seconds))), + FULL_THIS: SUITE_RUNS[FULL_THIS], + } + fake = FakeGh( + self.tmp(f"current-{label}-{seconds}"), + default=ok(body(page(self.failed_core(401), self.green_validation()))), + routes=event_routes(gate_run=gate_run, suites=suites), + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + self.assertEqual(len(fake.check_run_calls), 1, fake.calls) + self.assertEqual(len(fake.calls_matching(commit_runs_lookup(CORE_WORKFLOW))), 1, fake.calls) + + def test_runs_of_another_workflow_or_commit_never_supersede(self) -> None: + # Even were the listing's filters ignored. + routes = event_routes() + routes[commit_runs_lookup(CORE_WORKFLOW)] = ok(workflow_runs( + suite_run(CORE_THIS, SUITE_RUNS[CORE_THIS]), + workflow_run(60_001, 901, LATER_EVENT, workflow=FULL_WORKFLOW, number=999), + dict(workflow_run(60_002, 902, LATER_EVENT, number=999), head_sha="1" * 40), + )) + fake = FakeGh( + self.tmp("other-runs"), + default=ok(body(page(self.failed_core(CORE_THIS), self.green_validation()))), + routes=routes, + ) + completed = self.run_gate(fake) + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + + def test_a_later_events_run_decides_over_this_events_failure(self) -> None: + # An event after this gate's has already started a newer run of the + # workflow: this event's failure is waited on, and that run decides. + suites = {**SUITE_RUNS, **LATER_RUNS} + validation = self.green_validation() + this_failed = self.failed_core(CORE_THIS) + for conclusion in ("success", "failure"): + with self.subTest(later=conclusion): + later = check_run( + "Core Checks Gate", "completed", conclusion, suite=CORE_LATER, + started_at="2026-10-06T14:50:00Z", completed_at="2026-10-06T15:04:00Z", + ) + fake = FakeGh( + self.tmp(f"later-decides-{conclusion}"), + responses=[ok(body(page(this_failed, validation)))] * 2, + default=ok(body(page(this_failed, later, validation))), + routes=event_routes(suites=suites), + ) + completed = self.run_gate(fake) + self.assertIn("but run #460, created at " + LATER_EVENT, completed.stdout) + if conclusion == "success": + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + else: + self.assertGateFailed( + completed, + f"{ERROR_CHECK_FAILED}Core Checks Gate concluded failure.", + forbidden=(ERROR_TIMED_OUT, ERROR_UNCORRELATED), + ) + def test_a_success_from_an_earlier_event_still_counts(self) -> None: # A success on this commit is a verdict whichever event produced it, # so it is never dated: no workflow run is read for it. @@ -1104,6 +1295,9 @@ def test_a_failure_that_cannot_be_dated_fails_closed_with_a_distinct_error(self) forbidden = (1, "", "gh: Resource not accessible by integration (HTTP 403)\n") failed = self.failed_core(CORE_THIS) suite_route = suite_lookup(CORE_THIS) + own_run = suite_run(CORE_THIS, SUITE_RUNS[CORE_THIS]) + dated = {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(own_run))} + listing = commit_runs_lookup(CORE_WORKFLOW) # case: (routes, check run, env overrides, the route read until it gave up) cases = { "the check's run cannot be read": ({GATE_RUN_PATH: gate_run, suite_route: server_error}, failed, {}, suite_route), @@ -1125,6 +1319,10 @@ def test_a_failure_that_cannot_be_dated_fails_closed_with_a_distinct_error(self) {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, THIS_EVENT, number=None)))}, failed, {}, suite_route, ), + "the check's run names no workflow": ( + {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, THIS_EVENT, workflow=None)))}, + failed, {}, suite_route, + ), "the check's run has fractional seconds": ( {GATE_RUN_PATH: gate_run, suite_route: ok(workflow_runs(workflow_run(1, CORE_THIS, "2026-10-06T14:12:00.5Z")))}, failed, {}, suite_route, @@ -1138,6 +1336,25 @@ def test_a_failure_that_cannot_be_dated_fails_closed_with_a_distinct_error(self) {GATE_RUN_PATH: ok(json.dumps(workflow_run(GATE_RUN_ID, 999, "")))}, failed, {}, GATE_RUN_PATH, ), "this gate's run id is missing": ({}, failed, {"RUN_ID": ""}, None), + # Whether a newer run of the check's workflow is on the commit. + "its workflow's runs cannot be listed": ({**dated, listing: server_error}, failed, {}, listing), + "the token cannot list its workflow's runs": ({**dated, listing: forbidden}, failed, {}, listing), + "the listing is not json": ({**dated, listing: ok("502")}, failed, {}, listing), + "the listing is empty": ({**dated, listing: ok(workflow_runs())}, failed, {}, listing), + "the listing lacks the check's own run": ( + {**dated, listing: ok(workflow_runs(suite_run(CORE_EARLIER, SUITE_RUNS[CORE_EARLIER])))}, + failed, {}, listing, + ), + "the listing has the run under another workflow": ( + {**dated, listing: ok(workflow_runs(dict(own_run, workflow_id=FULL_WORKFLOW)))}, failed, {}, listing, + ), + "the listing has the run under another commit": ( + {**dated, listing: ok(workflow_runs(dict(own_run, head_sha="1" * 40)))}, failed, {}, listing, + ), + "a listed run has no run_number": ( + {**dated, listing: ok(workflow_runs(own_run, workflow_run(1, 999, LATER_EVENT, number=None)))}, + failed, {}, listing, + ), "the check run names no check suite": ( {GATE_RUN_PATH: gate_run}, check_run("Core Checks Gate", "completed", "failure", started_at="2026-10-06T14:20:00Z"), @@ -1163,13 +1380,13 @@ def test_a_failure_that_cannot_be_dated_fails_closed_with_a_distinct_error(self) if gave_up is not None: # Bounded by the same retries as the check-runs query. self.assertEqual(len(fake.calls_matching(gave_up)), 2, fake.calls) - self.assertIn(f"{WARNING_RETRYING}Core Checks Gate: workflow-run ", completed.stdout) + self.assertIn(f"{WARNING_RETRYING}Core Checks Gate: workflow-run", completed.stdout) def test_a_transient_lookup_failure_is_retried(self) -> None: routes = event_routes() routes[suite_lookup(CORE_THIS)] = [ (1, "", "gh: Server Error (HTTP 502)\n"), - ok(workflow_runs(workflow_run(60_201, CORE_THIS, THIS_EVENT))), + ok(workflow_runs(suite_run(CORE_THIS, SUITE_RUNS[CORE_THIS]))), ] fake = FakeGh( self.tmp("lookup-transient"),