From 32d435e121d7c2473544d7d674d445326cd5b951 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 15 Sep 2026 09:03:23 -0500 Subject: [PATCH] fix(ci): quarantine flaky completion-observer-coder test (#2692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quarantines tests/unit/background/completion-observer-coder.test.ts in the Windows-only ledger (single-OS evidence: windows-latest unit-shard 1 retry-pass; sibling ubuntu/macos shards green in CI run 34410703321 2026-09-09T22:14:39Z). Files: - scripts/ci/quarantined-tests-windows.txt — append entry with OWNER + EXPIRY metadata (issue #2477 Check 7), bump STATUS header to 3. - tests/unit/scripts/ci/ci-yml-quarantine-2692.test.ts — 4 pinning tests proving ledger placement, scope isolation, on-disk path presence, and OWNER/EXPIRY grammar. - docs/releases/pending/issue-2692-quarantine-completion-observer-coder-windows.md — release fragment. Flakiness pattern: every test builds a real git repository via spawnSync("git", ...) with a 5s timeout and drives async completion-observer events; slow windows-latest runners or FS contention can push git spawns / past-HEAD checks over the line. ci.yml retries discard attempt-1 output on retry-pass so no assertion text is captured. File passes locally (9/9) — environment-sensitive, not a logic bug. Tracked under #1782 (test-stability sprint). Closure of #1729 quarantine debt remains paid down. --- ...ntine-completion-observer-coder-windows.md | 67 +++++++++ scripts/ci/quarantined-tests-windows.txt | 34 ++++- .../scripts/ci/ci-yml-quarantine-2692.test.ts | 133 ++++++++++++++++++ 3 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 docs/releases/pending/issue-2692-quarantine-completion-observer-coder-windows.md create mode 100644 tests/unit/scripts/ci/ci-yml-quarantine-2692.test.ts diff --git a/docs/releases/pending/issue-2692-quarantine-completion-observer-coder-windows.md b/docs/releases/pending/issue-2692-quarantine-completion-observer-coder-windows.md new file mode 100644 index 000000000..c587b8d03 --- /dev/null +++ b/docs/releases/pending/issue-2692-quarantine-completion-observer-coder-windows.md @@ -0,0 +1,67 @@ +# Quarantine flaky completion-observer-coder test on Windows (issue #2692) + +## What changed + +- Appended one new entry to the Windows-only CI quarantine ledger, + `scripts/ci/quarantined-tests-windows.txt`: + - `tests/unit/background/completion-observer-coder.test.ts` + (windows-latest unit-shard 1, passed-on-retry flake) +- Updated the ledger's `# STATUS: N active entries` header from 2 to 3. +- Added a 4-test pinning regression file + (`tests/unit/scripts/ci/ci-yml-quarantine-2692.test.ts`) covering ledger + placement, scope isolation (windows-only), on-disk path presence, and the + OWNER/EXPIRY metadata grammar required by `scripts/check-invariants.ts` + Check 7 (issue #2477). +- No source, hook, or workflow code changed. The change is confined to the + ledger file, the new pinning test file, and this pending release fragment. + +## Why + +Issue #2692 was auto-filed by the flake-detection workflow (issue #1782, +`.github/workflows/flake-detection.yml`) after merge-group CI run 34410703321 +(`merge_group` pr-2644, head `740b66aa4`, 2026-09-09T22:08:51Z) produced one +flake annotation: + +``` +::notice file=tests/unit/background/completion-observer-coder.test.ts::Passed on retry 1 (flaky): tests/unit/background/completion-observer-coder.test.ts +``` + +Tracing the detection job back to its upstream CI run (via the +`Fetching artifact list for workflow run N` log line in +flake-detection.yml), the annotation artifact +`flake-annotations-unit-shard-1` was uploaded by `unit (windows-latest, 1)` +and its job log shows `Attempt 1 failed, retrying (1/2)` → `Passed on retry 1` +(windows-only passed-on-retry flake). Sibling shards in the same run ran the +same file green: `unit (ubuntu-latest, 1)` (1.182s) and `unit (macos-latest, +1)` (2.414s). The evidence is single-OS, so the entry goes in the Windows +ledger — the general ledger would suppress the file on ubuntu/macos too and +the macOS ledger applies only on macOS runners. + +The test file is integration-heavy: every case builds a real git repository +via `spawnSync('git', ...)` with a 5s timeout +(`tests/unit/background/completion-observer-coder.test.ts:26-39`) and drives +async completion-observer events, so slow windows-latest runners or +filesystem contention can push the git spawns/past-HEAD checks over the line. +The ci.yml retry loop discards attempt-1 output when a retry passes, so no +assertion text exists to drive a root-cause fix. The file passes locally +(this checkout), confirming the flake is environment-sensitive rather than a +logic bug. + +## Migration steps + +None. This is a CI test-skip data change — no runtime, config, or API change. + +## Known caveats + +- The quarantined suite pins safety-critical coder settlement behavior + (HEAD-drift staleness fencing, terminal-claim idempotency, ingestion + fencing). It is skipped only on windows-latest merge-group/CI unit shards; + ubuntu and macOS continue to run it, and the Windows skip has an EXPIRY of + 2026-10-31 with a root-fix criterion. The EXPIRY is what forces the + retirement conversation; `scripts/check-invariants.ts` Check 7 hard-fails + the CI gate once the EXPIRY passes the 14-day grace window. +- `#1737`/`#1782`/`#2477` (the historic quarantine-debt trackers) are CLOSED; + live tracking refs are this issue #2692 (per-flake) and #1782 is the sprint + issue referenced in the entry prose for continuity. +- The entry carries `# OWNER: zaxbysauce` and `# EXPIRY: 2026-10-31` metadata + per the issue #2477 grammar; re-add/edit must preserve both lines. diff --git a/scripts/ci/quarantined-tests-windows.txt b/scripts/ci/quarantined-tests-windows.txt index 9a19bb90e..f6eb335d6 100644 --- a/scripts/ci/quarantined-tests-windows.txt +++ b/scripts/ci/quarantined-tests-windows.txt @@ -12,9 +12,10 @@ # gate; an EXPIRY more than 14 days in the past also fails (inside the grace # window it only warns). # -# STATUS: 2 active entries (pr-monitor-status.test.ts per issue #1982 and -# win32-wrapper-runtime.test.ts per issue #2185 — the issue #1729 debt itself -# remains paid down; see the entries below). +# STATUS: 3 active entries (pr-monitor-status.test.ts per issue #1982, +# win32-wrapper-runtime.test.ts per issue #2185, and +# completion-observer-coder.test.ts per issue #2692 — the issue #1729 debt +# itself remains paid down; see the entries below). # # History: this list previously held 9 Windows-only failures. They were fixed # in issue #1729 by a combination of: @@ -94,3 +95,30 @@ tests/unit/commands/pr-monitor-status.test.ts # EXPIRY: 2026-10-01 — root-fix the PowerShell cold-spawn flake or renew with # an updated diagnosis under #1737/#2477 tests/unit/sandbox/win32-wrapper-runtime.test.ts +# +# completion-observer-coder.test.ts (issue #2692) +# --------------------------------------------------------------------------- +# Integration tests for the background-coder completion ingestion path +# (trusted fresh completion wires exact parent state/files/evidence/advisory, +# HEAD-drift staleness fencing, terminal-claim idempotency, review-state +# overwrite protection, shared-root attribution, replay fencing-token +# rotation, fenced-commit failure). Every case builds a real git repository +# via spawnSync git init/config/add/commit with a 5s timeout +# (tests/unit/background/completion-observer-coder.test.ts:26-39) and drives +# async completion-observer events, so slow windows-latest runners or +# filesystem contention can push the git spawns/past-HEAD checks over the +# line. Flaky on windows-latest merge-group unit-shard 1 only: CI run +# 34410703321 (merge_group pr-2644, head 740b66aa4, 2026-09-09T22:08:51Z) had +# Attempt 1 fail and Passed on retry 1 (passed-on-retry flake) at +# 2026-09-09T22:14:39Z, while sibling ubuntu-latest shard 1 (1.182s) and +# macos-latest shard 1 (2.414s) ran the same file green in the same run. +# The ci.yml retry loop discards attempt-1 output when a retry passes, so no +# assertion text exists to drive a root-cause fix. Pre-existing, unrelated +# to any specific PR. Quarantined per issue #2692; tracked under #1782 +# (test-stability sprint). +# OWNER: zaxbysauce — issue #2692 (windows-latest-only merge-group flake; +# single-OS evidence: windows-1 retry-pass, ubuntu-1/macos-1 green) +# EXPIRY: 2026-10-31 — root-fix the windows git-spawn/observer-timing flake +# or renew with an updated criterion. Inside the 14-day grace window +# Check 7 only warns. +tests/unit/background/completion-observer-coder.test.ts diff --git a/tests/unit/scripts/ci/ci-yml-quarantine-2692.test.ts b/tests/unit/scripts/ci/ci-yml-quarantine-2692.test.ts new file mode 100644 index 000000000..54bffba2a --- /dev/null +++ b/tests/unit/scripts/ci/ci-yml-quarantine-2692.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, test } from 'bun:test'; +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +// Regression pinning tests for the issue #2692 quarantine entry. +// +// Issue #2692 was auto-filed by the flake-detection workflow (issue #1782, +// .github/workflows/flake-detection.yml) after merge-group CI run 34410703321 +// (merge_group pr-2644, head 740b66aa4, 2026-09-09T22:08:51Z) produced one +// flake annotation: +// - tests/unit/background/completion-observer-coder.test.ts +// (windows-latest unit-shard 1, Attempt 1 failed → Passed on retry 1; +// passed-on-retry flake; sibling ubuntu-latest shard 1 and +// macos-latest shard 1 ran the same file green in the same run) +// +// The entry belongs in the windows-only ledger (single-OS evidence): the +// general ledger applies on every RUNNER_OS and would suppress the file on +// ubuntu/macos too, which the evidence does not justify. +// +// Each pinning test below reads the real ledger files off disk and asserts +// the entry is present in the windows ledger and absent from the others, +// that the on-disk test file exists, and that the entry block carries the +// OWNER/EXPIRY metadata required by Check 7 (issue #2477). + +const REPO_ROOT = join(import.meta.dir, '../../../..'); +const GENERAL_LEDGER_PATH = join( + REPO_ROOT, + 'scripts/ci/quarantined-tests.txt', +); +const MACOS_LEDGER_PATH = join( + REPO_ROOT, + 'scripts/ci/quarantined-tests-macos.txt', +); +const WINDOWS_LEDGER_PATH = join( + REPO_ROOT, + 'scripts/ci/quarantined-tests-windows.txt', +); +const INTEGRATION_LEDGER_PATH = join( + REPO_ROOT, + 'scripts/ci/quarantined-integration-tests.txt', +); + +const COMPLETION_OBSERVER_CODER = + 'tests/unit/background/completion-observer-coder.test.ts'; + +// Mirror ci.yml's active-entry extraction exactly: +// grep -vE '^\s*#|^\s*$' scripts/ci/quarantined-tests-.txt +// (CRLF is normalized first so the assertion holds on any checkout config.) +function activeEntries(ledgerPath: string): string[] { + const raw = readFileSync(ledgerPath, 'utf8').replace(/\r\n/g, '\n'); + return raw + .split('\n') + .filter((line: string) => !/^\s*#/.test(line) && !/^\s*$/.test(line)) + .map((line: string) => line.trim()); +} + +describe('ci.yml integration — quarantine ledger entry for issue #2692 merge-group flake detection', () => { + test('completion-observer-coder.test.ts is an active entry in the windows ledger', () => { + // Without this entry, the flake-detection script keeps re-filing the + // candidate (Rule A only drops already-quarantined files). The flake + // originated on windows-latest unit-shard 1 (CI run 34410703321, + // 2026-09-09T22:14:39Z, Attempt 1 failed → Passed on retry 1). + expect(existsSync(WINDOWS_LEDGER_PATH)).toBe(true); + expect(activeEntries(WINDOWS_LEDGER_PATH)).toContain( + COMPLETION_OBSERVER_CODER, + ); + }); + + test('is scoped to the windows ledger only (single-OS evidence)', () => { + // The general ledger applies on every RUNNER_OS — listing it there + // would suppress the file on ubuntu/macos too, but the evidence is + // windows-only (ubuntu-latest shard 1 ran the file green in 1.18s and + // macos-latest shard 1 in 2.41s in the same CI run 34410703321). + expect(activeEntries(GENERAL_LEDGER_PATH)).not.toContain( + COMPLETION_OBSERVER_CODER, + ); + expect(activeEntries(MACOS_LEDGER_PATH)).not.toContain( + COMPLETION_OBSERVER_CODER, + ); + if (existsSync(INTEGRATION_LEDGER_PATH)) { + expect(activeEntries(INTEGRATION_LEDGER_PATH)).not.toContain( + COMPLETION_OBSERVER_CODER, + ); + } + }); + + test('the quarantined path exists and is discovered by the ci.yml find chain', () => { + // A typo'd ledger path would be a silent no-op: CI's comm -23 gated set + // would never exclude it (the path never appears in all-tests.txt) and + // the flake would keep re-filing. The discovery chain globs + // tests/unit/**/*.test.ts, so the on-disk file must exist at exactly + // the ledger path relative to the repo root. + expect(existsSync(join(REPO_ROOT, COMPLETION_OBSERVER_CODER))).toBe( + true, + ); + }); + + test('the entry block carries OWNER + EXPIRY metadata (issue #2477 Check 7)', () => { + // Check 7 hard-fails any active entry missing OWNER/EXPIRY or with + // EXPIRY past the 14-day grace window. Reading the raw file here is + // more direct than invoking check:invariants and gives a focused, + // fast RED signal if the next entry edit drops one of the two + // required lines. + const raw = readFileSync(WINDOWS_LEDGER_PATH, 'utf8').replace( + /\r\n/g, + '\n', + ); + const lines = raw.split('\n'); + const entryIdx = lines.findIndex( + (l: string) => l.trim() === COMPLETION_OBSERVER_CODER, + ); + expect(entryIdx).toBeGreaterThan(-1); + const blockAbove: string[] = []; + for (let i = entryIdx - 1; i >= 0; i -= 1) { + const above = lines[i] ?? ''; + if (above.trim() === '' || /^\s*#/.test(above)) { + blockAbove.push(above); + } else { + break; + } + } + const block = blockAbove.join('\n'); + expect(block).toMatch(/^#\s*OWNER:\s*\S.*$/m); + const expiry = block.match(/^#\s*EXPIRY:\s*(\d{4}-\d{2}-\d{2})\b/m); + expect(expiry).not.toBeNull(); + const [y, m, d] = (expiry?.[1] ?? '').split('-').map(Number); + expect(y).toBeGreaterThan(2020); + expect(m).toBeGreaterThanOrEqual(1); + expect(m).toBeLessThanOrEqual(12); + expect(d).toBeGreaterThanOrEqual(1); + expect(d).toBeLessThanOrEqual(31); + }); +});