diff --git a/packages/app/package.json b/packages/app/package.json index c9eed66ae..e178d55a8 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -26,7 +26,7 @@ "measure:fuzz": "bash scripts/measure-fuzz.sh", "measure:stress": "bash scripts/measure-stress.sh", "perf:compare": "bash scripts/perf-compare.sh", - "test:e2e": "playwright test tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-compact-folders.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/lint-config-toggle.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts tests/stress/qa-canary-authoring-both-modes.e2e.ts tests/stress/qa-canary-live-typing.e2e.ts tests/stress/keystroke-cadence-danger-space.e2e.ts tests/stress/jsx-unregistered-backspace-delete.e2e.ts tests/stress/jsx-unregistered-ime-concurrent.e2e.ts tests/stress/jsx-backspace-delete.e2e.ts tests/stress/link-authoring-bytes.e2e.ts tests/stress/link-authoring-apex.e2e.ts tests/stress/command-palette-parity.e2e.ts tests/stress/settings-search.e2e.ts", + "test:e2e": "playwright test tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/lint-config-toggle.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts tests/stress/qa-canary-authoring-both-modes.e2e.ts tests/stress/qa-canary-live-typing.e2e.ts tests/stress/keystroke-cadence-danger-space.e2e.ts tests/stress/jsx-unregistered-backspace-delete.e2e.ts tests/stress/jsx-unregistered-ime-concurrent.e2e.ts tests/stress/jsx-backspace-delete.e2e.ts tests/stress/link-authoring-bytes.e2e.ts tests/stress/link-authoring-apex.e2e.ts tests/stress/command-palette-parity.e2e.ts tests/stress/settings-search.e2e.ts tests/stress/chip-popover-positioning.e2e.ts tests/stress/docs-open-scroll-restore.e2e.ts tests/stress/file-tree-decoration-chip-strip-repro.e2e.ts tests/stress/grip-click-nodeselect.e2e.ts tests/stress/jsx-halo-semantics.e2e.ts tests/stress/jsx-image-body-click.e2e.ts tests/stress/jsx-prop-panel-placeholder.e2e.ts tests/stress/jsx-range-encompass-halo.e2e.ts tests/stress/jsx-wildcard-convert.e2e.ts tests/stress/markdown-lint.e2e.ts tests/stress/mid-type-recovery.e2e.ts tests/stress/observer-a-multi-client.e2e.ts tests/stress/outline-toolbar-occlusion.e2e.ts tests/stress/quiet-tree-hidden-doc.e2e.ts tests/stress/raw-mdx-fallback-onblur-upgrade.e2e.ts tests/stress/rename-noext-probe.e2e.ts tests/stress/show-ok-folders.e2e.ts tests/stress/source-polish.e2e.ts tests/stress/warm-skeleton-scroll-restore.e2e.ts tests/stress/yjs-no-dual-import.e2e.ts", "test:e2e:install-browsers": "playwright install chromium webkit firefox", "test:visual": "playwright test --config playwright.visual.config.ts", "test:visual:update": "playwright test --config playwright.visual.config.ts --update-snapshots", diff --git a/packages/app/tests/meta/e2e-ci-membership.test.ts b/packages/app/tests/meta/e2e-ci-membership.test.ts new file mode 100644 index 000000000..a3833439f --- /dev/null +++ b/packages/app/tests/meta/e2e-ci-membership.test.ts @@ -0,0 +1,241 @@ +/** + * `test:e2e` membership meta-guard. + * + * CI's Playwright tier runs the explicit file enumeration in packages/app + * package.json `test:e2e` — Playwright treats CLI file args as filters, so a + * stress e2e file missing from that list (or a listed file that was later + * renamed/deleted) is silently invisible: no failure, no skip, no signal. + * Before this guard, 32 of 83 `tests/stress/*.e2e.ts` files had drifted into + * that invisible set, including one deterministically failing test. + * + * Every `tests/stress/*.e2e.ts` file must therefore be in EXACTLY ONE of: + * 1. the `test:e2e` enumeration (runs in CI), or + * 2. the exclusion ledger (`tests/stress/e2e-ci-ledger.ts`) with a reviewed + * reason + local-run evidence. + * + * The guard also fails on staleness in either direction: a ledger entry whose + * file no longer exists, or an enumerated file that no longer exists (a + * deleted file left in the script matches nothing and shrinks CI coverage + * without any red). + * + * The membership predicate is extracted pure and exercised against planted + * fixtures below — an absence-checker without a planted positive is a vacuous + * green waiting to happen (same pattern as e2e-stop-rules.test.ts). + */ + +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, test } from 'vitest'; +import { E2E_CI_EXCLUSIONS } from '../stress/e2e-ci-ledger'; + +const APP_ROOT = join(import.meta.dirname, '..', '..'); +const STRESS_DIR = join(APP_ROOT, 'tests', 'stress'); +const PKG_JSON_PATH = join(APP_ROOT, 'package.json'); +const LEDGER_HINT = + 'add the file to the test:e2e script in packages/app/package.json (runs in CI) ' + + 'OR add a ledger entry with reason + evidence in packages/app/tests/stress/e2e-ci-ledger.ts'; + +/** Bare filenames of every on-disk stress e2e file. */ +function listStressE2eFiles(): string[] { + return readdirSync(STRESS_DIR) + .filter((name) => name.endsWith('.e2e.ts')) + .sort(); +} + +/** + * Relative paths of `.e2e.ts` files that sit in a tests/stress SUBDIRECTORY. + * Playwright discovers its testDir recursively, so such a file runs under + * `playwright test` — yet the membership predicate keys on top-level bare + * filenames, so a nested file falls outside the guard entirely: CI-invisible AND + * unguarded, the exact class this guard exists to catch. Pure over its input (a + * recursive readdir) so the sanity check can plant fixtures. The guard is bounded + * to top-level files by convention — e2e files live at the top of tests/stress/, + * subdirs hold helpers + fixtures — so a hit means either flatten the file or + * extend the guard + ledger to carry subpaths. + */ +function nestedE2ePaths(relPaths: readonly string[]): string[] { + return relPaths + .filter((p) => p.endsWith('.e2e.ts') && /[/\\]/.test(p)) + .map((p) => p.replace(/\\/g, '/')) + .sort(); +} + +/** + * Bare filenames enumerated by the `test:e2e` script. Only `tests/stress/` + * tokens are governed here — other suites (visual, a11y) run under their own + * configs and are out of scope. + */ +function parseEnumeratedFiles(script: string): string[] { + return script + .split(/\s+/) + .filter((token) => token.startsWith('tests/stress/') && token.endsWith('.e2e.ts')) + .map((token) => token.slice('tests/stress/'.length)); +} + +interface MembershipViolations { + /** On disk, but in neither the enumeration nor the ledger — CI-invisible. */ + unlisted: string[]; + /** In BOTH the enumeration and the ledger — contradictory state. */ + dual: string[]; + /** Ledger entries whose file no longer exists on disk. */ + staleLedger: string[]; + /** Enumerated files that no longer exist on disk. */ + staleEnumeration: string[]; +} + +/** + * Pure membership predicate over the three sets, so the planted-fixture + * self-test below can exercise every violation class without touching the + * real package.json or ledger. + */ +function computeMembershipViolations( + onDisk: readonly string[], + enumerated: readonly string[], + ledgered: readonly string[], +): MembershipViolations { + const diskSet = new Set(onDisk); + const enumSet = new Set(enumerated); + const ledgerSet = new Set(ledgered); + return { + unlisted: onDisk.filter((f) => !enumSet.has(f) && !ledgerSet.has(f)), + dual: onDisk.filter((f) => enumSet.has(f) && ledgerSet.has(f)), + staleLedger: ledgered.filter((f) => !diskSet.has(f)), + staleEnumeration: enumerated.filter((f) => !diskSet.has(f)), + }; +} + +function loadRealSets(): { onDisk: string[]; enumerated: string[]; ledgered: string[] } { + const pkg = JSON.parse(readFileSync(PKG_JSON_PATH, 'utf-8')) as { + scripts?: Record; + }; + const script = pkg.scripts?.['test:e2e'] ?? ''; + return { + onDisk: listStressE2eFiles(), + enumerated: parseEnumeratedFiles(script), + ledgered: E2E_CI_EXCLUSIONS.map((entry) => entry.file), + }; +} + +describe('test:e2e membership meta-guard', () => { + test('the guard has inputs (sanity: stress files exist, enumeration parses)', () => { + const { onDisk, enumerated } = loadRealSets(); + expect(onDisk.length).toBeGreaterThan(0); + // An empty parse means the test:e2e script shape changed (e.g. moved to a + // directory/glob invocation). If the whole stress dir now runs in CI, + // retire this guard and the ledger together; otherwise fix the parser. + expect(enumerated.length).toBeGreaterThan(0); + // A nested *.e2e.ts runs under Playwright's recursive testDir scan yet + // escapes the flat top-level membership sets — fail loud if one appears. + const nested = nestedE2ePaths(readdirSync(STRESS_DIR, { recursive: true })); + expect(nested).toEqual([]); + }); + + test('every tests/stress/*.e2e.ts is in the test:e2e enumeration or the exclusion ledger', () => { + const { onDisk, enumerated, ledgered } = loadRealSets(); + const { unlisted } = computeMembershipViolations(onDisk, enumerated, ledgered); + if (unlisted.length > 0) { + throw new Error( + `CI-invisible stress e2e file(s) — ${LEDGER_HINT}:\n${unlisted + .map((f) => ` tests/stress/${f}`) + .join('\n')}`, + ); + } + }); + + test('no file is in BOTH the test:e2e enumeration and the exclusion ledger', () => { + const { onDisk, enumerated, ledgered } = loadRealSets(); + const { dual } = computeMembershipViolations(onDisk, enumerated, ledgered); + if (dual.length > 0) { + throw new Error( + `File(s) present in both the test:e2e enumeration and the ledger — a promoted file must have its ledger entry deleted:\n${dual + .map((f) => ` tests/stress/${f}`) + .join('\n')}`, + ); + } + }); + + test('every ledger entry points at a file that still exists', () => { + const { onDisk, enumerated, ledgered } = loadRealSets(); + const { staleLedger } = computeMembershipViolations(onDisk, enumerated, ledgered); + if (staleLedger.length > 0) { + throw new Error( + `Stale ledger entr(ies) — the file no longer exists; delete the entry from e2e-ci-ledger.ts:\n${staleLedger + .map((f) => ` ${f}`) + .join('\n')}`, + ); + } + }); + + test('every enumerated test:e2e file still exists', () => { + const { onDisk, enumerated, ledgered } = loadRealSets(); + const { staleEnumeration } = computeMembershipViolations(onDisk, enumerated, ledgered); + if (staleEnumeration.length > 0) { + throw new Error( + `Stale test:e2e entr(ies) — the file no longer exists on disk. Playwright file args are filters, so a stale entry silently matches nothing; remove it from the script:\n${staleEnumeration + .map((f) => ` tests/stress/${f}`) + .join('\n')}`, + ); + } + }); + + test('ledger entries are unique and carry non-empty reason + evidence', () => { + const problems: string[] = []; + const seen = new Set(); + for (const entry of E2E_CI_EXCLUSIONS) { + if (seen.has(entry.file)) problems.push(` duplicate entry: ${entry.file}`); + seen.add(entry.file); + if (entry.reason.trim() === '') problems.push(` empty reason: ${entry.file}`); + if (entry.evidence.trim() === '') problems.push(` empty evidence: ${entry.file}`); + } + if (problems.length > 0) { + throw new Error(`Ledger hygiene violation(s):\n${problems.join('\n')}`); + } + }); + + test('membership predicate fires on planted violations and not on adjacent negatives', () => { + // Clean state: disk fully partitioned across the two lists → no violations. + const clean = computeMembershipViolations( + ['a.e2e.ts', 'b.e2e.ts', 'c.e2e.ts'], + ['a.e2e.ts', 'b.e2e.ts'], + ['c.e2e.ts'], + ); + expect(clean.unlisted).toEqual([]); + expect(clean.dual).toEqual([]); + expect(clean.staleLedger).toEqual([]); + expect(clean.staleEnumeration).toEqual([]); + + // Planted: d.e2e.ts on disk but in neither list → CI-invisible. + const unlisted = computeMembershipViolations(['a.e2e.ts', 'd.e2e.ts'], ['a.e2e.ts'], []); + expect(unlisted.unlisted).toEqual(['d.e2e.ts']); + + // Planted: a.e2e.ts in both lists → contradictory. + const dual = computeMembershipViolations(['a.e2e.ts'], ['a.e2e.ts'], ['a.e2e.ts']); + expect(dual.dual).toEqual(['a.e2e.ts']); + expect(dual.unlisted).toEqual([]); + + // Planted: ledger references a deleted file → stale ledger. + const staleLedger = computeMembershipViolations(['a.e2e.ts'], ['a.e2e.ts'], ['gone.e2e.ts']); + expect(staleLedger.staleLedger).toEqual(['gone.e2e.ts']); + + // Planted: enumeration references a deleted file → stale enumeration. + const staleEnum = computeMembershipViolations(['a.e2e.ts'], ['a.e2e.ts', 'gone.e2e.ts'], []); + expect(staleEnum.staleEnumeration).toEqual(['gone.e2e.ts']); + }); + + test('parseEnumeratedFiles reads only tests/stress e2e tokens from the script', () => { + const parsed = parseEnumeratedFiles( + 'playwright test tests/stress/a.e2e.ts tests/visual/v.e2e.ts tests/stress/b.e2e.ts --grep foo', + ); + expect(parsed).toEqual(['a.e2e.ts', 'b.e2e.ts']); + }); + + test('nestedE2ePaths flags subdirectory e2e files and ignores top-level + non-e2e', () => { + expect( + nestedE2ePaths(['a.e2e.ts', '_helpers/b.e2e.ts', 'fixtures/data.json', 'c.e2e.ts']), + ).toEqual(['_helpers/b.e2e.ts']); + // Windows-style separators normalize to '/'. + expect(nestedE2ePaths(['_fixtures\\d.e2e.ts'])).toEqual(['_fixtures/d.e2e.ts']); + // Purely top-level input has no nested offenders. + expect(nestedE2ePaths(['a.e2e.ts', 'b.e2e.ts'])).toEqual([]); + }); +}); diff --git a/packages/app/tests/stress/e2e-ci-ledger.ts b/packages/app/tests/stress/e2e-ci-ledger.ts new file mode 100644 index 000000000..68c55af58 --- /dev/null +++ b/packages/app/tests/stress/e2e-ci-ledger.ts @@ -0,0 +1,108 @@ +/** + * CI-exclusion ledger for `tests/stress/*.e2e.ts`. + * + * CI's Playwright tier runs the fixed file list in this package's `test:e2e` + * script — not a glob. Any stress e2e file absent from that list NEVER runs in + * CI, silently. This ledger is the explicit, reviewed record of every file + * that is deliberately NOT in the CI list, each with a reason and the local + * run evidence backing it. The membership meta-guard + * (`tests/meta/e2e-ci-membership.test.ts`) fails when a stress e2e file is in + * neither the CI list nor this ledger, when a file is in both, or when either + * side references a file that no longer exists. + * + * To promote a ledgered file into CI: verify it is deterministic-green on + * consecutive local runs and cheap enough for the 15-minute CI job, append it + * to the `test:e2e` script in package.json, and delete its entry here. + */ + +export interface E2eCiLedgerEntry { + /** Bare filename under tests/stress/, e.g. 'edit-with-ai.e2e.ts'. */ + file: string; + /** Why the file is excluded from the CI `test:e2e` list. */ + reason: string; + /** Observed local-run evidence backing the reason (verdict + wall-clock). */ + evidence: string; +} + +export const E2E_CI_EXCLUSIONS: readonly E2eCiLedgerEntry[] = [ + { + file: 'asset-embed-real-fidelity.e2e.ts', + reason: + 'fails-locally: QA-004 (real ZIP) and QA-006 (real CSV) byte-identity assertions fail — on-disk sha256/content does not match the uploaded bytes', + evidence: 'same 2 tests failed in 2/2 local runs (2026-07-20); other 5 tests green', + }, + { + file: 'clipboard-relative-url-source-fallback.e2e.ts', + reason: + 'fails-locally: QA-005 inline image in a paragraph emits chunk-wrapper HTML instead of the inline markdown source-fallback; deterministic, fix open in PR #2505', + evidence: 'QA-005 failed deterministically in local run (2026-07-20); remaining tests green', + }, + { + file: 'edit-with-ai.e2e.ts', + reason: + 'fails-locally: all 13 tests fail toBeVisible — Edit-with-AI rows are desktop-gated (window.okDesktop) and never render in the browser e2e harness without a forced-gate init script', + evidence: '13/18 failed, uniform toBeVisible timeouts, local run (2026-07-20)', + }, + { + file: 'frontmatter-edit.e2e.ts', + reason: + 'fails-locally: 5 PropertyPanel assertions fail (rename keeps position, drag reorder, keyboard drag, duplicate-name marker, malformed-YAML banner)', + evidence: 'same 5 tests failed in 2/2 local runs (2026-07-20)', + }, + { + file: 'graph-panel-surfaces.e2e.ts', + reason: + 'fails-locally + ci-budget: fullscreen-graph interaction tests hit the 120s per-test timeout one after another even solo at low machine load; a solo run exceeds 15 minutes', + evidence: + '11 failed under batch run; solo re-run at low load kept failing each test at the 120s cap and was abandoned after ~15m (2026-07-20)', + }, + { + file: 'list-keymap.e2e.ts', + reason: + 'fails-locally: Tab/Shift-Tab list-depth tests receive a flat list (Tab no longer indents) and ordered-item Enter assertion fails', + evidence: 'same 3 tests failed in 2/2 local runs (2026-07-20); other 12 green, 1 skipped', + }, + { + file: 'okignore-settings.e2e.ts', + reason: + 'fails-locally: 6 tests fail (US-010 advanced-textarea group + US-013 hide-file/folder patterns); also expensive at ~2.9m solo', + evidence: '6 failed / 13 passed solo at low load, 2.9m wall (2026-07-20)', + }, + { + file: 'outline-navigation.e2e.ts', + reason: + 'fails-locally: outline-click landing assertions fail (WYSIWYG scroll landing, source-mode cursor line, code-fence # disambiguation)', + evidence: '4/5 failed in re-run at low load (2026-07-20)', + }, + { + file: 'prop-upload.e2e.ts', + reason: + 'fails-locally: 5 assertions expect src "initial.png" but receive "/initial.png" (leading-slash drift in upload src handling)', + evidence: 'same 5 tests failed in 2/2 local runs (2026-07-20)', + }, + { + file: 'rename-consolidation.e2e.ts', + reason: + 'fails-locally: 3 browser-fidelity assertions fail (rename response contract returns undefined where true expected; /api/history reachability)', + evidence: + 'same 3 tests failed in 2/2 local runs (2026-07-20); tmpdir content dir is not a git repo, rename falls back to fs rename', + }, + { + file: 'rename-content-preservation.e2e.ts', + reason: + 'fails-locally: "renaming the active doc keeps navigation on the renamed doc" fails toBeGreaterThan(0) on 0', + evidence: 'same single test failed in 2/2 local runs (2026-07-20); other 3 green', + }, + { + file: 'slash-command-auto-open.e2e.ts', + reason: + 'fails-locally: SLASH-AUTOOPEN-IMG-MULTI expects 2 [data-jsx-component] nodes after inserting a second Image, receives 1', + evidence: 'same single test failed in 2/2 local runs (2026-07-20); other 21 tests green', + }, + { + file: 'timeline-diff-sidepane.e2e.ts', + reason: + 'fails-locally: all 14 side-pane tests fail element-not-found / toBeVisible — the timeline diff side pane never appears in the harness', + evidence: '14/16 failed in 2/2 local runs (2026-07-20); also expensive (>4m per run)', + }, +];