diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index 149dd30152..0e807302f0 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -26,6 +26,46 @@ import { fileURLToPath } from 'node:url'; const scriptPath = fileURLToPath(import.meta.url); const defaultRepoRoot = dirname(dirname(scriptPath)); +/** npm gate scripts whose `scripts/*.test.mjs` inputs select a planner lane. */ +export const GATE_SCRIPT_LANES = [ + ['check:release', 'releaseContract'], + ['check:asf-source', 'asfSource'], +]; + +const SCRIPT_TEST_FILE_PATTERN = /scripts\/[\w.-]+\.test\.mjs/gu; +const NPM_RUN_SCRIPT_PATTERN = /\bnpm run ([\w:-]+)/gu; + +export function extractScriptTestFiles(command) { + return [...new Set(command.match(SCRIPT_TEST_FILE_PATTERN) ?? [])]; +} + +export function collectGateScriptTestFiles(scriptName, scripts, visited = new Set()) { + if (visited.has(scriptName)) return []; + visited.add(scriptName); + const command = scripts[scriptName]; + if (typeof command !== 'string') return []; + + const files = extractScriptTestFiles(command); + for (const match of command.matchAll(NPM_RUN_SCRIPT_PATTERN)) { + files.push(...collectGateScriptTestFiles(match[1], scripts, visited)); + } + return [...new Set(files)]; +} + +export function loadGateTestFiles(repoRoot = defaultRepoRoot, readFile = readFileSync) { + const { scripts } = JSON.parse(readFile(join(repoRoot, 'package.json'), 'utf8')); + const byLane = { + releaseContract: new Set(), + asfSource: new Set(), + }; + for (const [scriptName, lane] of GATE_SCRIPT_LANES) { + for (const file of collectGateScriptTestFiles(scriptName, scripts)) { + byLane[lane].add(file); + } + } + return byLane; +} + const FULL_SUITE_FILES = new Set([ '.github/workflows/ci.yml', 'package-lock.json', @@ -56,27 +96,19 @@ const RELEASE_CONTRACT_FILES = new Set([ 'scripts/package-windows-autoupdate-next.mjs', 'scripts/package-windows-x64.mjs', 'scripts/prepare-windows-upgrade-baseline.mjs', - 'scripts/prepare-windows-upgrade-baseline.test.mjs', - 'scripts/generate-third-party-notices.test.mjs', - 'scripts/product-release.test.mjs', - 'scripts/qualify-released-cli-state-root.test.mjs', 'scripts/release-eval-smoke-sitecustomize.py', 'scripts/release-version.mjs', - 'scripts/third-party-closure.test.mjs', 'scripts/verify-macos-arm64-cli.mjs', 'scripts/verify-macos-arm64-dmg.mjs', 'scripts/verify-macos-autoupdate.mjs', 'scripts/desktop-update-contract.mjs', 'scripts/product-nightly.mjs', - 'scripts/product-nightly.test.mjs', 'scripts/verify-packaged-app.mjs', - 'scripts/verify-packaged-app.test.mjs', 'scripts/verify-windows-autoupdate.mjs', 'scripts/verify-windows-installer-lifecycle.mjs', 'scripts/verify-windows-x64.mjs', 'scripts/windows-upgrade-baseline.json', 'scripts/windows-package-source-closure.mjs', - 'scripts/windows-package-source-closure.test.mjs', ]); // What decides whether a build can read durable state an earlier release wrote. @@ -127,19 +159,15 @@ const ASF_SOURCE_FILES = new Set([ 'package.json', 'packages/eval/harbor/deepseek-harness-profile/cordis.patch.yml', 'scripts/asf-license-headers.mjs', - 'scripts/asf-license-headers.test.mjs', 'scripts/asf-source-release.mjs', - 'scripts/asf-source-release.test.mjs', - 'scripts/asf-source-workflow-policy.test.mjs', 'scripts/model-metadata/models-dev-api.snapshot.json', - 'scripts/source-legal-inventory.test.mjs', 'scripts/sync-model-metadata.mjs', - 'scripts/sync-model-metadata.test.mjs', ]); -function isAsfSourcePath(path) { +function isAsfSourcePath(path, gateTestFiles) { return ( ASF_SOURCE_FILES.has(path) || + gateTestFiles.asfSource.has(path) || path.startsWith('patches/') || path.startsWith('apps/desktop/resources/licenses/renderer/') || path.startsWith('apps/desktop/src/renderer/assets/provider-brands/') @@ -166,9 +194,10 @@ function isCliPackagePath(path) { ); } -function isReleaseContractPath(path) { +function isReleaseContractPath(path, gateTestFiles) { return ( RELEASE_CONTRACT_FILES.has(path) || + gateTestFiles.releaseContract.has(path) || path.startsWith('scripts/desktop-nightly') || path.startsWith('scripts/product-release-') || path.startsWith('scripts/release-cli-') @@ -380,6 +409,8 @@ function workspaceLanes(workspaces, graph) { export function planTests(changedFiles, options = {}) { const graph = options.graph ?? loadWorkspaceGraph(options.repoRoot); + const gateTestFiles = + options.gateTestFiles ?? loadGateTestFiles(options.repoRoot, options.readFile); const files = [...new Set(changedFiles.map(normalizePath).filter(Boolean))]; const forceFull = options.forceFull ?? false; const full = forceFull || files.some((path) => FULL_SUITE_FILES.has(path)); @@ -457,7 +488,7 @@ export function planTests(changedFiles, options = {}) { const cliPackage = files.some((path) => isCliPackagePath(path)); return { appIcons: files.some((path) => isAppIconPath(path)), - asfSource: files.some((path) => isAsfSourcePath(path)), + asfSource: files.some((path) => isAsfSourcePath(path, gateTestFiles)), astryxSurface: files.some((path) => isAstryxSurfaceInventoryPath(path)), cliPackage, code, @@ -466,7 +497,8 @@ export function planTests(changedFiles, options = {}) { // boots, and packages/ui unit-test-only PRs must not either. e2e: files.some((path) => isE2eProductPath(path)), full: false, - releaseContract: cliPackage || files.some((path) => isReleaseContractPath(path)), + releaseContract: + cliPackage || files.some((path) => isReleaseContractPath(path, gateTestFiles)), // packages/cli/src/__tests__/runtime-host-session-driver.test.ts executes real sandboxed // shell tools, so the bubblewrap + user-namespace setup is required whenever // the cli workspace runs in the dependency closure, not only for direct diff --git a/scripts/ci-test-plan.test.mjs b/scripts/ci-test-plan.test.mjs index a9d03ddeb0..bd4e729c9d 100644 --- a/scripts/ci-test-plan.test.mjs +++ b/scripts/ci-test-plan.test.mjs @@ -33,7 +33,10 @@ import test from 'node:test'; import { changedFilesBetween, + collectGateScriptTestFiles, formatGitHubOutputs, + GATE_SCRIPT_LANES, + loadGateTestFiles, planTests, requiresHeavyValidation, } from './ci-test-plan.mjs'; @@ -248,28 +251,41 @@ test('release authority changes select their dedicated contract gate', () => { assert.equal(planTests(['.github/RELEASE_CHECKLIST.md'], { graph }).releaseContract, false); }); -// Derived from the gate scripts themselves, because the sets above are hand -// maintained and drift silently in both directions: a test the gate runs but -// no lane selects can be edited green, and a listed path that no longer exists -// is dead weight nothing reports. Both had happened — three of the release -// gate's own tests reached no lane, and the set named a -// `prepare-windows-upgrade-baseline.test.mjs` that never existed. -test('every test a gate script runs reaches a lane that runs that gate', () => { - const { scripts } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +// Gate-owned contract tests are inputs to their lane because the planner reads +// them from the gate script text in package.json — not from a parallel list that +// can drift in either direction. +test('gate script test files select the lane that gate script runs', () => { + const gateTestFiles = loadGateTestFiles(); + assert.ok(gateTestFiles.releaseContract.size > 0); + assert.ok(gateTestFiles.asfSource.size > 0); + + for (const [scriptName, lane] of GATE_SCRIPT_LANES) { + for (const file of collectGateScriptTestFiles( + scriptName, + JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).scripts, + )) { + const plan = planTests([file], { graph, gateTestFiles }); + assert.equal(plan[lane], true, `${file} must select ${lane} via ${scriptName}`); + assert.ok(existsSync(new URL(`../${file}`, import.meta.url)), file); + } + } +}); + +test('a test file two gates share selects at least one of their lanes', () => { + const gateTestFiles = loadGateTestFiles(); const lanesByTest = new Map(); - for (const [script, lane] of [ - ['check:release', 'releaseContract'], - ['check:asf-source', 'asfSource'], - ]) { - for (const file of scripts[script].match(/scripts\/[\w.-]+\.test\.mjs/gu) ?? []) { + for (const [scriptName, lane] of GATE_SCRIPT_LANES) { + const { scripts } = JSON.parse( + readFileSync(new URL('../package.json', import.meta.url), 'utf8'), + ); + for (const file of collectGateScriptTestFiles(scriptName, scripts)) { lanesByTest.set(file, (lanesByTest.get(file) ?? new Set()).add(lane)); } } - assert.ok(lanesByTest.size > 0, 'no gate script names a test file'); - // A test two gates share needs only one of them: either run executes it. for (const [file, lanes] of lanesByTest) { - const plan = planTests([file], { graph }); + if (lanes.size < 2) continue; + const plan = planTests([file], { graph, gateTestFiles }); assert.ok( [...lanes].some((lane) => plan[lane]), `${file} reaches no ${[...lanes].join('/')}`, @@ -277,12 +293,7 @@ test('every test a gate script runs reaches a lane that runs that gate', () => { } }); -// The other direction of the same drift. Every literal path in the planner is -// matched against a changed file, so one that no longer exists can never match -// and nothing reports it: the phantom baseline test sat here for a month, and -// `agent-run-store.test.ts` stayed in the storage stress set for a month after -// #1994 deleted it. -test('the planner names no path that no longer exists', () => { +test('the planner names no static path that no longer exists', () => { const source = readFileSync(new URL('ci-test-plan.mjs', import.meta.url), 'utf8'); const paths = [...source.matchAll(/^ {2}'([\w.-]+(?:\/[\w.-]+)+)',$/gmu)].map(([, path]) => path); assert.ok(paths.length > 0, 'the planner names no paths'); @@ -292,6 +303,16 @@ test('the planner names no path that no longer exists', () => { } }); +test('nested npm run invocations contribute gate-owned test files', () => { + const { scripts } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); + const releaseTests = collectGateScriptTestFiles('check:release', scripts); + + assert.ok( + releaseTests.includes('scripts/asf-npm-workflow-policy.test.mjs'), + 'check:release delegates to check:asf-npm', + ); +}); + test('Product Nightly authority changes select the release contract gate', () => { for (const path of [ '.github/workflows/desktop-nightly.yml',