Skip to content

Derive CI planner gate test inputs from package.json scripts #4479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Rangsh wants to merge 1 commit into
base: main
Choose a base branch
from
+92 −39
Open

Derive CI planner gate test inputs from package.json scripts #4479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 49 additions & 17 deletions scripts/ci-test-plan.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,46 @@ import { fileURLToPath } from 'node:url';
const scriptPath = fileURLToPath(import.meta.url);
const defaultRepoRoot = dirname(dirname(scriptPath));

/** npm gate scripts whose `scripts/*.test.mjs` inputs select a planner lane. */
export const GATE_SCRIPT_LANES = [
['check:release', 'releaseContract'],
['check:asf-source', 'asfSource'],
];

const SCRIPT_TEST_FILE_PATTERN = /scripts\/[\w.-]+\.test\.mjs/gu;
const NPM_RUN_SCRIPT_PATTERN = /\bnpm run ([\w:-]+)/gu;

export function extractScriptTestFiles(command) {
return [...new Set(command.match(SCRIPT_TEST_FILE_PATTERN) ?? [])];
}

export function collectGateScriptTestFiles(scriptName, scripts, visited = new Set()) {
if (visited.has(scriptName)) return [];
visited.add(scriptName);
const command = scripts[scriptName];
if (typeof command !== 'string') return [];

const files = extractScriptTestFiles(command);
for (const match of command.matchAll(NPM_RUN_SCRIPT_PATTERN)) {
files.push(...collectGateScriptTestFiles(match[1], scripts, visited));
}
return [...new Set(files)];
}

export function loadGateTestFiles(repoRoot = defaultRepoRoot, readFile = readFileSync) {
const { scripts } = JSON.parse(readFile(join(repoRoot, 'package.json'), 'utf8'));
const byLane = {
releaseContract: new Set(),
asfSource: new Set(),
};
for (const [scriptName, lane] of GATE_SCRIPT_LANES) {
for (const file of collectGateScriptTestFiles(scriptName, scripts)) {
byLane[lane].add(file);
}
}
return byLane;
}

const FULL_SUITE_FILES = new Set([
'.github/workflows/ci.yml',
'package-lock.json',
Expand Down Expand Up @@ -56,27 +96,19 @@ const RELEASE_CONTRACT_FILES = new Set([
'scripts/package-windows-autoupdate-next.mjs',
'scripts/package-windows-x64.mjs',
'scripts/prepare-windows-upgrade-baseline.mjs',
'scripts/prepare-windows-upgrade-baseline.test.mjs',
'scripts/generate-third-party-notices.test.mjs',
'scripts/product-release.test.mjs',
'scripts/qualify-released-cli-state-root.test.mjs',
'scripts/release-eval-smoke-sitecustomize.py',
'scripts/release-version.mjs',
'scripts/third-party-closure.test.mjs',
'scripts/verify-macos-arm64-cli.mjs',
'scripts/verify-macos-arm64-dmg.mjs',
'scripts/verify-macos-autoupdate.mjs',
'scripts/desktop-update-contract.mjs',
'scripts/product-nightly.mjs',
'scripts/product-nightly.test.mjs',
'scripts/verify-packaged-app.mjs',
'scripts/verify-packaged-app.test.mjs',
'scripts/verify-windows-autoupdate.mjs',
'scripts/verify-windows-installer-lifecycle.mjs',
'scripts/verify-windows-x64.mjs',
'scripts/windows-upgrade-baseline.json',
'scripts/windows-package-source-closure.mjs',
'scripts/windows-package-source-closure.test.mjs',
]);

// What decides whether a build can read durable state an earlier release wrote.
Expand Down Expand Up @@ -127,19 +159,15 @@ const ASF_SOURCE_FILES = new Set([
'package.json',
'packages/eval/harbor/deepseek-harness-profile/cordis.patch.yml',
'scripts/asf-license-headers.mjs',
'scripts/asf-license-headers.test.mjs',
'scripts/asf-source-release.mjs',
'scripts/asf-source-release.test.mjs',
'scripts/asf-source-workflow-policy.test.mjs',
'scripts/model-metadata/models-dev-api.snapshot.json',
'scripts/source-legal-inventory.test.mjs',
'scripts/sync-model-metadata.mjs',
'scripts/sync-model-metadata.test.mjs',
]);

function isAsfSourcePath(path) {
function isAsfSourcePath(path, gateTestFiles) {
return (
ASF_SOURCE_FILES.has(path) ||
gateTestFiles.asfSource.has(path) ||
path.startsWith('patches/') ||
path.startsWith('apps/desktop/resources/licenses/renderer/') ||
path.startsWith('apps/desktop/src/renderer/assets/provider-brands/')
Expand All @@ -166,9 +194,10 @@ function isCliPackagePath(path) {
);
}

function isReleaseContractPath(path) {
function isReleaseContractPath(path, gateTestFiles) {
return (
RELEASE_CONTRACT_FILES.has(path) ||
gateTestFiles.releaseContract.has(path) ||
path.startsWith('scripts/desktop-nightly') ||
path.startsWith('scripts/product-release-') ||
path.startsWith('scripts/release-cli-')
Expand Down Expand Up @@ -380,6 +409,8 @@ function workspaceLanes(workspaces, graph) {

export function planTests(changedFiles, options = {}) {
const graph = options.graph ?? loadWorkspaceGraph(options.repoRoot);
const gateTestFiles =
options.gateTestFiles ?? loadGateTestFiles(options.repoRoot, options.readFile);
const files = [...new Set(changedFiles.map(normalizePath).filter(Boolean))];
const forceFull = options.forceFull ?? false;
const full = forceFull || files.some((path) => FULL_SUITE_FILES.has(path));
Expand Down Expand Up @@ -457,7 +488,7 @@ export function planTests(changedFiles, options = {}) {
const cliPackage = files.some((path) => isCliPackagePath(path));
return {
appIcons: files.some((path) => isAppIconPath(path)),
asfSource: files.some((path) => isAsfSourcePath(path)),
asfSource: files.some((path) => isAsfSourcePath(path, gateTestFiles)),
astryxSurface: files.some((path) => isAstryxSurfaceInventoryPath(path)),
cliPackage,
code,
Expand All @@ -466,7 +497,8 @@ export function planTests(changedFiles, options = {}) {
// boots, and packages/ui unit-test-only PRs must not either.
e2e: files.some((path) => isE2eProductPath(path)),
full: false,
releaseContract: cliPackage || files.some((path) => isReleaseContractPath(path)),
releaseContract:
cliPackage || files.some((path) => isReleaseContractPath(path, gateTestFiles)),
// packages/cli/src/__tests__/runtime-host-session-driver.test.ts executes real sandboxed
// shell tools, so the bubblewrap + user-namespace setup is required whenever
// the cli workspace runs in the dependency closure, not only for direct
Expand Down
65 changes: 43 additions & 22 deletions scripts/ci-test-plan.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ import test from 'node:test';

import {
changedFilesBetween,
collectGateScriptTestFiles,
formatGitHubOutputs,
GATE_SCRIPT_LANES,
loadGateTestFiles,
planTests,
requiresHeavyValidation,
} from './ci-test-plan.mjs';
Expand Down Expand Up @@ -248,41 +251,49 @@ test('release authority changes select their dedicated contract gate', () => {
assert.equal(planTests(['.github/RELEASE_CHECKLIST.md'], { graph }).releaseContract, false);
});

// Derived from the gate scripts themselves, because the sets above are hand
// maintained and drift silently in both directions: a test the gate runs but
// no lane selects can be edited green, and a listed path that no longer exists
// is dead weight nothing reports. Both had happened — three of the release
// gate's own tests reached no lane, and the set named a
// `prepare-windows-upgrade-baseline.test.mjs` that never existed.
test('every test a gate script runs reaches a lane that runs that gate', () => {
const { scripts } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
// Gate-owned contract tests are inputs to their lane because the planner reads
// them from the gate script text in package.json — not from a parallel list that
// can drift in either direction.
test('gate script test files select the lane that gate script runs', () => {
const gateTestFiles = loadGateTestFiles();
assert.ok(gateTestFiles.releaseContract.size > 0);
assert.ok(gateTestFiles.asfSource.size > 0);

for (const [scriptName, lane] of GATE_SCRIPT_LANES) {
for (const file of collectGateScriptTestFiles(
scriptName,
JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).scripts,
)) {
const plan = planTests([file], { graph, gateTestFiles });
assert.equal(plan[lane], true, `${file} must select ${lane} via ${scriptName}`);
assert.ok(existsSync(new URL(`../${file}`, import.meta.url)), file);
}
}
});

test('a test file two gates share selects at least one of their lanes', () => {
const gateTestFiles = loadGateTestFiles();
const lanesByTest = new Map();
for (const [script, lane] of [
['check:release', 'releaseContract'],
['check:asf-source', 'asfSource'],
]) {
for (const file of scripts[script].match(/scripts\/[\w.-]+\.test\.mjs/gu) ?? []) {
for (const [scriptName, lane] of GATE_SCRIPT_LANES) {
const { scripts } = JSON.parse(
readFileSync(new URL('../package.json', import.meta.url), 'utf8'),
);
for (const file of collectGateScriptTestFiles(scriptName, scripts)) {
lanesByTest.set(file, (lanesByTest.get(file) ?? new Set()).add(lane));
}
}
assert.ok(lanesByTest.size > 0, 'no gate script names a test file');

// A test two gates share needs only one of them: either run executes it.
for (const [file, lanes] of lanesByTest) {
const plan = planTests([file], { graph });
if (lanes.size < 2) continue;
const plan = planTests([file], { graph, gateTestFiles });
assert.ok(
[...lanes].some((lane) => plan[lane]),
`${file} reaches no ${[...lanes].join('/')}`,
);
}
});

// The other direction of the same drift. Every literal path in the planner is
// matched against a changed file, so one that no longer exists can never match
// and nothing reports it: the phantom baseline test sat here for a month, and
// `agent-run-store.test.ts` stayed in the storage stress set for a month after
// #1994 deleted it.
test('the planner names no path that no longer exists', () => {
test('the planner names no static path that no longer exists', () => {
const source = readFileSync(new URL('ci-test-plan.mjs', import.meta.url), 'utf8');
const paths = [...source.matchAll(/^ {2}'([\w.-]+(?:\/[\w.-]+)+)',$/gmu)].map(([, path]) => path);
assert.ok(paths.length > 0, 'the planner names no paths');
Expand All @@ -292,6 +303,16 @@ test('the planner names no path that no longer exists', () => {
}
});

test('nested npm run invocations contribute gate-owned test files', () => {
const { scripts } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
const releaseTests = collectGateScriptTestFiles('check:release', scripts);

assert.ok(
releaseTests.includes('scripts/asf-npm-workflow-policy.test.mjs'),
'check:release delegates to check:asf-npm',
);
});

test('Product Nightly authority changes select the release contract gate', () => {
for (const path of [
'.github/workflows/desktop-nightly.yml',
Expand Down