diff --git a/scripts/__tests__/failsafe.test.mjs b/scripts/__tests__/failsafe.test.mjs new file mode 100644 index 000000000..a1c26d8c0 --- /dev/null +++ b/scripts/__tests__/failsafe.test.mjs @@ -0,0 +1,284 @@ +import { describe, expect, it } from 'vitest' +import { readWorkflow, workflowFiles } from './lib/workflows.mjs' + +/** + * The failsafe: the test that fails when the selector job does not decide + * whether a job runs. + * + * ADR 0003 and `.github/CONTEXT.md` describe the design. A selector job starts + * each workflow that runs on a pull request or in a merge queue. It compares + * the files a change touches with each check's trigger paths. Each job's `if:` + * reads its answer. A job that does not read it runs on every change. That + * costs money, and nobody notices. This test makes it visible. + * + * It reads every workflow that runs on `pull_request` or `merge_group`. In + * each one, a job passes when one of these is true: + * + * - It is the selector job, or the summary job that ends the workflow. + * - Its `needs:` includes the selector job and its `if:` reads the selector + * job's outputs. Each output it reads must be one the selector job declares: + * an output that does not exist is empty, and the job would never run. + * - It needs a job that passes by the rule above, and its `if:` calls no + * `always()`, `failure()` or `cancelled()`. GitHub then skips it whenever + * that job is skipped, so the selector decides it too. + * - It has a written reason in `EXEMPT_JOBS`. + * + * A whole workflow can be exempt, with a written reason, in `NOT_CHECKS` or + * `NOT_YET_CONVERTED`. An exemption that matches nothing, or is no longer + * needed, fails the test. So the lists only hold what is true today. + * + * The failsafe reads workflow files. It does not act while a workflow runs. + */ + +const SELECTOR = 'selector' + +/** + * Workflows that will never have a selector job. They are not checks, and no + * branch rule requires them. + */ +const NOT_CHECKS = { + '.github/workflows/claude-review.yml': + 'Review bot, not a check. No branch rule requires it.', + '.github/workflows/code-review.yml': + 'Review bot, not a check. No branch rule requires it.', + '.github/workflows/impact.yml': + 'Advisory report, not a check. No branch rule requires it.', +} + +/** + * Workflows that do not have a selector job yet. Each conversion removes its + * entry, and this list shrinks to empty. + */ +const NOT_YET_CONVERTED = Object.fromEntries( + [ + 'codeql.yml', + 'crap-crates.yml', + 'fta-v3.yml', + 'fuzz.yml', + 'integration-drizzle.yml', + 'integration-prisma-next.yml', + 'integration-protect-ffi.yml', + 'integration-supabase.yml', + 'lint-release.yml', + 'miri.yml', + 'musl-build-image.yml', + 'mutants.yml', + 'osv-scanner.yml', + 'prisma-example-readme-e2e.yml', + 'prisma-next-e2e.yml', + 'require-auth-npm-changeset.yml', + 'tests-auth-refresh.yml', + 'tests-bench.yml', + 'tests-crates.yml', + 'tests-golang.yml', + 'tests-rust.yml', + 'tests-supply-chain.yml', + 'tests.yml', + 'udeps.yml', + ].map((name) => [ + `.github/workflows/${name}`, + 'Not yet converted to a selector job.', + ]), +) + +/** + * Jobs in a workflow with a selector job that the selector job does not + * decide, on purpose. Most run on every change. Some never run on a pull + * request at all. Keyed `#`. + */ +const EXEMPT_JOBS = { + '.github/workflows/test-eql.yml#setup': + 'Sets the Postgres versions and shard count that the gated jobs read. It is a bash step with no checkout.', + '.github/workflows/test-eql.yml#docs-static': + 'Checks the SQL documentation on every pull request, including the docs-only ones the selector skips. It needs no database or credentials.', + '.github/workflows/test-eql.yml#known-failures': + 'Checks that each suppressed test names an open issue, on every pull request. It needs no database or credentials.', + '.github/workflows/test-eql.yml#doc-anchors': + 'Checks Markdown anchor links. Its inputs are the docs themselves, which the selector does not select for.', +} + +const IN_SCOPE_EVENTS = ['pull_request', 'merge_group'] + +function events(workflow) { + // `on:` parses as the boolean `true` under YAML 1.1. + const on = workflow?.on ?? workflow?.[true] + if (typeof on === 'string') return [on] + if (Array.isArray(on)) return on + return Object.keys(on ?? {}) +} + +const needsOf = (job) => [job?.needs ?? []].flat() + +const ifText = (job) => String(job?.if ?? '') + +/** The selector outputs a job's `if:` reads. */ +const outputsRead = (job) => + [ + ...ifText(job).matchAll( + new RegExp(`needs\\.${SELECTOR}\\.outputs\\.([A-Za-z0-9_-]+)`, 'g'), + ), + ].map((match) => match[1]) + +/** A status function stops GitHub skipping a job when a job it needs skipped. */ +const keepsRunningAfterSkip = (job) => + /\b(always|failure|cancelled)\s*\(/.test(ifText(job)) + +/** + * The job that ends the workflow: its `if:` is `always()` and it needs every + * other job. + */ +function summaryJobId(jobs) { + const ids = Object.keys(jobs) + const last = ids.at(-1) + const others = ids.filter((id) => id !== last).sort() + const needs = [...needsOf(jobs[last])].sort() + const isSummary = + ifText(jobs[last]).trim() === 'always()' && + JSON.stringify(needs) === JSON.stringify(others) + return isSummary ? last : undefined +} + +/** The ids of the jobs the selector job decides, directly or through `needs:`. */ +function decidedJobIds(jobs) { + const declared = Object.keys(jobs[SELECTOR]?.outputs ?? {}) + const readsSelector = (job) => { + const read = outputsRead(job) + return read.length > 0 && read.every((name) => declared.includes(name)) + } + const decided = new Set( + Object.entries(jobs) + .filter( + ([, job]) => needsOf(job).includes(SELECTOR) && readsSelector(job), + ) + .map(([id]) => id), + ) + let grew = true + while (grew) { + grew = false + for (const [id, job] of Object.entries(jobs)) { + if (decided.has(id) || keepsRunningAfterSkip(job)) continue + if (needsOf(job).some((need) => decided.has(need))) { + decided.add(id) + grew = true + } + } + } + return decided +} + +const WORKFLOWS = workflowFiles() + .map((file) => ({ file, workflow: readWorkflow(file) })) + .filter(({ workflow }) => + events(workflow).some((event) => IN_SCOPE_EVENTS.includes(event)), + ) + +const hasSelector = ({ workflow }) => Boolean(workflow?.jobs?.[SELECTOR]) + +const isExemptWorkflow = ({ file }) => + file in NOT_CHECKS || file in NOT_YET_CONVERTED + +/** The workflows checked job by job. */ +const CHECKED = WORKFLOWS.filter( + (entry) => hasSelector(entry) && !isExemptWorkflow(entry), +).map(({ file, workflow }) => { + const jobs = workflow.jobs + return { + file, + jobs, + summary: summaryJobId(jobs), + decided: decidedJobIds(jobs), + } +}) + +const reasonMissing = (map) => + Object.entries(map) + .filter(([, reason]) => typeof reason !== 'string' || !reason.trim()) + .map(([key]) => key) + +describe('the failsafe', () => { + it('finds the workflows it checks', () => { + // An empty scan would pass every check below. + expect(WORKFLOWS.length).toBeGreaterThan(0) + expect(CHECKED.map(({ file }) => file)).toContain( + '.github/workflows/test-eql.yml', + ) + }) + + it('fails when the selector job does not decide whether a job runs', () => { + const undecided = CHECKED.flatMap(({ file, jobs, summary, decided }) => + Object.keys(jobs) + .filter((id) => id !== SELECTOR && id !== summary) + .filter((id) => !decided.has(id)) + .map((id) => `${file}#${id}`) + .filter((ref) => !(ref in EXEMPT_JOBS)), + ) + expect( + undecided, + `The selector job does not decide whether these jobs run, so they can run on every change. Give each one \`needs: [${SELECTOR}]\` and an \`if:\` that reads an output the selector job declares. If the selector job must not decide a job, add it to EXEMPT_JOBS with the reason. If the summary job is listed, make it the last job, with \`if: always()\` and every other job in its \`needs:\`.`, + ).toEqual([]) + }) + + it('fails when a workflow on pull requests or in a merge queue has no selector job', () => { + const missing = WORKFLOWS.filter( + (entry) => !hasSelector(entry) && !isExemptWorkflow(entry), + ).map(({ file }) => file) + expect( + missing, + `These workflows run on pull requests or in a merge queue and have no \`${SELECTOR}\` job. Add one (ADR 0003), or exempt the workflow with a reason.`, + ).toEqual([]) + }) + + it('fails when an exemption has no written reason', () => { + expect([ + ...reasonMissing(NOT_CHECKS), + ...reasonMissing(NOT_YET_CONVERTED), + ...reasonMissing(EXEMPT_JOBS), + ]).toEqual([]) + }) + + it('fails when a workflow exemption matches no workflow it would exempt', () => { + const inScope = new Map(WORKFLOWS.map((entry) => [entry.file, entry])) + const stale = [ + ...Object.keys(NOT_CHECKS) + .filter((file) => !inScope.has(file)) + .map( + (file) => + `${file}: in NOT_CHECKS, but it runs on no pull request or merge queue`, + ), + ...Object.keys(NOT_YET_CONVERTED) + .filter((file) => !inScope.has(file)) + .map( + (file) => + `${file}: in NOT_YET_CONVERTED, but it runs on no pull request or merge queue`, + ), + ...Object.keys(NOT_YET_CONVERTED) + .filter((file) => inScope.has(file) && hasSelector(inScope.get(file))) + .map( + (file) => + `${file}: in NOT_YET_CONVERTED, but it has a selector job. Remove the entry.`, + ), + ...Object.keys(NOT_CHECKS) + .filter((file) => file in NOT_YET_CONVERTED) + .map((file) => `${file}: in both NOT_CHECKS and NOT_YET_CONVERTED`), + ] + expect(stale).toEqual([]) + }) + + it('fails when a job exemption matches no job the selector leaves undecided', () => { + const undecidedRefs = new Set( + CHECKED.flatMap(({ file, jobs, summary, decided }) => + Object.keys(jobs) + .filter((id) => id !== SELECTOR && id !== summary) + .filter((id) => !decided.has(id)) + .map((id) => `${file}#${id}`), + ), + ) + const stale = Object.keys(EXEMPT_JOBS).filter( + (ref) => !undecidedRefs.has(ref), + ) + expect( + stale, + 'These exemptions match no job that the selector job leaves undecided. The job is gone, its workflow is exempt as a whole, or the selector now decides it. Remove each entry.', + ).toEqual([]) + }) +})