From 8f357b242319ac5d77a8217884eec2e549bb71ae Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Wed, 28 Oct 2026 23:25:35 +1100 Subject: [PATCH 1/3] test(ci): check every selector job the same way Test EQL has a selector job, and the other pull request workflows are getting one each. Each workflow's selector must follow the same rules: no path filter on pull_request, one dorny/paths-filter step, an output named after each filter, every job selected on other events, a failure when a filter gives no answer, `if:` conditions that exclude pull_request, and a summary job that needs every other job. workflow-selectors.test.mjs checks those rules for every workflow that has a selector job, found by scanning. It runs the selector's own decision script and the summary job's own check. lib/selector.mjs holds the shared code. It reads a selector job, runs its script, and works out which jobs a pull request runs from the files it changes. It reads dorny's `every` quantifier and brace patterns, which a filter needs to exclude documentation. The parity test now compares a selector workflow's push paths with the union of its selector filters, by what each selects from the tracked files. That replaces Test EQL's written exemption. The dispatch test holds every selector output open, whatever its name. --- scripts/__tests__/lib/selector.mjs | 361 ++++++++++++++++++ .../workflow-dispatch-job-conditions.test.mjs | 6 +- .../workflow-paths-filter-parity.test.mjs | 71 +++- scripts/__tests__/workflow-selectors.test.mjs | 220 +++++++++++ 4 files changed, 638 insertions(+), 20 deletions(-) create mode 100644 scripts/__tests__/lib/selector.mjs create mode 100644 scripts/__tests__/workflow-selectors.test.mjs diff --git a/scripts/__tests__/lib/selector.mjs b/scripts/__tests__/lib/selector.mjs new file mode 100644 index 000000000..ec2f910bb --- /dev/null +++ b/scripts/__tests__/lib/selector.mjs @@ -0,0 +1,361 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import yaml from 'js-yaml' +import { runsWhen } from './expressions.mjs' +import { pathsGlobToRegExp } from './paths-filter.mjs' +import { readWorkflow } from './workflows.mjs' + +/** + * Reading and running a workflow's selector job, for the guards that pin what + * it decides. + * + * The selector job is the first job of a workflow. It compares the files a + * pull request changes with each check's trigger paths, using + * dorny/paths-filter, and writes one output per filter. Each gated job's `if:` + * reads the output named after its filter. ADR 0003 and `.github/CONTEXT.md` + * describe the design. + * + * Everything here runs the workflow's own text: the filter rules as written, + * the decision script as written, and each job's real `if:`. A guard that + * reimplemented the decision would agree with a rewritten one by luck. + */ + +export const SELECTOR_JOB = 'selector' + +/** The events the selector decides nothing for: each one selects every job. */ +export const NON_PULL_REQUEST_EVENTS = [ + 'push', + 'schedule', + 'merge_group', + 'workflow_dispatch', +] + +const DORNY = 'dorny/paths-filter@' + +const stepOutputRef = (expression) => + /steps\.([A-Za-z0-9_-]+)\.outputs\.([A-Za-z0-9_-]+)/.exec( + String(expression ?? ''), + ) + +const selectorOutputRefs = (expression) => + [ + ...String(expression ?? '').matchAll( + new RegExp(`needs\\.${SELECTOR_JOB}\\.outputs\\.([A-Za-z0-9_-]+)`, 'g'), + ), + ].map((match) => match[1]) + +/** + * `{a,b}` expanded into its alternatives, as picomatch reads a brace pattern. + * One level only: a nested brace throws, rather than being read wrongly. + */ +export function expandBraces(pattern) { + const open = pattern.indexOf('{') + if (open === -1) return [pattern] + const close = pattern.indexOf('}', open) + const inner = pattern.slice(open + 1, close) + if (close === -1 || inner.includes('{')) { + throw new Error(`Cannot expand the brace pattern ${pattern}`) + } + const head = pattern.slice(0, open) + return inner + .split(',') + .flatMap((alternative) => + expandBraces(`${head}${alternative}${pattern.slice(close + 1)}`), + ) +} + +/** One dorny/paths-filter rule against one file. A `!` rule negates. */ +export function dornyRuleMatches(rule, file) { + if (rule.startsWith('!')) return !dornyRuleMatches(rule.slice(1), file) + return expandBraces(rule).some((glob) => pathsGlobToRegExp(glob).test(file)) +} + +/** + * Whether a file counts for a filter. Under dorny's default quantifier, `some`, + * one matching rule is enough; under `every`, all of them must match. + */ +export function dornyFilterMatches(rules, quantifier, file) { + return quantifier === 'every' + ? rules.every((rule) => dornyRuleMatches(rule, file)) + : rules.some((rule) => dornyRuleMatches(rule, file)) +} + +/** + * A workflow's selector job, parsed, or `null` when it has none. + * + * `filters` maps each filter name to its rules. A filter that includes an + * anchored list (`- *name`) parses to a nested list, and dorny flattens it; + * so does this. `outputs` maps each selector output to the decision step + * output it reads. + */ +export function readSelectorJob(relPath) { + const workflow = readWorkflow(relPath) + const job = workflow?.jobs?.[SELECTOR_JOB] + if (!job) return null + const steps = Array.isArray(job.steps) ? job.steps : [] + const filterSteps = steps.filter((step) => + String(step?.uses ?? '').startsWith(DORNY), + ) + const filterStep = filterSteps.length === 1 ? filterSteps[0] : null + const filters = Object.fromEntries( + Object.entries(yaml.load(filterStep?.with?.filters ?? '') ?? {}).map( + ([name, rules]) => [name, [rules].flat(Infinity).map(String)], + ), + ) + const outputs = Object.fromEntries( + Object.entries(job.outputs ?? {}).map(([name, expression]) => [ + name, + stepOutputRef(expression), + ]), + ) + const decideIds = [ + ...new Set(Object.values(outputs).map((ref) => ref?.[1])), + ].filter(Boolean) + const decideStep = + decideIds.length === 1 + ? steps.find((step) => step?.id === decideIds[0]) + : null + return { + workflow, + job, + filterSteps, + filterStep, + quantifier: filterStep?.with?.['predicate-quantifier'] ?? 'some', + filters, + outputs: Object.fromEntries( + Object.entries(outputs).map(([name, ref]) => [name, ref?.[2]]), + ), + decideStep, + } +} + +/** The selector outputs a job's `if:` reads. */ +export function selectorOutputsRead(job) { + return [...new Set(selectorOutputRefs(job?.if))] +} + +/** + * Runs the selector's decision script the way GitHub would: `answers` stands + * in for the filter step's outputs, keyed by filter name. An env entry that + * names no answer gets the empty string, as GitHub substitutes for an + * expression that resolves to nothing. + * + * Returns the exit status, the log, and the selector job's outputs. + */ +export function runSelectorDecision(relPath, eventName, answers) { + const selector = readSelectorJob(relPath) + const scratch = mkdtempSync(join(tmpdir(), 'selector-')) + try { + const outputFile = join(scratch, 'outputs.txt') + writeFileSync(outputFile, '') + const env = { PATH: process.env.PATH, GITHUB_OUTPUT: outputFile } + for (const [key, value] of Object.entries( + selector?.decideStep?.env ?? {}, + )) { + const text = String(value) + if (/github\.event_name/.test(text)) env[key] = eventName + else env[key] = answers[stepOutputRef(text)?.[2]] ?? '' + } + const run = spawnSync( + 'bash', + ['-c', String(selector?.decideStep?.run ?? 'exit 99')], + { cwd: scratch, env, encoding: 'utf8' }, + ) + const written = Object.fromEntries( + readFileSync(outputFile, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => [ + line.slice(0, line.indexOf('=')), + line.slice(line.indexOf('=') + 1), + ]), + ) + return { + status: run.status, + log: `${run.stdout}${run.stderr}`, + outputs: Object.fromEntries( + Object.entries(selector?.outputs ?? {}).map(([name, stepOutput]) => [ + name, + written[stepOutput], + ]), + ), + } + } finally { + rmSync(scratch, { recursive: true, force: true }) + } +} + +/** What each filter answers for a pull request that changes `files`. */ +export function filterAnswers(relPath, files) { + const selector = readSelectorJob(relPath) + return Object.fromEntries( + Object.entries(selector?.filters ?? {}).map(([name, rules]) => [ + name, + String( + files.some((file) => + dornyFilterMatches(rules, selector.quantifier, file), + ), + ), + ]), + ) +} + +const needsOf = (job) => [job?.needs ?? []].flat() + +/** + * The jobs that run for a same-repository pull request changing `files`, + * sorted. The selector job's outputs come from its own script; each job's + * `if:` is evaluated as GitHub does, including the implicit `success()` that + * skips a job when one it needs did not succeed. + * + * `github` adds to the context, for a job whose `if:` reads more than the + * event and the selector (such as `head_ref`). + */ +export function selectedJobs(relPath, files, { github = {} } = {}) { + const selector = readSelectorJob(relPath) + const decision = runSelectorDecision( + relPath, + 'pull_request', + filterAnswers(relPath, files), + ) + if (decision.status !== 0) { + throw new Error(`The selector failed:\n${decision.log}`) + } + const repository = 'cipherstash/stack' + const context = { + github: { + event_name: 'pull_request', + repository, + head_ref: 'feature', + event: { + pull_request: { + draft: false, + head: { repo: { full_name: repository } }, + user: { type: 'User' }, + }, + }, + ...github, + }, + needs: {}, + vars: {}, + } + const jobs = selector.workflow.jobs + const results = { + [SELECTOR_JOB]: { result: 'success', outputs: decision.outputs }, + } + const resolve = (id) => { + if (results[id]) return results[id].result + const job = jobs[id] + const needs = needsOf(job) + const needResults = needs.map(resolve) + const condition = String(job?.if ?? '') + const statusFunction = /\b(always|failure|cancelled|success)\(\)/.test( + condition, + ) + let runs + if (!statusFunction && needResults.some((result) => result !== 'success')) { + runs = false + } else if (!condition) { + runs = true + } else { + runs = runsWhen(condition, { + ...context, + needs: Object.fromEntries(needs.map((need) => [need, results[need]])), + }) + } + results[id] = { result: runs ? 'success' : 'skipped', outputs: {} } + return results[id].result + } + return Object.keys(jobs) + .filter((id) => resolve(id) === 'success') + .sort() +} + +// GitHub reads a `paths:` list in order, and the last matching entry decides. +export function pathsListSelects(paths, file) { + let selected = false + for (const entry of paths) { + if (entry.startsWith('!')) { + if (pathsGlobToRegExp(entry.slice(1)).test(file)) selected = false + } else if (pathsGlobToRegExp(entry).test(file)) { + selected = true + } + } + return selected +} + +/** + * A filter as a GitHub `paths:` list: entries in order, a `!` entry excluding + * what came before it. Lets the guards that check a workflow's `paths:` + * coverage read a selector filter with the same code. + * + * Under `every`, the one positive rule's brace alternatives come first and the + * `!` rules after, which selects exactly the files dorny selects. Under + * `some`, a filter has no `!` rule (`workflow-selectors.test.mjs`), so it is + * already that list. + */ +export function filterAsPathsList(rules, quantifier) { + if (quantifier !== 'every') return rules.flatMap(expandBraces) + const positive = rules.filter((rule) => !rule.startsWith('!')) + const negative = rules.filter((rule) => rule.startsWith('!')) + return [...positive.flatMap(expandBraces), ...negative] +} + +/** + * The path lists that decide whether a job runs on a change, as GitHub + * `paths:` lists: the workflow's `on.push.paths` and `on.pull_request.paths` + * where they exist, and each selector filter the job's `if:` reads. + * + * A job in a selector workflow that reads no filter runs on every pull + * request, so only the `push` list, if any, decides it. + */ +export function jobTriggerLists(relPath, jobId) { + const workflow = readWorkflow(relPath) + const on = workflow?.on ?? workflow?.[true] + const lists = ['push', 'pull_request'] + .filter((event) => Array.isArray(on?.[event]?.paths)) + .map((event) => ({ + source: `on.${event}.paths`, + paths: on[event].paths.map(String), + })) + const selector = readSelectorJob(relPath) + for (const name of selectorOutputsRead(workflow?.jobs?.[jobId])) { + const rules = selector?.filters?.[name] + if (!rules) continue + lists.push({ + source: `selector filter "${name}"`, + paths: filterAsPathsList(rules, selector.quantifier), + }) + } + return lists +} + +/** + * Runs the summary job's check with the given results of the jobs it needs, + * and returns its exit status. The results stand in for + * `join(needs.*.result, ' ')`, wherever the step reads it. + */ +export function runSummaryCheck(relPath, summaryId, results) { + const workflow = readWorkflow(relPath) + const steps = workflow?.jobs?.[summaryId]?.steps ?? [] + const joined = results.join(' ') + const resultsExpression = /\$\{\{\s*join\(needs\.\*\.result,\s*' '\)\s*\}\}/g + let status = 0 + for (const step of steps) { + if (typeof step?.run !== 'string') continue + const env = { PATH: process.env.PATH } + for (const [key, value] of Object.entries(step.env ?? {})) { + env[key] = String(value).replace(resultsExpression, joined) + } + const run = spawnSync( + 'bash', + ['-c', step.run.replace(resultsExpression, joined)], + { env, encoding: 'utf8' }, + ) + status = run.status + if (status !== 0) break + } + return status +} diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index e727e71fe..d7a6ea95e 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -180,7 +180,11 @@ function extractForkClause(condition) { * run. */ const PERMISSIVE_NEEDS = { - selector: { outputs: { eql: 'true', rust_crates: 'true' } }, + // Every selector output, whatever its name: each workflow's selector names + // its outputs after its own filters. + selector: { + outputs: new Proxy({}, { has: () => true, get: () => 'true' }), + }, // The release workflows' gates, none of which is about how the run was // triggered. Held open, or "skips because there is nothing to publish" would // be indistinguishable from "skips on a dispatch". diff --git a/scripts/__tests__/workflow-paths-filter-parity.test.mjs b/scripts/__tests__/workflow-paths-filter-parity.test.mjs index 74ab3af62..93d4facab 100644 --- a/scripts/__tests__/workflow-paths-filter-parity.test.mjs +++ b/scripts/__tests__/workflow-paths-filter-parity.test.mjs @@ -1,4 +1,11 @@ +import { execFileSync } from 'node:child_process' import { describe, expect, it } from 'vitest' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { + dornyFilterMatches, + pathsListSelects, + readSelectorJob, +} from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -57,6 +64,14 @@ function pathsFilters(relPath) { ) } +/** Every tracked file: what a `paths:` list or a filter can select. */ +const TRACKED = execFileSync('git', ['ls-files', '-z'], { + cwd: REPO_ROOT, + encoding: 'utf8', +}) + .split('\0') + .filter(Boolean) + const CLASSIFIED = workflowFiles().map((relPath) => { const filters = pathsFilters(relPath) const declared = FILTERED_EVENTS.filter((event) => filters[event] !== null) @@ -66,8 +81,23 @@ const CLASSIFIED = workflowFiles().map((relPath) => { /** Workflows writing the list twice — the ones the parity check applies to. */ const PARITY = CLASSIFIED.filter((entry) => entry.declared.length === 2) +/** + * Workflows whose selector job decides what a pull request runs, and whose + * `push` keeps a `paths:` list. Their `pull_request` has no filter, by design + * (ADR 0003), so the second copy of the list is the selector's filters. The + * check below compares the two by what they select. + */ +const SELECTOR_PUSH = CLASSIFIED.filter( + (entry) => + entry.filters.push !== null && + entry.filters.pull_request === null && + readSelectorJob(entry.relPath) !== null, +) + /** Workflows filtering one event and not the other. */ -const ASYMMETRIC = CLASSIFIED.filter((entry) => entry.declared.length === 1) +const ASYMMETRIC = CLASSIFIED.filter( + (entry) => entry.declared.length === 1 && !SELECTOR_PUSH.includes(entry), +) /** * The guard on the scan, in the shape the rest of this directory uses. Every @@ -145,24 +175,6 @@ const EXPECTED_ASYMMETRIES = new Map([ // check-auth-npm-changeset.test.mjs, which is the second copy here. 'pull_request is the only trigger; the check diffs a pull request against its base, which a push does not have', ], - [ - '.github/workflows/test-eql.yml', - // `pull_request` is deliberately UNFILTERED and `push: branches: [main]` - // carries the list, which is the same shape as tests-rust.yml with the two - // events swapped. Unfiltered on the PR side because this workflow's - // `ci-required` is the check branch protection is meant to reference, and a - // workflow skipped by a path filter leaves a required check stuck Pending; - // relevance is applied per-job via `if:` instead. Filtered on the push side - // because nothing waits on a push run, so skipping one costs nothing. - // - // The direction that would hurt — a `pull_request` filter narrower than the - // `push` one, so the PR introducing a change never runs the suite — is - // unreachable when `pull_request` has no filter at all. The push list is - // not unchecked: `eql-workflow-filters.test.mjs` compares it against the - // union of the selector job's filters, which is the second copy of the - // same list and the one this file cannot see. - 'pull_request is deliberately unfiltered (per-job relevance instead), so PRs run a superset of what push runs; the push list is compared against the selector job filters by eql-workflow-filters.test.mjs', - ], [ '.github/workflows/bench-eql.yml', // The mirror image of lint-release.yml: `push: branches: [main]` with a @@ -235,6 +247,27 @@ describe('paths filters are written twice, identically', () => { ).toEqual([]) }) + for (const { relPath, filters } of SELECTOR_PUSH) { + it(`${relPath} selects on push what its selector selects on a pull request`, () => { + // A selector workflow writes its list twice as well: as the `push` + // `paths:` list, and split across the selector job's filters. The two + // are written in different syntaxes, so they are compared by what they + // select from the tracked files, not by their text. + const selector = readSelectorJob(relPath) + const differ = TRACKED.filter( + (file) => + pathsListSelects(filters.push, file) !== + Object.values(selector.filters).some((rules) => + dornyFilterMatches(rules, selector.quantifier, file), + ), + ) + expect( + differ, + `In ${relPath}, the \`push\` \`paths:\` list and the union of the selector job's filters disagree about these files. Change both in the same edit.`, + ).toEqual([]) + }) + } + for (const { relPath } of PARITY) { it(`${relPath} filters push and pull_request identically`, () => { const filters = pathsFilters(relPath) diff --git a/scripts/__tests__/workflow-selectors.test.mjs b/scripts/__tests__/workflow-selectors.test.mjs new file mode 100644 index 000000000..341b95f8d --- /dev/null +++ b/scripts/__tests__/workflow-selectors.test.mjs @@ -0,0 +1,220 @@ +import { describe, expect, it } from 'vitest' +import { runsWhen } from './lib/expressions.mjs' +import { + NON_PULL_REQUEST_EVENTS, + readSelectorJob, + runSelectorDecision, + runSummaryCheck, + SELECTOR_JOB, + selectorOutputsRead, +} from './lib/selector.mjs' +import { workflowFiles } from './lib/workflows.mjs' + +/** + * Every workflow with a selector job follows the same design (ADR 0003): the + * selector job decides which checks a pull request runs, each gated job reads + * the selector's answer in its `if:`, and a summary job ends the workflow. + * `.github/CONTEXT.md` defines the terms. + * + * The workflows are discovered, not listed, so a workflow that gains a + * selector job is held to this the day it lands. What each selector decides + * for particular changes is pinned per workflow, in the `*-selector*.test.mjs` + * files. + * + * Whether every job is decided by the selector, or has a written reason to run + * on every change, is the failsafe's question, not this file's. + */ + +const SELECTOR_WORKFLOWS = workflowFiles().filter( + (relPath) => readSelectorJob(relPath) !== null, +) + +/** + * The guard on the scan. A minimum: a workflow that gains a selector job must + * not fail this. Each workflow's own `*-selector*.test.mjs` file checks that + * its selector job exists. + */ +const EXPECTED_SELECTOR_WORKFLOWS = ['.github/workflows/test-eql.yml'] + +const triggersOf = (workflow) => workflow?.on ?? workflow?.[true] + +const needsOf = (job) => [job?.needs ?? []].flat() + +/** The jobs whose `if:` reads a selector output. */ +function gatedJobs(workflow) { + return Object.entries(workflow?.jobs ?? {}) + .map(([id, job]) => ({ id, job, reads: selectorOutputsRead(job) })) + .filter((entry) => entry.reads.length > 0) +} + +describe('the workflows with a selector job', () => { + it('include the converted workflows', () => { + expect(SELECTOR_WORKFLOWS).toEqual( + expect.arrayContaining(EXPECTED_SELECTOR_WORKFLOWS), + ) + }) +}) + +for (const relPath of SELECTOR_WORKFLOWS) { + const selector = readSelectorJob(relPath) + const { workflow } = selector + const jobIds = Object.keys(workflow.jobs) + const summaryId = jobIds.at(-1) + const filterNames = Object.keys(selector.filters) + const outputNames = Object.keys(selector.outputs) + + describe(relPath, () => { + it('has no path filter on pull_request', () => { + // A workflow that a path filter skips leaves its required checks + // pending, and GitHub's documentation on required status checks says + // that blocks a merge. The selector job decides instead. + const pullRequest = triggersOf(workflow)?.pull_request ?? {} + expect(Object.keys(pullRequest)).not.toContain('paths') + expect(Object.keys(pullRequest)).not.toContain('paths-ignore') + }) + + it('starts with the selector job, on a GitHub-hosted runner', () => { + // ADR 0003: every gated job waits for it, and a self-provisioned + // machine takes longer to start. + expect(jobIds[0]).toBe(SELECTOR_JOB) + expect(String(selector.job['runs-on'])).toMatch(/^ubuntu-/) + expect(selector.job.needs).toBeUndefined() + expect(selector.job.if).toBeUndefined() + }) + + it('has one path filter step, run only on a pull request', () => { + expect(selector.filterSteps).toHaveLength(1) + expect(String(selector.filterStep?.if)).toBe( + "github.event_name == 'pull_request'", + ) + expect(filterNames).not.toEqual([]) + }) + + it('writes no `!` rule that dorny would read as matching everything', () => { + // Under dorny's default quantifier, `some`, a file counts when ANY rule + // matches, so `!**/*.md` matches every file that is not Markdown. A `!` + // rule only excludes under `every`, where a file must match every rule, + // and there a second positive rule would select nothing at all. + expect(['some', 'every']).toContain(selector.quantifier) + for (const [name, rules] of Object.entries(selector.filters)) { + const positive = rules.filter((rule) => !rule.startsWith('!')) + const negative = rules.filter((rule) => rule.startsWith('!')) + if (selector.quantifier === 'every') { + expect(positive, `filter "${name}"`).toHaveLength(1) + } else { + expect(negative, `filter "${name}"`).toEqual([]) + } + } + }) + + it('names each output after the filter that decides it', () => { + // The guards follow a job's `if:` to its filter by this name. + expect(outputNames.sort()).toEqual([...filterNames].sort()) + for (const [name, stepOutput] of Object.entries(selector.outputs)) { + expect(stepOutput, `output "${name}"`).toBe(name) + } + expect( + selector.decideStep, + 'No single step writes the selector outputs.', + ).toBeTruthy() + }) + + it('selects every job on events other than pull_request', () => { + for (const event of NON_PULL_REQUEST_EVENTS) { + const { status, log, outputs } = runSelectorDecision(relPath, event, {}) + expect(status, log).toBe(0) + for (const name of outputNames) { + expect(outputs[name], `${name} on ${event}`).toBe('true') + } + } + }) + + it('passes the filter answers through on a pull request', () => { + for (const answer of ['true', 'false']) { + const answers = Object.fromEntries( + filterNames.map((name) => [name, answer]), + ) + const { status, log, outputs } = runSelectorDecision( + relPath, + 'pull_request', + answers, + ) + expect(status, log).toBe(0) + expect(outputs).toEqual(answers) + } + }) + + it('fails when a pull request filter gives no answer', () => { + // A missing answer would skip every gated job, and the summary job + // counts skipped as passed: a green run that ran nothing. + for (const name of filterNames) { + for (const bad of ['', 'maybe']) { + const answers = Object.fromEntries( + filterNames.map((other) => [other, other === name ? bad : 'true']), + ) + const { status } = runSelectorDecision( + relPath, + 'pull_request', + answers, + ) + expect(status, `filter "${name}" answering '${bad}'`).not.toBe(0) + } + } + }) + + it('gates jobs only on outputs the selector writes, and needs it', () => { + const gated = gatedJobs(workflow) + expect(gated).not.toEqual([]) + for (const { id, job, reads } of gated) { + expect(needsOf(job), `job "${id}"`).toContain(SELECTOR_JOB) + for (const name of reads) { + expect(outputNames, `job "${id}" reads "${name}"`).toContain(name) + } + } + }) + + it("writes each gated job's `if:` as an exclusion of pull_request", () => { + // Outside a pull request the selector's answers must not matter: a job + // written as a list of allowed events skips on the event nobody listed. + for (const { id, job } of gatedJobs(workflow)) { + for (const event of NON_PULL_REQUEST_EVENTS) { + const verdict = (answer) => + runsWhen(job.if, { + github: { + event_name: event, + repository: 'cipherstash/stack', + event: {}, + }, + needs: { + [SELECTOR_JOB]: { + result: 'success', + outputs: Object.fromEntries( + outputNames.map((name) => [name, answer]), + ), + }, + }, + vars: {}, + }) + expect(verdict('false'), `job "${id}" on ${event}`).toBe( + verdict('true'), + ) + } + } + }) + + it('ends with a summary job that needs every other job', () => { + const summary = workflow.jobs[summaryId] + expect(String(summary.if)).toBe('always()') + expect([...needsOf(summary)].sort()).toEqual( + jobIds.filter((id) => id !== summaryId).sort(), + ) + }) + + it('passes the summary only when every needed job passed or skipped', () => { + const status = (results) => runSummaryCheck(relPath, summaryId, results) + expect(status(['success', 'skipped', 'success'])).toBe(0) + expect(status(['success', 'failure'])).not.toBe(0) + expect(status(['skipped', 'cancelled'])).not.toBe(0) + }) + }) +} From ec6a8da1abb53fae4edfdb8c14d0a9089a77acee Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Wed, 28 Oct 2026 23:31:51 +1100 Subject: [PATCH 2/3] test(ci): read a job's trigger paths from its selector filter Three guards check that a workflow's path filter covers what its jobs read: the binding build inputs, the auth binding sources, and the sources the integration suites import. They read `on.push.paths` and `on.pull_request.paths`. Once a workflow has a selector job, its pull request list moves into the selector filter of each job, and those guards would stop checking it. They now read, per job, the lists that decide whether that job runs: the workflow's push and pull_request lists, and the selector filter the job reads. For a workflow without a selector job this is the same two lists as before. They also check only the jobs that build or load the binding, not every job in the workflow. --- .../auth-binding-step-order.test.mjs | 41 ++++++---- .../integration-workflow-paths.test.mjs | 32 +++++--- .../wasm-build-inputs-paths-filter.test.mjs | 80 ++++++++++++------- 3 files changed, 95 insertions(+), 58 deletions(-) diff --git a/scripts/__tests__/auth-binding-step-order.test.mjs b/scripts/__tests__/auth-binding-step-order.test.mjs index aa1b604ab..82dd4317e 100644 --- a/scripts/__tests__/auth-binding-step-order.test.mjs +++ b/scripts/__tests__/auth-binding-step-order.test.mjs @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { filterCovers } from './lib/paths-filter.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -84,13 +85,20 @@ describe('the @cipherstash/auth binding is built wherever the SDK runs', () => { const same = (entry, input) => entry === input -/** The `paths:` lists of a workflow's filtered events. */ -const filters = (relPath) => { - const wf = readWorkflow(relPath) - const on = wf?.on ?? wf?.[true] - return ['push', 'pull_request'] - .map((event) => on?.[event]?.paths) - .filter(Array.isArray) +/** + * The path lists that decide whether the given jobs run, once each: the + * workflow's `push` and `pull_request` `paths:` lists, and the selector filter + * each job reads. Every job of the workflow when none are given. + */ +const filters = (relPath, jobNames) => { + const lists = new Map() + for (const jobName of jobNames ?? + Object.keys(readWorkflow(relPath)?.jobs ?? {})) { + for (const { source, paths } of jobTriggerLists(relPath, jobName)) { + lists.set(source, paths) + } + } + return [...lists.values()] } /** @@ -105,14 +113,19 @@ const AUTH_SOURCE_WORKFLOWS = [ ] describe('a change to the auth binding starts the jobs that load it', () => { - const users = [ - ...new Set( - JOBS.filter((job) => job.auth).map((job) => job.id.split(' / ')[0]), - ), - ] + /** Each workflow with a job that builds the binding, and those jobs. */ + const usersByWorkflow = new Map() + for (const job of JOBS.filter((entry) => entry.auth)) { + const [relPath, jobName] = job.id.split(' / ') + usersByWorkflow.set(relPath, [ + ...(usersByWorkflow.get(relPath) ?? []), + jobName, + ]) + } + const users = [...usersByWorkflow.keys()] it.each(users)('%s filters on the action', (relPath) => { - for (const paths of filters(relPath)) { + for (const paths of filters(relPath, usersByWorkflow.get(relPath))) { expect( filterCovers(paths, '.github/actions/build-auth-binding/**', same), ).toBe(true) @@ -133,7 +146,7 @@ describe('a change to the auth binding starts the jobs that load it', () => { '%s filters on the auth sources', (relPath) => { expect(shipped.length).toBeGreaterThanOrEqual(4) - const lists = filters(relPath) + const lists = filters(relPath, usersByWorkflow.get(relPath)) expect(lists.length).toBe(2) for (const paths of lists) { expect( diff --git a/scripts/__tests__/integration-workflow-paths.test.mjs b/scripts/__tests__/integration-workflow-paths.test.mjs index c3b1e812a..a44f65539 100644 --- a/scripts/__tests__/integration-workflow-paths.test.mjs +++ b/scripts/__tests__/integration-workflow-paths.test.mjs @@ -2,6 +2,7 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -40,9 +41,6 @@ const STACK_SRC = 'languages/typescript/packages/stack/src' const STACK_MANIFEST = 'languages/typescript/packages/stack/package.json' const CATALOG_MANIFEST = 'pnpm-workspace.yaml' -/** Both trigger events, in the order GitHub evaluates them. */ -const TRIGGER_EVENTS = ['push', 'pull_request'] - /** * The integration workflows, discovered rather than listed: any workflow whose * `CS_IT_SUITE` globs select suites out of `languages/typescript/packages/stack/` is in scope. A @@ -56,14 +54,21 @@ function discoverWorkflows() { } /** - * `on:` parses as the boolean `true` under YAML 1.1 (the "Norway problem"), - * which is why this reads both keys rather than `wf.on`. + * The path lists that decide whether the jobs running a suite run, once each: + * the workflow's `push` and `pull_request` `paths:` lists, and the selector + * filter each such job reads. */ -function triggerBlocks(wf) { - const on = wf.on ?? wf[true] - return TRIGGER_EVENTS.map((event) => on?.[event]).filter( - (block) => block && Array.isArray(block.paths), - ) +function triggerBlocks(relPath) { + const lists = new Map() + for (const [jobId, job] of Object.entries( + readWorkflow(relPath)?.jobs ?? {}, + )) { + if (!job?.env?.CS_IT_SUITE) continue + for (const { source, paths } of jobTriggerLists(relPath, jobId)) { + lists.set(source, { source, paths }) + } + } + return [...lists.values()] } /** Every `CS_IT_SUITE` glob set declared anywhere in the workflow. */ @@ -190,8 +195,9 @@ describe('integration workflow paths filters', () => { for (const relPath of WORKFLOWS) { it(`${relPath} triggers on the manifests pinning its suites' dependencies`, () => { const wf = readWorkflow(relPath) - const blocks = triggerBlocks(wf) - expect(blocks.length).toBe(TRIGGER_EVENTS.length) + const blocks = triggerBlocks(relPath) + // A `push` list, and a `pull_request` list or a selector filter. + expect(blocks.length).toBe(2) const deps = stackDependencies() const files = suiteFiles(suiteGlobs(wf)) @@ -215,7 +221,7 @@ describe('integration workflow paths filters', () => { it(`${relPath} triggers on every source path its suites import`, () => { const wf = readWorkflow(relPath) - const blocks = triggerBlocks(wf) + const blocks = triggerBlocks(relPath) expect(blocks.length).toBeGreaterThan(0) const files = suiteFiles(suiteGlobs(wf)) diff --git a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs index 28d96ad94..8f28725ad 100644 --- a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs +++ b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs @@ -4,6 +4,7 @@ import yaml from 'js-yaml' import { describe, expect, it } from 'vitest' import { filterCovers } from './lib/paths-filter.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -57,9 +58,6 @@ const ACTION = '.github/actions/build-ffi-binding/action.yml' const ACTION_USES = './.github/actions/build-ffi-binding' const CACHE_ACTION = /^actions\/cache(\/(restore|save))?@/ -/** Both trigger events that carry a `paths:` filter, in GitHub's own order. */ -const FILTERED_EVENTS = ['push', 'pull_request'] - /** The package whose Rust both cache keys are about. */ const FFI_PACKAGE = 'languages/typescript/packages/protect-ffi' const FFI_PACKAGE_NAME = '@cipherstash/protect-ffi' @@ -276,21 +274,34 @@ function stepDir(wf, job, step) { /** Every job step of a workflow, paired with the job that owns it. */ function steps(wf) { - return Object.values(wf?.jobs ?? {}).flatMap((job) => - (Array.isArray(job?.steps) ? job.steps : []).map((step) => ({ job, step })), + return Object.entries(wf?.jobs ?? {}).flatMap(([jobId, job]) => + (Array.isArray(job?.steps) ? job.steps : []).map((step) => ({ + jobId, + job, + step, + })), ) } -/** Runs `./.github/actions/build-ffi-binding`, optionally with `wasm: 'true'`. */ -function usesAction(relPath, { wasmOnly = false } = {}) { - return steps(readWorkflow(relPath)).some( - ({ step }) => - typeof step?.uses === 'string' && - step.uses.trim() === ACTION_USES && - (!wasmOnly || String(step?.with?.wasm) === 'true'), +const jobIdsOf = (entries) => [...new Set(entries.map(({ jobId }) => jobId))] + +/** + * The jobs that run `./.github/actions/build-ffi-binding`, optionally with + * `wasm: 'true'`. + */ +function actionJobs(relPath, { wasmOnly = false } = {}) { + return jobIdsOf( + steps(readWorkflow(relPath)).filter( + ({ step }) => + typeof step?.uses === 'string' && + step.uses.trim() === ACTION_USES && + (!wasmOnly || String(step?.with?.wasm) === 'true'), + ), ) } +const usesAction = (relPath, options) => actionJobs(relPath, options).length > 0 + /** * Compiles this package's Rust — through the action, or directly. * @@ -300,10 +311,9 @@ function usesAction(relPath, { wasmOnly = false } = {}) { * EQL workspace has its own, with its own workflows — is not mistaken for this * one. */ -function compilesFfiRust(relPath) { - if (usesAction(relPath)) return true +function ffiRustJobs(relPath) { const wf = readWorkflow(relPath) - return steps(wf).some(({ job, step }) => { + const direct = steps(wf).filter(({ job, step }) => { const run = String(step?.run ?? '') if (!run) return false const targetsPackage = @@ -316,24 +326,28 @@ function compilesFfiRust(relPath) { (inPackage && mentionsAny(run, CARGO_TASKS)) ) }) + return [...new Set([...actionJobs(relPath), ...jobIdsOf(direct)])] } +const compilesFfiRust = (relPath) => ffiRustJobs(relPath).length > 0 + /** - * The `paths:` filters a workflow declares, one entry per filtered event. + * The path lists that decide whether the given jobs run: the workflow's `push` + * and `pull_request` `paths:` lists, and the selector filter each job reads. + * One entry per list, labelled with where it comes from. * - * `on:` parses as the boolean `true` under YAML 1.1 (the "Norway problem"), - * hence `wf.on ?? wf[true]`. An event with no `paths:` is UNFILTERED and is - * skipped rather than failed — it already runs on every diff, so it cannot miss - * one. `tests.yml` and `tests-rust.yml` are both half-cases: `push` is - * `branches: [main]` with no filter, so only their `pull_request` lists are - * here. `tests.yml`'s is `'**'` with one negation, which `filterCovers` reads. + * A job that no list decides is skipped rather than failed: it runs on every + * diff, so it cannot miss one. A selector filter is read as a `paths:` list, + * so a `'**'` with negations is read the same way in either. */ -function filteredEvents(relPath) { - const wf = readWorkflow(relPath) - const on = wf?.on ?? wf?.[true] - return FILTERED_EVENTS.filter((event) => - Array.isArray(on?.[event]?.paths), - ).map((event) => ({ relPath, event, paths: on[event].paths })) +function filteredEvents(relPath, jobIds) { + const lists = new Map() + for (const jobId of jobIds) { + for (const { source, paths } of jobTriggerLists(relPath, jobId)) { + lists.set(source, { relPath, event: source, paths }) + } + } + return [...lists.values()] } const ALL_WORKFLOWS = workflowFiles() @@ -381,6 +395,7 @@ const KEYS = [ what: '`index.node`, the binding every credentialed suite encrypts through', reaches: 'compiles the crate graph behind `index.node`', workflows: ALL_WORKFLOWS.filter(compilesFfiRust), + jobs: ffiRustJobs, expected: [ '.github/workflows/_build-ffi-artifacts.yml', '.github/workflows/integration-drizzle.yml', @@ -401,6 +416,7 @@ const KEYS = [ workflows: ALL_WORKFLOWS.filter((relPath) => usesAction(relPath, { wasmOnly: true }), ), + jobs: (relPath) => actionJobs(relPath, { wasmOnly: true }), expected: [ '.github/workflows/integration-drizzle.yml', '.github/workflows/integration-protect-ffi.yml', @@ -447,7 +463,9 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { for (const key of KEYS) { const inputs = buildInputs(key.step) - const filtered = key.workflows.flatMap(filteredEvents) + const filtered = key.workflows.flatMap((relPath) => + filteredEvents(relPath, key.jobs(relPath)), + ) describe(`the ${key.id} key`, () => { it('hashes build inputs that name real files', () => { @@ -506,7 +524,7 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { ) expect( uncovered, - `These files are hashed into the ${key.id} cache key of \`${ACTION}\`, so an edit to one MISSES the cache and rebuilds ${key.what}. ${relPath} ${key.reaches}, but its \`${event}\` \`paths:\` filter does not list them, so a PR touching only one of them rebuilds the artifact and starts no suite that exercises it. Nothing goes red; nothing ran.\nAdd each to BOTH copies of the filter — GitHub Actions has no YAML anchors, and \`workflow-paths-filter-parity.test.mjs\` fails a one-sided edit.`, + `These files are hashed into the ${key.id} cache key of \`${ACTION}\`, so an edit to one MISSES the cache and rebuilds ${key.what}. ${relPath} ${key.reaches}, but its \`${event}\` \`paths:\` filter does not list them, so a PR touching only one of them rebuilds the artifact and starts no suite that exercises it. Nothing goes red; nothing ran.\nAdd each to every copy of the list — GitHub Actions has no YAML anchors, and \`workflow-paths-filter-parity.test.mjs\` fails a one-sided edit.`, ).toEqual([]) }) } @@ -521,7 +539,7 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { KEYS.flatMap((key) => { const inputs = buildInputs(key.step) return key.workflows - .flatMap(filteredEvents) + .flatMap((relPath) => filteredEvents(relPath, key.jobs(relPath))) .flatMap(({ relPath, paths }) => inputs .filter((pattern) => !filterCovers(paths, pattern, entryCovers)) From a951fff6b9270c98184ed0d3888eb63a89516536 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Wed, 28 Oct 2026 23:37:30 +1100 Subject: [PATCH 3/3] ci(crates): decide in selector jobs which crate checks run Six of the Rust crate workflows used a workflow-level path filter on pull_request. A workflow that a path filter skips leaves its required checks pending, and GitHub's documentation on required status checks says that blocks a merge. Tests (auth refresh cancellation) already had a `changes` job, without the rest of the design. Each workflow now runs on every pull request. A `selector` job on a GitHub-hosted runner decides which jobs run, and fails when a filter gives no answer. A summary job ends each workflow, and passes when every needed job passed or was skipped. The trigger paths are per check, and list what it builds: - CRAP gets one filter per crate. stack-auth's gate also reads stack-profile, and stack-encrypt's reads stack-encrypt-derive, stack-kms, stack-auth and stack-profile. Both read .config/nextest.toml, because the mise tasks use its `ci` profile. - Fuzz adds stack-profile and stack-encrypt-derive, which the fuzzed crates build by path. - Tests (crates) splits in two. The cargo job no longer runs for a change to pnpm-lock.yaml alone. The node bindings job no longer runs for stack-kms, stack-encrypt or stack-guest-abi, and gains package.json and pnpm-workspace.yaml, which its pnpm install reads. - Unused dependencies and Tests (Rust) add the root mise.toml, which pins the rust they build with. - The long auth refresh soak adds stack-profile. CRAP, Fuzz, Miri and Unused dependencies keep excluding Markdown and `.example` files, through dorny's `every` quantifier. Push lists that remain match the union of each selector's filters, which the parity test now checks. --- .github/workflows/crap-crates.yml | 166 +++++++++++++-- .github/workflows/fuzz.yml | 143 +++++++++++-- .github/workflows/miri.yml | 126 ++++++++++-- .github/workflows/tests-auth-refresh.yml | 127 +++++++++--- .github/workflows/tests-crates.yml | 174 +++++++++++++--- .github/workflows/tests-rust.yml | 150 +++++++++++--- .github/workflows/udeps.yml | 148 ++++++++++++-- .../__tests__/auth-refresh-soak-ci.test.mjs | 6 +- scripts/__tests__/crates-selectors.test.mjs | 190 ++++++++++++++++++ .../workflow-paths-filter-parity.test.mjs | 23 +-- 10 files changed, 1090 insertions(+), 163 deletions(-) create mode 100644 scripts/__tests__/crates-selectors.test.mjs diff --git a/.github/workflows/crap-crates.yml b/.github/workflows/crap-crates.yml index 8a325ab1a..f281c94bd 100644 --- a/.github/workflows/crap-crates.yml +++ b/.github/workflows/crap-crates.yml @@ -12,24 +12,14 @@ name: CRAP gate (crates) # which is `cargo crap`; `--format github` turns each offending function into # an inline `::warning` annotation before the job fails. # -# Pull requests only, so `push` has no filter to mirror; recorded in -# scripts/__tests__/workflow-paths-filter-parity.test.mjs. +# The `selector` job decides which crate's gate a pull request runs, and +# `summary` reports the result (ADR 0003). on: - pull_request: - paths: - - packages/stack-auth/** - - packages/stack-encrypt/** - - packages/stack-encrypt-derive/** - - Cargo.toml - - Cargo.lock - - mise.toml - - mise.test.toml - - .cargo-crap.toml - - .github/workflows/crap-crates.yml - # Keep these excludes last so docs-only changes are skipped. - - "!**.md" - - "!**.example" + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. + pull_request: {} workflow_dispatch: {} # Read-only: failures surface as job status and inline annotations. @@ -57,8 +47,118 @@ env: NEXTEST_PROFILE: ci jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + stack_auth: ${{ steps.decide.outputs.stack_auth }} + stack_encrypt: ${{ steps.decide.outputs.stack_encrypt }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + # + # `every` makes a file count only when it matches every rule. That is + # what makes the `!` rules exclude documentation: under dorny's default, + # `some`, a `!` rule matches every other file in the repository. So each + # filter's trigger paths are one brace pattern, written across lines: + # YAML drops a line break that ends in `\`, with the next line's indent. + # + # Each gate scores one crate, and its tests compile that crate's path + # dependencies in the workspace: stack-auth needs stack-profile, and + # stack-encrypt needs stack-encrypt-derive, stack-kms and both of theirs. + # The mise tasks run nextest with the `ci` profile in .config/nextest.toml. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + predicate-quantifier: every + filters: | + stack_auth: + - "{\ + packages/stack-auth/**,\ + packages/stack-profile/**,\ + Cargo.toml,\ + Cargo.lock,\ + mise.toml,\ + mise.test.toml,\ + .cargo-crap.toml,\ + .config/nextest.toml,\ + .github/workflows/crap-crates.yml\ + }" + - "!**/*.md" + - "!**/*.example" + stack_encrypt: + - "{\ + packages/stack-encrypt/**,\ + packages/stack-encrypt-derive/**,\ + packages/stack-kms/**,\ + packages/stack-auth/**,\ + packages/stack-profile/**,\ + Cargo.toml,\ + Cargo.lock,\ + mise.toml,\ + mise.test.toml,\ + .cargo-crap.toml,\ + .config/nextest.toml,\ + .github/workflows/crap-crates.yml\ + }" + - "!**/*.md" + - "!**/*.example" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + STACK_AUTH: ${{ steps.filter.outputs.stack_auth }} + STACK_ENCRYPT: ${{ steps.filter.outputs.stack_encrypt }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + STACK_AUTH=true + STACK_ENCRYPT=true + fi + answers=( + "stack_auth=$STACK_AUTH" + "stack_encrypt=$STACK_ENCRYPT" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + crap-stack-auth: name: CRAP (stack-auth) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.stack_auth == 'true' runs-on: blacksmith-16vcpu-ubuntu-2204 timeout-minutes: 45 steps: @@ -83,6 +183,12 @@ jobs: crap-stack-encrypt: name: CRAP (stack-encrypt) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.stack_encrypt == 'true' runs-on: blacksmith-16vcpu-ubuntu-2204 timeout-minutes: 45 steps: @@ -103,3 +209,31 @@ jobs: - name: Run the CRAP gate run: mise run crap:stack-encrypt -- --format github + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: CRAP gate summary + needs: [selector, crap-stack-auth, crap-stack-encrypt] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index baa8e8cd8..3e84989a3 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -14,24 +14,15 @@ name: Fuzz (crates) # cache, so coverage compounds; it is minimised with `cargo fuzz cmin`, and # any crash reproducer is uploaded as an artifact. # -# cargo-fuzz needs nightly; the tasks run `cargo +nightly fuzz`. Pull requests -# are the only filtered event, so `push` has no filter to mirror; recorded in -# scripts/__tests__/workflow-paths-filter-parity.test.mjs. +# cargo-fuzz needs nightly; the tasks run `cargo +nightly fuzz`. The `selector` +# job decides whether a pull request runs the regression replay, and `summary` +# reports the result (ADR 0003). on: - pull_request: - paths: - - packages/stack-auth/** - - packages/stack-kms/** - - packages/stack-encrypt/** - - Cargo.toml - - Cargo.lock - - mise.toml - - mise.test.toml - - .github/workflows/fuzz.yml - # Keep these excludes last so docs-only changes are skipped. - - "!**.md" - - "!**.example" + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. + pull_request: {} schedule: # Nightly at 04:47 UTC. Scheduled runs fire from the default branch only. - cron: "47 4 * * *" @@ -60,11 +51,99 @@ env: CARGO_NET_GIT_FETCH_WITH_CLI: true jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + fuzz: ${{ steps.decide.outputs.fuzz }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + # + # `every` makes a file count only when it matches every rule. That is + # what makes the `!` rules exclude documentation: under dorny's default, + # `some`, a `!` rule matches every other file in the repository. So each + # filter's trigger paths are one brace pattern, written across lines: + # YAML drops a line break that ends in `\`, with the next line's indent. + # + # Each fuzz crate builds its parent crate by path, and that crate's path + # dependencies with it: stack-profile and stack-encrypt-derive. Their + # `workspace = true` keys are read from the root Cargo.toml. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + predicate-quantifier: every + filters: | + fuzz: + - "{\ + packages/stack-auth/**,\ + packages/stack-profile/**,\ + packages/stack-kms/**,\ + packages/stack-encrypt/**,\ + packages/stack-encrypt-derive/**,\ + Cargo.toml,\ + Cargo.lock,\ + mise.toml,\ + mise.test.toml,\ + .github/workflows/fuzz.yml\ + }" + - "!**/*.md" + - "!**/*.example" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + FUZZ: ${{ steps.filter.outputs.fuzz }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + FUZZ=true + fi + answers=( + "fuzz=$FUZZ" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + fuzz-regression: name: fuzz regression (${{ matrix.slug }}) + needs: [selector] # Not on the nightly schedule, which is the campaign's. A manual dispatch - # runs both jobs. - if: github.event_name != 'schedule' + # runs both jobs. On a pull request, the selector decides. Written as + # exclusions, not a list of events. + if: >- + github.event_name != 'schedule' + && (github.event_name != 'pull_request' + || needs.selector.outputs.fuzz == 'true') runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 strategy: @@ -189,3 +268,31 @@ jobs: name: fuzz-artifacts-${{ matrix.slug }} path: ${{ matrix.dir }}/fuzz/artifacts/** if-no-files-found: ignore + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Fuzz summary + needs: [selector, fuzz-regression, fuzz-campaign] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml index 75711f5fd..0a8a14b56 100644 --- a/.github/workflows/miri.yml +++ b/.github/workflows/miri.yml @@ -9,20 +9,14 @@ name: Miri (guest ABI) # deterministic, so a red run is a real defect or a harness that stopped # compiling, never flake. # -# Pull requests only, so `push` has no filter to mirror; recorded in -# scripts/__tests__/workflow-paths-filter-parity.test.mjs. +# The `selector` job decides whether a pull request runs Miri, and `summary` +# reports the result (ADR 0003). on: - pull_request: - paths: - - packages/stack-guest-abi/** - - Cargo.toml - - Cargo.lock - - mise.toml - - .github/workflows/miri.yml - # Keep these excludes last so docs-only changes are skipped. - - "!**.md" - - "!**.example" + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. + pull_request: {} workflow_dispatch: {} defaults: @@ -38,8 +32,88 @@ env: CARGO_NET_GIT_FETCH_WITH_CLI: true jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + miri: ${{ steps.decide.outputs.miri }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + # + # `every` makes a file count only when it matches every rule. That is + # what makes the `!` rules exclude documentation: under dorny's default, + # `some`, a `!` rule matches every other file in the repository. So each + # filter's trigger paths are one brace pattern, written across lines: + # YAML drops a line break that ends in `\`, with the next line's indent. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + predicate-quantifier: every + filters: | + miri: + - "{\ + packages/stack-guest-abi/**,\ + Cargo.toml,\ + Cargo.lock,\ + mise.toml,\ + .github/workflows/miri.yml\ + }" + - "!**/*.md" + - "!**/*.example" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + MIRI: ${{ steps.filter.outputs.miri }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + MIRI=true + fi + answers=( + "miri=$MIRI" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + miri-stack-guest-abi: name: Miri (stack-guest-abi) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.miri == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 steps: @@ -66,3 +140,31 @@ jobs: - name: Miri run: mise run miri:stack-guest-abi + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Miri summary + needs: [selector, miri-stack-guest-abi] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests-auth-refresh.yml b/.github/workflows/tests-auth-refresh.yml index 505979d30..d4e4eb97e 100644 --- a/.github/workflows/tests-auth-refresh.yml +++ b/.github/workflows/tests-auth-refresh.yml @@ -2,6 +2,8 @@ name: Tests (auth refresh cancellation) # The short run covers every PR, including Version Packages. The longer run # also covers release PRs and changes to the engine, refreshers or toolchain. +# The `selector` job decides whether a pull request runs the longer one, and +# `summary` reports the result (ADR 0003). on: pull_request: {} workflow_dispatch: {} @@ -14,37 +16,116 @@ concurrency: cancel-in-progress: true jobs: - short: - uses: ./.github/workflows/_test-auth-refresh-soak.yml - - changes: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - pull-requests: read + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. outputs: - refresh: ${{ steps.paths.outputs.refresh }} + refresh: ${{ steps.decide.outputs.refresh }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 - id: paths + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + # + # The soak tests are in stack-auth, which builds stack-profile by path. + # release-plz.yml runs the long soak before it publishes stack-auth. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 with: filters: | refresh: - - 'packages/stack-auth/**' - - 'Cargo.toml' - - 'Cargo.lock' - - 'mise*.toml' - - '.config/nextest.toml' - - '.github/workflows/*auth-refresh*.yml' - - '.github/workflows/release-plz.yml' + - "packages/stack-auth/**" + - "packages/stack-profile/**" + - "Cargo.toml" + - "Cargo.lock" + - "mise*.toml" + - ".config/nextest.toml" + - ".github/workflows/*auth-refresh*.yml" + - ".github/workflows/release-plz.yml" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + REFRESH: ${{ steps.filter.outputs.refresh }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + REFRESH=true + fi + answers=( + "refresh=$REFRESH" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + + # Runs on every change, so the selector does not decide it. + short: + uses: ./.github/workflows/_test-auth-refresh-soak.yml long: - needs: changes - if: github.event_name == 'workflow_dispatch' || github.head_ref == 'changeset-release/main' || needs.changes.outputs.refresh == 'true' + needs: [selector] + # Written as an exclusion, not a list of events: every event other than a + # pull request runs the long soak. On a pull request, Version Packages + # runs it too. + if: >- + github.event_name != 'pull_request' + || github.head_ref == 'changeset-release/main' + || needs.selector.outputs.refresh == 'true' uses: ./.github/workflows/_test-auth-refresh-soak.yml with: long: true + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Tests (auth refresh cancellation) summary + needs: [selector, short, long] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests-crates.yml b/.github/workflows/tests-crates.yml index 40cc572cc..381a9bcd1 100644 --- a/.github/workflows/tests-crates.yml +++ b/.github/workflows/tests-crates.yml @@ -10,35 +10,22 @@ name: Tests (crates) # `scripts/__tests__/crates-ci.test.mjs` pins that, and that every mise task in # the five crate `tasks.toml` files is reached from a root workflow. # -# The filter keeps Markdown in: `stack-auth`'s crate docs are its README +# The filters keep Markdown in: `stack-auth`'s crate docs are its README # (`#![doc = include_str!("../README.md")]`), so a README edit changes the # doctests. The suite's `!**.md` exclude would have skipped that change. +# +# The `selector` job decides which jobs a pull request runs, and `summary` +# reports the result (ADR 0003). on: - pull_request: - paths: - - packages/stack-auth/** - - packages/stack-profile/** - - packages/stack-kms/** - - packages/stack-encrypt/** - - packages/stack-encrypt-derive/** - - packages/stack-guest-abi/** - - languages/typescript/packages/auth/** - - languages/typescript/packages/profile/** - - languages/typescript/packages/stack-auth-wasm/** - # Root manifest, lockfile, toolchain pin and cargo config: each can - # change what compiles without touching a crate. The toolchain pin in - # particular changes the trybuild UI snapshots. - - Cargo.toml - - Cargo.lock - - mise.toml - - mise.test.toml - - .cargo/config.toml - - .config/nextest.toml - - pnpm-lock.yaml - - .github/workflows/tests-crates.yml + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. + pull_request: {} push: branches: [main] + # The union of the `selector` job's filters. + # scripts/__tests__/workflow-paths-filter-parity.test.mjs compares them. paths: - packages/stack-auth/** - packages/stack-profile/** @@ -55,7 +42,9 @@ on: - mise.test.toml - .cargo/config.toml - .config/nextest.toml + - package.json - pnpm-lock.yaml + - pnpm-workspace.yaml - .github/workflows/tests-crates.yml workflow_dispatch: {} @@ -77,8 +66,113 @@ env: NEXTEST_PROFILE: ci jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + rust: ${{ steps.decide.outputs.rust }} + node_bindings: ${{ steps.decide.outputs.node_bindings }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # Cargo over the whole root workspace. Root manifest, lockfile, + # toolchain pin and cargo config each change what compiles without + # touching a crate. The toolchain pin in particular changes the + # trybuild UI snapshots. + rust: + - "packages/stack-auth/**" + - "packages/stack-profile/**" + - "packages/stack-kms/**" + - "packages/stack-encrypt/**" + - "packages/stack-encrypt-derive/**" + - "packages/stack-guest-abi/**" + - "languages/typescript/packages/auth/**" + - "languages/typescript/packages/profile/**" + - "languages/typescript/packages/stack-auth-wasm/**" + - "Cargo.toml" + - "Cargo.lock" + - "mise.toml" + - "mise.test.toml" + - ".cargo/config.toml" + - ".config/nextest.toml" + - ".github/workflows/tests-crates.yml" + # The node bindings and stack-auth-wasm, the two crates they wrap, and + # the pnpm install that runs their vitest suites. .cargo/config.toml + # carries the wasm32 rustflags the stack-auth-wasm build needs. + node_bindings: + - "packages/stack-auth/**" + - "packages/stack-profile/**" + - "languages/typescript/packages/auth/**" + - "languages/typescript/packages/profile/**" + - "languages/typescript/packages/stack-auth-wasm/**" + - "Cargo.toml" + - "Cargo.lock" + - "mise.toml" + - ".cargo/config.toml" + - "package.json" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - ".github/workflows/tests-crates.yml" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + RUST: ${{ steps.filter.outputs.rust }} + NODE_BINDINGS: ${{ steps.filter.outputs.node_bindings }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + RUST=true + NODE_BINDINGS=true + fi + answers=( + "rust=$RUST" + "node_bindings=$NODE_BINDINGS" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + rust: name: fmt, clippy, nextest, doctests, rustdoc + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.rust == 'true' runs-on: blacksmith-16vcpu-ubuntu-2204 timeout-minutes: 60 env: @@ -158,6 +252,12 @@ jobs: node-bindings: name: node bindings, napi typings, stack-auth-wasm + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.node_bindings == 'true' runs-on: blacksmith-16vcpu-ubuntu-2204 timeout-minutes: 45 steps: @@ -234,3 +334,31 @@ jobs: - name: wasm-pack test stack-auth-wasm run: mise x -- wasm-pack test --node languages/typescript/packages/stack-auth-wasm + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Tests (crates) summary + needs: [selector, rust, node-bindings] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests-rust.yml b/.github/workflows/tests-rust.yml index 1a1bdc388..7f7ca0774 100644 --- a/.github/workflows/tests-rust.yml +++ b/.github/workflows/tests-rust.yml @@ -10,36 +10,17 @@ name: Tests (Rust) # `languages/typescript/packages/protect-ffi/src/lintWiring.test.ts` asserts the split from the # manifest side; this is the other half, and that test now reads this file. # -# The filter is broader than `crates/**`: dependency, feature and toolchain -# changes all alter what cargo builds without touching a .rs file. +# The trigger paths are broader than `crates/**`: dependency, feature and +# toolchain changes all alter what cargo builds without touching a .rs file. +# The `selector` job holds them, and `summary` reports the result (ADR 0003). on: - pull_request: - paths: - - 'languages/typescript/packages/protect-ffi/crates/**' - - 'languages/typescript/packages/protect-ffi/Cargo.toml' - - 'languages/typescript/packages/protect-ffi/Cargo.lock' - - 'languages/typescript/packages/protect-ffi/mise.toml' - - 'languages/typescript/packages/protect-ffi/package.json' - # Out of that package, and compiled by every check below all the same: - # `crates/protect-ffi/Cargo.toml` carries - # `eql-bindings = { path = "../../../../../../packages/eql/crates/eql-bindings" }`, so - # `cargo test`, `cargo fmt --check` and both clippy passes build that - # tree. A change there can break this job in a PR touching no file under - # languages/typescript/packages/protect-ffi at all. The workspace root manifest comes with it: - # cargo reads it for the crate's workspace context, so a - # `.workspace = true` added to eql-bindings becomes a compile input - # with no other trace. Same two entries as the native cache key in - # `.github/actions/build-ffi-binding`, which is where the list is derived - # from — pinned by - # scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs. - # - # This is the only copy: `push:` below is `branches: [main]` with no - # `paths:`, so main runs a superset of what PRs run (recorded as a - # deliberate asymmetry in workflow-paths-filter-parity.test.mjs). - - 'packages/eql/crates/**' - - 'packages/eql/Cargo.toml' - - '.github/workflows/tests-rust.yml' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. + pull_request: {} + # No path filter here either, so main runs a superset of what pull requests + # run. push: branches: [main] workflow_dispatch: {} @@ -56,8 +37,93 @@ concurrency: cancel-in-progress: true jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + rust: ${{ steps.decide.outputs.rust }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # Out of the package, and compiled by every check all the same: + # `crates/protect-ffi/Cargo.toml` carries + # `eql-bindings = { path = "../../../../../../packages/eql/crates/eql-bindings" }`, + # so `cargo test`, `cargo fmt --check` and both clippy passes build that + # tree. The EQL workspace root manifest comes with it: cargo reads it for + # the crate's workspace context. Same two entries as the native cache key + # in `.github/actions/build-ffi-binding`, which is where the list is + # derived from, pinned by + # scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs. + # + # The root mise.toml pins the rust this job builds with: mise reads the + # parent directory's config, and protect-ffi's own does not pin rust. + rust: + - "languages/typescript/packages/protect-ffi/crates/**" + - "languages/typescript/packages/protect-ffi/Cargo.toml" + - "languages/typescript/packages/protect-ffi/Cargo.lock" + - "languages/typescript/packages/protect-ffi/mise.toml" + - "languages/typescript/packages/protect-ffi/package.json" + - "packages/eql/crates/**" + - "packages/eql/Cargo.toml" + - "mise.toml" + - ".github/workflows/tests-rust.yml" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + RUST: ${{ steps.filter.outputs.rust }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + RUST=true + fi + answers=( + "rust=$RUST" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + rust: name: cargo test + clippy + rustfmt + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.rust == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 45 steps: @@ -132,3 +198,31 @@ jobs: - name: clippy (host + wasm32) working-directory: languages/typescript/packages/protect-ffi run: mise run lint:rust + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Tests (Rust) summary + needs: [selector, rust] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/udeps.yml b/.github/workflows/udeps.yml index db3d33b98..a4480dbb7 100644 --- a/.github/workflows/udeps.yml +++ b/.github/workflows/udeps.yml @@ -7,28 +7,19 @@ name: Unused dependencies (crates) # PRs. Bump it deliberately, after checking a newer nightly builds the # workspace. (The suite pinned 2026-07-10 because the 2026-07-14 nightly ICEs # on the test-harness entrypoint attribute.) +# +# The `selector` job decides whether a pull request runs the check, and +# `summary` reports the result (ADR 0003). on: - pull_request: - paths: - - packages/stack-auth/** - - packages/stack-profile/** - - packages/stack-kms/** - - packages/stack-encrypt/** - - packages/stack-encrypt-derive/** - - packages/stack-guest-abi/** - - languages/typescript/packages/auth/** - - languages/typescript/packages/profile/** - - languages/typescript/packages/stack-auth-wasm/** - - Cargo.toml - - Cargo.lock - - mise.test.toml - - .github/workflows/udeps.yml - # Keep these excludes last so docs-only changes are skipped. - - "!**.md" - - "!**.example" + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. + pull_request: {} push: branches: [main] + # The same files as the `selector` job's filter. + # scripts/__tests__/workflow-paths-filter-parity.test.mjs compares them. paths: - packages/stack-auth/** - packages/stack-profile/** @@ -41,6 +32,7 @@ on: - languages/typescript/packages/stack-auth-wasm/** - Cargo.toml - Cargo.lock + - mise.toml - mise.test.toml - .github/workflows/udeps.yml - "!**.md" @@ -66,8 +58,100 @@ env: NIGHTLY_TOOLCHAIN: nightly-2026-07-10 jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + udeps: ${{ steps.decide.outputs.udeps }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + # + # `every` makes a file count only when it matches every rule. That is + # what makes the `!` rules exclude documentation: under dorny's default, + # `some`, a `!` rule matches every other file in the repository. So each + # filter's trigger paths are one brace pattern, written across lines: + # YAML drops a line break that ends in `\`, with the next line's indent. + # + # Every crate of the root workspace, and the files that change what it + # resolves. mise.toml pins the rust that mise-action installs. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + predicate-quantifier: every + filters: | + udeps: + - "{\ + packages/stack-auth/**,\ + packages/stack-profile/**,\ + packages/stack-kms/**,\ + packages/stack-encrypt/**,\ + packages/stack-encrypt-derive/**,\ + packages/stack-guest-abi/**,\ + languages/typescript/packages/auth/**,\ + languages/typescript/packages/profile/**,\ + languages/typescript/packages/stack-auth-wasm/**,\ + Cargo.toml,\ + Cargo.lock,\ + mise.toml,\ + mise.test.toml,\ + .github/workflows/udeps.yml\ + }" + - "!**/*.md" + - "!**/*.example" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + UDEPS: ${{ steps.filter.outputs.udeps }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + UDEPS=true + fi + answers=( + "udeps=$UDEPS" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + udeps: name: Check unused Rust dependencies + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.udeps == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 45 steps: @@ -92,3 +176,31 @@ jobs: - name: Check for unused dependencies run: cargo "+$NIGHTLY_TOOLCHAIN" udeps --workspace --all-features --all-targets + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Unused dependencies summary + needs: [selector, udeps] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/scripts/__tests__/auth-refresh-soak-ci.test.mjs b/scripts/__tests__/auth-refresh-soak-ci.test.mjs index d35763e35..5f2583970 100644 --- a/scripts/__tests__/auth-refresh-soak-ci.test.mjs +++ b/scripts/__tests__/auth-refresh-soak-ci.test.mjs @@ -29,10 +29,10 @@ describe('auth refresh cancellation soak gates', () => { "github.head_ref == 'changeset-release/main'", ) expect(prs.jobs.long.if).toContain( - "needs.changes.outputs.refresh == 'true'", + "needs.selector.outputs.refresh == 'true'", ) - const paths = prs.jobs.changes.steps.find((step) => step.id === 'paths') - expect(paths.with.filters).toContain("'packages/stack-auth/**'") + const paths = prs.jobs.selector.steps.find((step) => step.id === 'filter') + expect(paths.with.filters).toContain('"packages/stack-auth/**"') }) it('requires a successful long soak before publishing stack-auth', () => { diff --git a/scripts/__tests__/crates-selectors.test.mjs b/scripts/__tests__/crates-selectors.test.mjs new file mode 100644 index 000000000..874048797 --- /dev/null +++ b/scripts/__tests__/crates-selectors.test.mjs @@ -0,0 +1,190 @@ +import { describe, expect, it } from 'vitest' +import { readSelectorJob, selectedJobs } from './lib/selector.mjs' + +/** + * What the selector jobs of the Rust crate workflows decide for a pull + * request. Each check's trigger paths list the crates it builds, including + * their path dependencies in the workspace, and the configuration its tools + * read. The gates that excluded documentation before still do. + */ + +const WORKFLOWS = { + crap: '.github/workflows/crap-crates.yml', + fuzz: '.github/workflows/fuzz.yml', + miri: '.github/workflows/miri.yml', + udeps: '.github/workflows/udeps.yml', + crates: '.github/workflows/tests-crates.yml', + rust: '.github/workflows/tests-rust.yml', + refresh: '.github/workflows/tests-auth-refresh.yml', +} + +/** The jobs a pull request runs, without the two that always run. */ +const checks = (relPath, files, options) => + selectedJobs(relPath, files, options).filter( + (id) => id !== 'selector' && id !== 'summary', + ) + +describe('the crate workflows', () => { + it.each(Object.values(WORKFLOWS))('%s has a selector job', (relPath) => { + expect(readSelectorJob(relPath)).not.toBeNull() + }) +}) + +describe('the CRAP gate selector', () => { + const run = (files) => checks(WORKFLOWS.crap, files) + + it('runs the gate of the crate that changed', () => { + expect(run(['packages/stack-auth/src/lib.rs'])).toEqual([ + 'crap-stack-auth', + 'crap-stack-encrypt', + ]) + expect(run(['packages/stack-encrypt/src/lib.rs'])).toEqual([ + 'crap-stack-encrypt', + ]) + }) + + it("runs a gate for a change to that crate's path dependencies", () => { + expect(run(['packages/stack-kms/src/lib.rs'])).toEqual([ + 'crap-stack-encrypt', + ]) + expect(run(['packages/stack-profile/src/lib.rs'])).toEqual([ + 'crap-stack-auth', + 'crap-stack-encrypt', + ]) + }) + + it('skips both gates for documentation and other crates', () => { + expect(run(['packages/stack-auth/README.md'])).toEqual([]) + expect(run(['packages/stack-guest-abi/src/lib.rs'])).toEqual([]) + }) +}) + +describe('the fuzz selector', () => { + const run = (files) => checks(WORKFLOWS.fuzz, files) + + it('replays the corpus for a change to a fuzzed crate or its dependencies', () => { + expect(run(['packages/stack-kms/fuzz/fuzz_targets/a.rs'])).toEqual([ + 'fuzz-regression', + ]) + expect(run(['packages/stack-profile/src/lib.rs'])).toEqual([ + 'fuzz-regression', + ]) + expect(run(['packages/stack-encrypt-derive/src/lib.rs'])).toEqual([ + 'fuzz-regression', + ]) + }) + + it('skips the replay for documentation and other crates', () => { + expect(run(['packages/stack-encrypt/docs/adr/0001-example.md'])).toEqual([]) + expect(run(['packages/stack-guest-abi/src/lib.rs'])).toEqual([]) + }) + + it('never runs the campaign on a pull request', () => { + expect(run(['packages/stack-auth/src/lib.rs'])).not.toContain( + 'fuzz-campaign', + ) + }) +}) + +describe('the Miri selector', () => { + const run = (files) => checks(WORKFLOWS.miri, files) + + it('runs Miri for the guest ABI crate only', () => { + expect(run(['packages/stack-guest-abi/src/lib.rs'])).toEqual([ + 'miri-stack-guest-abi', + ]) + expect(run(['packages/stack-auth/src/lib.rs'])).toEqual([]) + expect(run(['packages/stack-guest-abi/README.md'])).toEqual([]) + }) +}) + +describe('the unused dependencies selector', () => { + const run = (files) => checks(WORKFLOWS.udeps, files) + + it('runs for any crate of the workspace and the rust pin', () => { + expect(run(['languages/typescript/packages/auth/src/lib.rs'])).toEqual([ + 'udeps', + ]) + expect(run(['mise.toml'])).toEqual(['udeps']) + }) + + it('skips documentation', () => { + expect(run(['packages/stack-kms/README.md'])).toEqual([]) + }) +}) + +describe('the Tests (crates) selector', () => { + const run = (files) => checks(WORKFLOWS.crates, files) + + it('runs only the cargo job for a crate the bindings do not wrap', () => { + expect(run(['packages/stack-encrypt/src/lib.rs'])).toEqual(['rust']) + expect(run(['packages/stack-guest-abi/src/lib.rs'])).toEqual(['rust']) + expect(run(['.config/nextest.toml'])).toEqual(['rust']) + }) + + it('runs both jobs for a crate the bindings wrap', () => { + expect(run(['packages/stack-auth/src/lib.rs'])).toEqual([ + 'node-bindings', + 'rust', + ]) + // The README is stack-auth's crate docs, so its doctests read it. + expect(run(['packages/stack-auth/README.md'])).toEqual([ + 'node-bindings', + 'rust', + ]) + }) + + it('runs only the bindings job for a change to the pnpm install', () => { + expect(run(['pnpm-lock.yaml'])).toEqual(['node-bindings']) + expect(run(['pnpm-workspace.yaml'])).toEqual(['node-bindings']) + }) + + it('runs neither for a change outside the crates', () => { + expect(run(['languages/typescript/packages/stack/src/index.ts'])).toEqual( + [], + ) + }) +}) + +describe('the Tests (Rust) selector', () => { + const run = (files) => checks(WORKFLOWS.rust, files) + + it('runs for protect-ffi Rust, the EQL crates it builds, and the rust pin', () => { + for (const file of [ + 'languages/typescript/packages/protect-ffi/crates/protect-ffi/src/lib.rs', + 'packages/eql/crates/eql-bindings/src/lib.rs', + 'mise.toml', + ]) { + expect(run([file]), file).toEqual(['rust']) + } + }) + + it('skips protect-ffi TypeScript and the EQL SQL', () => { + expect( + run(['languages/typescript/packages/protect-ffi/src/index.ts']), + ).toEqual([]) + expect(run(['packages/eql/src/v3/schema.sql'])).toEqual([]) + }) +}) + +describe('the auth refresh selector', () => { + const run = (files, options) => checks(WORKFLOWS.refresh, files, options) + + it('runs the short soak on every pull request', () => { + expect(run(['README.md'])).toEqual(['short']) + }) + + it('runs the long soak for stack-auth and the crate it builds', () => { + expect(run(['packages/stack-auth/src/lib.rs'])).toEqual(['long', 'short']) + expect(run(['packages/stack-profile/src/lib.rs'])).toEqual([ + 'long', + 'short', + ]) + }) + + it('runs the long soak on Version Packages', () => { + expect( + run(['README.md'], { github: { head_ref: 'changeset-release/main' } }), + ).toEqual(['long', 'short']) + }) +}) diff --git a/scripts/__tests__/workflow-paths-filter-parity.test.mjs b/scripts/__tests__/workflow-paths-filter-parity.test.mjs index 93d4facab..73b76ae25 100644 --- a/scripts/__tests__/workflow-paths-filter-parity.test.mjs +++ b/scripts/__tests__/workflow-paths-filter-parity.test.mjs @@ -143,15 +143,6 @@ const EXPECTED_ASYMMETRIES = new Map([ // own `packages/stack-*/Cargo.toml`, which the filter names. 'push is the only trigger; a pull_request copy would make a crates.io publisher reachable from a fork', ], - [ - '.github/workflows/tests-rust.yml', - // `push:` is `branches: [main]` with no `paths:`, so every push to main runs - // the Rust checks and only pull requests are filtered. There is no second - // list to drift from, and the direction that would hurt — a `push` filter - // narrower than the `pull_request` one, letting a change land on main - // unchecked — is unreachable when `push` is unfiltered. - 'push is deliberately unfiltered (branches: [main]), so main runs a superset of what PRs run', - ], [ '.github/workflows/tests.yml', // `push: branches: [main]` has no `paths:`, so main runs everything and only @@ -186,26 +177,14 @@ const EXPECTED_ASYMMETRIES = new Map([ // PR filter) does not exist because PRs never run it. 'push to main and a nightly schedule are the only triggers; a 60-minute bench deliberately stays off the PR path', ], - // The four PR gates ported with the stack-* crates. Each has no `push:` + // The PR gates ported with the stack-* crates. Each has no `push:` // trigger, as in cipherstash-suite: they gate a change before it merges, // and the post-merge coverage of the same crates is tests-crates.yml and // tests-golang.yml, which filter both events and are parity-checked. - [ - '.github/workflows/crap-crates.yml', - 'pull_request is the only filtered trigger; a CRAP gate on a merged change reports too late to block it', - ], [ '.github/workflows/mutants.yml', 'pull_request is the only filtered trigger; `--in-diff` needs a PR base to diff against, and a dispatch sweeps unfiltered', ], - [ - '.github/workflows/fuzz.yml', - 'pull_request (corpus regression) is the only filtered trigger; the campaign runs on a nightly schedule, which takes no paths', - ], - [ - '.github/workflows/miri.yml', - 'pull_request is the only filtered trigger; Miri is deterministic, so a post-merge run would repeat the PR result', - ], ]) describe('paths filters are written twice, identically', () => {