From 62bfe0a896a50f501f071d31c8935e0e0bdabfcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Fri, 2 Oct 2026 18:17:26 +0200 Subject: [PATCH 01/50] fix(github): long marker fields keep a digest so distinct keys never share a comment (#720) * fix(github): long marker fields keep a digest so distinct keys never share a comment * docs(changeset): note one-time duplicate for long marker fields --- .changeset/github-long-key-marker.md | 5 ++++ packages/github/src/post.ts | 19 ++++++++++++--- packages/github/src/reporter.ts | 20 ++++++++++++--- packages/github/tests/unit/post.test.ts | 16 ++++++++++++ packages/github/tests/unit/reporter.test.ts | 27 ++++++++++++++++++--- 5 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 .changeset/github-long-key-marker.md diff --git a/.changeset/github-long-key-marker.md b/.changeset/github-long-key-marker.md new file mode 100644 index 000000000..4ca6da926 --- /dev/null +++ b/.changeset/github-long-key-marker.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/github": patch +--- + +Two matrix jobs whose `key`, workflow, or job differ only after the first 200 encoded characters now keep their own pull request comments instead of overwriting one. A long field's marker ends in a digest of the whole value; markers of shorter fields are unchanged, so existing comments are still found. A job with a field that long posts one new comment after upgrading and leaves its old one behind. The reporter only takes a comment whose first line is its marker, so a reply quoting the marker is never edited. diff --git a/packages/github/src/post.ts b/packages/github/src/post.ts index b3221d185..096a323b2 100644 --- a/packages/github/src/post.ts +++ b/packages/github/src/post.ts @@ -1,7 +1,7 @@ /** * One comment per pull request, kept current: the reporter looks for the - * comment carrying its marker and edits it, and creates one only when none - * exists. The REST calls are the three GitHub documents them as: list the + * comment whose first line is its marker and edits it, and creates one only + * when none exists. The REST calls are the three GitHub documents them as: list the * issue's comments, update a comment, create a comment. * * Every request carries the reporter's abort signal. Anything GitHub answers @@ -60,14 +60,25 @@ async function request(params: PostParams, method: Method, route: string, body?: throw new Error(`GitHub responded ${response.status} to ${method} ${route}${detail === '' ? '' : `: ${detail}`}`); } +/** + * Whether a comment body is the reporter's own for this marker: its first + * line is the marker, exactly. The reporter has always written it there, so + * a marker quoted further down, as in a reply, is not a match, and neither + * is a first line that only starts or ends with it. + */ +function carriesMarker(body: string, marker: string): boolean { + const end = body.indexOf('\n'); + const firstLine = end === -1 ? body : body.slice(0, end); + return firstLine.replace(/\r$/, '') === marker; +} + /** The id of the comment carrying the marker, or undefined when no page has one. */ async function findComment(params: PostParams, issue: string): Promise { for (let page = 1; page <= MAX_PAGES; page += 1) { const comments = await request(params, 'GET', `${issue}/comments?per_page=${PER_PAGE}&page=${page}`); if (!Array.isArray(comments)) throw new Error('GitHub answered the comment list with a body that is not a list'); const found = comments.find( - (comment: unknown) => - isRecord(comment) && typeof comment['body'] === 'string' && comment['body'].includes(params.marker), + (comment: unknown) => isRecord(comment) && typeof comment['body'] === 'string' && carriesMarker(comment['body'], params.marker), ); if (isRecord(found) && typeof found['id'] === 'number') return found['id']; if (comments.length < PER_PAGE) return undefined; diff --git a/packages/github/src/reporter.ts b/packages/github/src/reporter.ts index 6e9ec0b93..e60892afc 100644 --- a/packages/github/src/reporter.ts +++ b/packages/github/src/reporter.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { existsSync } from 'node:fs'; import { appendFile, readFile } from 'node:fs/promises'; import path from 'node:path'; @@ -58,17 +59,28 @@ export function github(options: GitHubOptions = {}): Reporter { } /** - * Each encoded marker field is cut to this many characters, so four fields + * Each encoded marker field is at most this many characters, so four fields * and their names stay under a kilobyte whatever the input, well inside the - * room the page leaves under GitHub's size limit. The cut - * never lands inside a percent escape, and it is the same on every run. + * room the page leaves under GitHub's size limit. */ const MAX_MARKER_FIELD_CHARS = 200; +/** Hex characters of the SHA-256 digest that stands in for a long field's cut tail. */ +const MARKER_DIGEST_CHARS = 16; +/** + * One marker field, URI-encoded. A field that fits is the encoded value + * itself, as it always was, so comments already on open pull requests are + * still found. A longer one keeps a readable prefix, never cut inside a + * percent escape, then `#` and a digest of the whole value: encoding escapes + * every `#`, so no value that fits can read the same, and two long values + * that share a prefix still differ. + */ function markerField(value: string): string { const encoded = encodeURIComponent(value); if (encoded.length <= MAX_MARKER_FIELD_CHARS) return encoded; - return encoded.slice(0, MAX_MARKER_FIELD_CHARS).replace(/%[0-9A-F]?$/, ''); + const digest = createHash('sha256').update(value).digest('hex').slice(0, MARKER_DIGEST_CHARS); + const prefix = encoded.slice(0, MAX_MARKER_FIELD_CHARS - MARKER_DIGEST_CHARS - 1).replace(/%[0-9A-F]?$/, ''); + return `${prefix}#${digest}`; } /** diff --git a/packages/github/tests/unit/post.test.ts b/packages/github/tests/unit/post.test.ts index 42e96758b..af6359b2d 100644 --- a/packages/github/tests/unit/post.test.ts +++ b/packages/github/tests/unit/post.test.ts @@ -49,6 +49,22 @@ describe('upsertComment', () => { expect(calls.filter((call) => call.method === 'POST')).toHaveLength(0); }); + it('matches only a comment whose first line is the marker, not one that quotes it or extends it', async () => { + const { fetch, calls } = fakeGitHub({ + 'GET *': () => + json(200, [ + { id: 1, body: `> ${MARKER}\n> quoted in a reply` }, + { id: 2, body: `### e2e\n${MARKER}` }, + { id: 3, body: `${MARKER.replace(' -->', ' key=a -->')}\nanother job` }, + { id: 4, body: `${MARKER} extra\nnot the marker line` }, + { id: 5, body: `${MARKER}\r\nedited on the web` }, + ]), + 'PATCH /repos/octo/app/issues/comments/5': () => json(200, { id: 5, html_url: 'https://github.com/octo/app/pull/41#issuecomment-5' }), + }); + await expect(upsertComment(params(fetch))).resolves.toBe('https://github.com/octo/app/pull/41#issuecomment-5'); + expect(calls.map((call) => call.method)).toEqual(['GET', 'PATCH']); + }); + it('stops searching after a hundred full pages and posts a new comment', async () => { const { fetch, calls } = fakeGitHub({ 'GET *': (call) => json(200, page(Number(new URL(call.url).searchParams.get('page')) * 100, 100)), diff --git a/packages/github/tests/unit/reporter.test.ts b/packages/github/tests/unit/reporter.test.ts index dcdd76d7b..176265e4c 100644 --- a/packages/github/tests/unit/reporter.test.ts +++ b/packages/github/tests/unit/reporter.test.ts @@ -160,17 +160,38 @@ describe('reportRun', () => { const blank = deps({ ...actionsEnv, GITHUB_TOKEN: 'ghs' }); await reportRun(failedRun, signal, { key: '' }, blank.deps); expect(postedBody(blank.calls).startsWith('\n')).toBe(true); - // A long key is cut the same way every run, so the marker stays bounded and findable. + // A key that fits is the encoded key itself, byte for byte, so comments already on open pull requests are found. + const fits = deps({ ...actionsEnv, GITHUB_TOKEN: 'ghs' }); + await reportRun(failedRun, signal, { key: 'k'.repeat(200) }, fits.deps); + expect(postedBody(fits.calls).startsWith(`\n`)).toBe(true); + // A long key keeps a prefix and a digest of the whole key, the same every run, so the marker stays bounded and findable. const long = deps({ ...actionsEnv, GITHUB_TOKEN: 'ghs' }); await reportRun(failedRun, signal, { key: 'k'.repeat(5_000) }, long.deps); - expect(postedBody(long.calls).startsWith(`\n`)).toBe(true); + expect(postedBody(long.calls).startsWith(`\n`)).toBe(true); // The cut is on the encoded form and never inside a percent escape, so emoji cannot outgrow the marker. const emoji = deps({ ...actionsEnv, GITHUB_TOKEN: 'ghs', GITHUB_WORKFLOW: '💥'.repeat(300), GITHUB_JOB: 'ü'.repeat(300) }); await reportRun(failedRun, signal, { key: '💥'.repeat(300) }, emoji.deps); const marker = /^/.exec(postedBody(emoji.calls))?.[0] ?? ''; expect(marker.length).toBeGreaterThan(0); expect(marker.length).toBeLessThanOrEqual(1_024); - expect(marker).toMatch(/ key=(?:%[0-9A-F]{2})+ -->$/); + expect(marker).toMatch(/ key=(?:%[0-9A-F]{2})+#[0-9a-f]{16} -->$/); + }); + + it('keeps two long keys that share their first 200 characters on their own comments', async () => { + const markers: string[] = []; + for (const key of [`android-${'x'.repeat(200)}-A`, `android-${'x'.repeat(200)}-B`]) { + const d = deps({ ...actionsEnv, GITHUB_TOKEN: 'ghs' }); + await reportRun(failedRun, signal, { key }, d.deps); + markers.push(postedBody(d.calls).split('\n', 1)[0] ?? ''); + } + expect(markers[0]).toMatch(/^$/); + expect(markers[0]).not.toBe(markers[1]); + // B does not take over the comment A posted; it posts its own. + const comments = [{ id: 1, body: `${markers[0]}\n### e2e android A` }]; + const gh = fakeGitHub({ 'GET *': () => json(200, comments), 'POST *': () => posted.clone() }); + const d = deps({ ...actionsEnv, GITHUB_TOKEN: 'ghs' }, gh.fetch); + await reportRun(failedRun, signal, { key: `android-${'x'.repeat(200)}-B` }, d.deps); + expect(gh.calls.map((call) => call.method)).toEqual(['GET', 'POST']); }); it.each([ From dec903bd4c20a58cd01a4c09e7ab55ecae6a1cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Fri, 2 Oct 2026 18:30:49 +0200 Subject: [PATCH 02/50] fix(e2e): redact plain-string secret values in titles, labels, and agent input (#727) * fix(e2e): redact plain-string secret values in titles, labels, and agent input * fix(e2e): redact explore goal, keep unique() templates aligned with redacted params * test(e2e): use literal escaped marker in junit assertion * fix(e2e): refuse param keys that redact alike, bound explore goal after redaction * fix(e2e): redact titles before NFC, drop duplicate-title heuristic, clarify docs * fix(e2e): redact agent context per step, repair text before cut, re-check title limit * fix(e2e): keep __proto__ param key when redacting, scope params wording to act --- .changeset/plain-string-secret-redaction.md | 5 + docs/executors.mdx | 2 +- docs/security.mdx | 18 +- packages/e2e/src/agent/act-validation.ts | 62 ++++++ packages/e2e/src/agent/act.ts | 26 ++- packages/e2e/src/agent/invocation.ts | 20 +- packages/e2e/src/collect/collect.ts | 4 +- packages/e2e/src/collect/registry.ts | 40 +++- packages/e2e/src/explore/index.ts | 23 ++- packages/e2e/src/run/execute.ts | 11 +- packages/e2e/src/run/realm.ts | 3 + packages/e2e/src/run/secrecy.ts | 41 +++- packages/e2e/src/run/standalone.ts | 5 +- packages/e2e/src/run/steps.ts | 7 +- packages/e2e/src/run/worker/entry.ts | 3 +- .../e2e/tests/integration/explore.test.ts | 54 +++++ .../integration/plain-string-secrets.test.ts | 192 ++++++++++++++++++ .../e2e/tests/unit/act-validation.test.ts | 44 +++- packages/e2e/tests/unit/realm.test.ts | 2 +- packages/e2e/tests/unit/registry.test.ts | 48 +++++ .../e2e/tests/unit/session-secrecy.test.ts | 20 +- packages/e2e/tests/unit/step-progress.test.ts | 14 ++ skills/e2e/references/writing-tests.md | 5 + 23 files changed, 606 insertions(+), 43 deletions(-) create mode 100644 .changeset/plain-string-secret-redaction.md create mode 100644 packages/e2e/tests/integration/plain-string-secrets.test.ts diff --git a/.changeset/plain-string-secret-redaction.md b/.changeset/plain-string-secret-redaction.md new file mode 100644 index 000000000..69aa70abf --- /dev/null +++ b/.changeset/plain-string-secret-redaction.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +A registered secret value passed as a plain string is now redacted in more places: test and describe titles (and the test ids, reporter output, and artifact and failure-page file names derived from them), step labels, `agent.act` and `agent.assert` instructions and params as executors and the model receive them, `agentContext`, judgment prompts, and an `e2e explore` goal. Error messages, observations, and worker console output already masked it. A test whose title spells out a secret gets a new id, so `--last-failed` from an earlier run does not select it once. Not covered: a value only a secret provider returns stays in titles, and a Playwright trace of a session no secret reached is kept as recorded. diff --git a/docs/executors.mdx b/docs/executors.mdx index 773692323..bbe7cc1d4 100644 --- a/docs/executors.mdx +++ b/docs/executors.mdx @@ -128,7 +128,7 @@ The executor receives this context: | Member | What it is | | --- | --- | -| `ctx.step` | The instruction, params, and `index` in the attempt's timeline | +| `ctx.step` | The instruction, params, and `index` in the attempt's timeline. A registered secret value the test spelled into the instruction or a param string reads ``; a `secrets.get()` handle is a placeholder you fill with `typeSecret` | | `ctx.observe({ tree, pixels })` | Redacted screen text, with an optional node tree and masked screenshot | | `ctx.actions` | Checked and recorded actions: `tap`, `doubleTap`, `longPress`, `secondaryTap`, `hover`, `type`, `typeSecret`, `press`, `select`, `check`, `drag`, `scrollTo`, `scrollUntil`, `upload`, `scroll`, `navigate`, `back`, and the point and keyboard verbs below | | `ctx.target` | The target's `name` and `platform`, and `verbs`, the actions its engine supports. Offer only these to the model. | diff --git a/docs/security.mdx b/docs/security.mdx index c99c4935d..a3b07a879 100644 --- a/docs/security.mdx +++ b/docs/security.mdx @@ -130,7 +130,18 @@ password on screen is not masked in pixels. The runner replaces every occurrence of a registered secret value with `` in model input, reports, a worker process's console output, and the trace text of a session a secret was filled on or an engine option -holds. It matches the value as written, JSON-escaped, HTML-escaped, and +holds. That includes a value the test passes as a plain string rather than +a `secrets.get()` handle: test and `describe` titles, step labels (an +`app.open` URL, a locator's text), an `agent.act` or `agent.assert` +instruction, the strings in `agent.act` params, `agentContext`, and an +`e2e explore` goal. Titles are redacted as the test file is collected, so +the test id, `--grep`, the reporters, and the artifact and failure-page file +names all see ``, and two titles in one file that differ only +in a secret value are a duplicate title path. A custom executor's +`ctx.step` and the built-in agent's prompt carry the marker, so only a +handle can type the real value. + +Matching covers the value as written, JSON-escaped, HTML-escaped, and percent-encoded, in any letter case (so a CSS `text-transform` does not hide it), and with its whitespace collapsed. Text cut at a length limit is masked too when it ends with the first 8 or more characters of a value (half of a @@ -157,6 +168,11 @@ ordinary text with it. - Text cut at a length limit is matched by its prefix only in the value's own form: a case mapping that changes length (`ß` shown as `SS`) or whitespace collapsed to one space escapes the prefix rule. +- A title is redacted when its file is collected, before any provider + runs, so a value a secret provider returns is not redacted from titles. +- A Playwright trace from a session no secret reached is kept as recorded, + so a value the test only spelled into a URL or a locator stays in its + text. - When tests run in the runner's own process, with the config or tests passed in memory by an embedding host, their console output is not redacted. diff --git a/packages/e2e/src/agent/act-validation.ts b/packages/e2e/src/agent/act-validation.ts index 5455b5bed..a042319ae 100644 --- a/packages/e2e/src/agent/act-validation.ts +++ b/packages/e2e/src/agent/act-validation.ts @@ -120,6 +120,68 @@ export function validateParams(params: AgentParams | undefined): { }; } +/** + * Projected params with `redact` applied to every string in them, object + * keys included, and their `unique()` templates moved with their leaves: a + * template's pointer follows the redacted keys and its value is redacted as + * the leaf is, so the trace cache slots what the params hold. + */ +export function redactParams( + params: Readonly>, + templates: readonly ParamTemplate[], + redact: (text: string) => string, +): { params: Readonly>; templates: readonly ParamTemplate[] } { + const pointers = new Map(); + const redacted = redactJson(params, redact, { from: '', to: '' }, pointers) as Readonly>; + return { + params: redacted, + templates: templates.map((template) => ({ + pointer: pointers.get(template.pointer) ?? template.pointer, + value: redact(template.value), + })), + }; +} + +/** + * `value` with `redact` applied to every string in it, object keys included. + * Records in `pointers` where each node's pointer (`at.from`) lands once its + * keys are redacted (`at.to`). Two keys of one object that redact alike would + * leave one value standing for both, so that is `INVALID_ARGUMENT`. + */ +function redactJson( + value: JsonValue, + redact: (text: string) => string, + at: { readonly from: string; readonly to: string }, + pointers: Map, +): JsonValue { + pointers.set(at.from, at.to); + if (typeof value === 'string') return redact(value); + if (Array.isArray(value)) { + return value.map((entry, index) => + redactJson(entry, redact, { from: paramPointer(at.from, index), to: paramPointer(at.to, index) }, pointers), + ); + } + if (typeof value !== 'object' || value === null) return value; + const out: Record = {}; + for (const [key, entry] of Object.entries(value)) { + const clean = redact(key); + if (Object.hasOwn(out, clean)) { + throw new TestError( + 'INVALID_ARGUMENT', + `agent.act params has two keys that read ${JSON.stringify(clean)} once secret values are redacted; a key cannot be told apart by a secret`, + ); + } + // Defined, not assigned, so a `__proto__` key stays an own property. + Object.defineProperty(out, clean, { + value: redactJson(entry, redact, { from: paramPointer(at.from, key), to: paramPointer(at.to, clean) }, pointers), + enumerable: true, + writable: true, + configurable: true, + }); + } + return out; +} + /** The marked leaves one walk of the params found. */ interface FoundParams { readonly secrets: Map; diff --git a/packages/e2e/src/agent/act.ts b/packages/e2e/src/agent/act.ts index b793e4664..4fc58c7e1 100644 --- a/packages/e2e/src/agent/act.ts +++ b/packages/e2e/src/agent/act.ts @@ -24,7 +24,7 @@ import { ConfigurationError } from '../internal/errors.ts'; import { withAbort, withTimeout } from '../internal/time.ts'; import type { ActOptions, ActResult, AgentErrorCode, JsonValue, ModelInstance, Secret } from '../types.ts'; import { AgentError, CATEGORY_BY_CODE, toAgentError } from './error.ts'; -import { validateActOptions, validateInstruction, validateParams, validateVerdict } from './act-validation.ts'; +import { redactParams, validateActOptions, validateInstruction, validateParams, validateVerdict } from './act-validation.ts'; import { ActionDispatcher } from './action-dispatcher.ts'; import { resolveBoundedBudget, resolveTimeout } from './call-options.ts'; import { RUNTIME_CODES, type ExecutorPixels, type StepExecutorContext, type StepVerdict } from './executor.ts'; @@ -137,13 +137,17 @@ export async function runAssertStep( */ export async function dispatchAgentStep( runtime: AgentContext, - spec: DispatchSpec, + raw: DispatchSpec, // Resolved before the step opens, so an unknown name fails the call, not a // recorded step, and the step carries the agent it ran with. - agent: DispatchAgent = runtime.select(spec.agent), + agent: DispatchAgent = runtime.select(raw.agent), ): Promise { + const spec = redactSpec(raw, runtime.redact); + // Redacted per step, not once per agent: the ledger is live, so a value + // resolved since the agent was selected is covered too. + const context = agent.agentContext === undefined ? undefined : runtime.redact(agent.agentContext); return runtime.steps.run('agent', spec.api, spec.instruction, async () => { - const dispatch = new ActDispatch(runtime, spec, agent); + const dispatch = new ActDispatch(runtime, spec, { ...agent, agentContext: context }); try { let verdict: StepVerdict; try { @@ -164,6 +168,20 @@ export async function dispatchAgentStep( }, { verifies: spec.kind === 'assert', agent: agent.name }); } +/** + * The spec with every registered secret value the test spelled into the + * instruction or a param string (a key or a leaf) rewritten to its marker, + * before anything reads it: the executor, its model, the step label, the + * trace cache, and the prior-step ledger all see the same redacted step. A + * `Secret` handle is already a placeholder here and stays fillable through + * `typeSecret`. + */ +function redactSpec(spec: DispatchSpec, redact: (text: string) => string): DispatchSpec { + const instruction = redact(spec.instruction); + if (spec.params === undefined) return { ...spec, instruction }; + return { ...spec, instruction, ...redactParams(spec.params, spec.templates, redact) }; +} + /** One step's wiring: the collaborators, the executor's context, and the verdict mapping. */ class ActDispatch { private readonly accounting: StepAccounting; diff --git a/packages/e2e/src/agent/invocation.ts b/packages/e2e/src/agent/invocation.ts index 828bcb984..2374b4cf2 100644 --- a/packages/e2e/src/agent/invocation.ts +++ b/packages/e2e/src/agent/invocation.ts @@ -184,7 +184,7 @@ export class Invocation { this.agent = runtime.select(options.agent); this.deadline = runtime.engine.deadline(options.timeoutMs); this.pixelTier = options.vision === true || options.vision === 'only'; - this.system = buildSystem(options.task, this.agent.agentContext); + this.system = runtime.redact(buildSystem(options.task, this.agent.agentContext)); this.systemBytes = tokenUpperBound(this.system); } @@ -337,11 +337,19 @@ export class Invocation { for (;;) { this.checkDeadline(); this.consumeModelCall(); - const prompt = buildPrompt({ - ...request.prompt, - ...(this.treeWithheld ? { withholdTree: true } : {}), - ...(repair === undefined ? {} : { repair }), - }); + // The observation is redacted already; the instruction is the test's + // own text, which may spell out a registered value. The repair's raw + // text is redacted before the prompt cuts it, so the cut never splits a + // value into a fragment whole-value matching would miss. + const prompt = this.runtime.redact( + buildPrompt({ + ...request.prompt, + ...(this.treeWithheld ? { withholdTree: true } : {}), + ...(repair === undefined + ? {} + : { repair: { ...repair, rawText: repair.rawText === undefined ? undefined : this.runtime.redact(repair.rawText) } }), + }), + ); // Pixels travel with the observation they were captured for, so the // image and the tree in one request always describe one revision. const images = imagesFor(request.prompt.observation); diff --git a/packages/e2e/src/collect/collect.ts b/packages/e2e/src/collect/collect.ts index b009f7f07..9bf2013b8 100644 --- a/packages/e2e/src/collect/collect.ts +++ b/packages/e2e/src/collect/collect.ts @@ -8,6 +8,7 @@ import { setupTestId, testId } from '../internal/ids.ts'; import { explainModuleError } from '../config/diagnose.ts'; import { importModule } from '../config/load.ts'; import type { ResolvedConfig } from '../config/resolve.ts'; +import { staticSecretLedger } from '../run/secrecy.ts'; import { collectModule, groupTitles, @@ -505,12 +506,13 @@ async function collectFiles( ): Promise<{ files: CollectedFile[]; uncollected: UncollectedFile[] }> { const files: CollectedFile[] = []; const uncollected: UncollectedFile[] = []; + const { redact } = staticSecretLedger(config.allSecrets); for (const file of discovered) { const absolutePath = path.join(config.projectRoot, file); const skippable = narrowed && !isSelected(file); let registration: ModuleRegistration; try { - registration = await collectModule(() => importModule(absolutePath, 'collect'), absolutePath); + registration = await collectModule(() => importModule(absolutePath, 'collect'), absolutePath, redact); } catch (cause) { if (skippable) { uncollected.push({ file, reason: cause instanceof CollectionError ? cause.message : explainModuleError(cause, absolutePath) }); diff --git a/packages/e2e/src/collect/registry.ts b/packages/e2e/src/collect/registry.ts index 994647ced..af790f80a 100644 --- a/packages/e2e/src/collect/registry.ts +++ b/packages/e2e/src/collect/registry.ts @@ -110,7 +110,11 @@ class Collector { /** Real path of the module being collected, the file a test's source prefers. */ private readonly moduleFile: string | undefined; - constructor(moduleFile: string | undefined) { + constructor( + moduleFile: string | undefined, + /** Rewrites a registered secret value in a title to its marker. */ + private readonly redactTitle: (title: string) => string, + ) { this.moduleFile = moduleFile === undefined ? undefined : realPath(moduleFile); } @@ -119,6 +123,20 @@ class Collector { return { tests: this.tests, hooks: this.hooks }; } + /** + * The title as it registers: validated as written, redacted, then + * normalized to NFC, and validated again, since a marker can be longer + * than the value it replaces. + */ + private title(raw: string): string { + const rawError = validateTitle(raw); + if (rawError !== null) throw new CollectionError(rawError); + const title = this.redactTitle(raw).normalize('NFC'); + const error = title === raw.normalize('NFC') ? null : validateTitle(title); + if (error !== null) throw new CollectionError(`${error} once secret values are redacted`); + return title; + } + private assertOpen(api: string): void { if (this.closed) { throw new CollectionError( @@ -137,9 +155,7 @@ class Collector { fixtures: readonly FixtureDefinition[], ): TestCase { this.assertOpen(kind === 'setup' ? 'test.setup()' : 'test()'); - const titleError = validateTitle(title); - if (titleError !== null) throw new CollectionError(titleError); - const normalizedTitle = title.normalize('NFC'); + const normalizedTitle = this.title(title); if (typeof fn !== 'function') throw new CollectionError('test body must be a function'); if (kind === 'setup') { if (this.currentGroup !== undefined) { @@ -187,12 +203,11 @@ class Collector { registerDescribe(title: string, options: DescribeOptions, body: () => unknown): void { this.assertOpen('describe()'); - const titleError = validateTitle(title); - if (titleError !== null) throw new CollectionError(titleError); + const normalizedTitle = this.title(title); if (typeof body !== 'function') throw new CollectionError('describe body must be a function'); validateDescribeOptions(options, this.currentGroup); const group: GroupNode = { - title: title.normalize('NFC'), + title: normalizedTitle, options, parent: this.currentGroup, serial: options.serial === true, @@ -203,7 +218,7 @@ class Collector { const result = body(); if (isPromiseLike(result)) { throw new CollectionError( - `describe body for ${JSON.stringify(title)} must finish synchronously`, + `describe body for ${JSON.stringify(group.title)} must finish synchronously`, ); } } finally { @@ -440,16 +455,21 @@ const collectorSlot = realmSlot('e2e.activeCollector.v1'); /** * Runs `load` with a fresh collector active and returns everything it * registered. `moduleFile` is the absolute path of the module `load` imports; - * a test's source prefers a frame in that file. + * a test's source prefers a frame in that file. `redactTitle` is applied to + * every test and describe title as it registers, so a secret a title spells + * out never reaches a test id, a report, or a path derived from either; the + * runner passes the static secrets of the config (`staticSecretLedger`), the + * same in every process, so ids agree across them. */ export async function collectModule( load: () => Promise, moduleFile?: string, + redactTitle: (title: string) => string = (title) => title, ): Promise { if (collectorSlot.get(globalThis) !== undefined) { throw new CollectionError('collection is already in progress'); } - const collector = new Collector(moduleFile); + const collector = new Collector(moduleFile, redactTitle); collectorSlot.set(globalThis, collector); try { await load(); diff --git a/packages/e2e/src/explore/index.ts b/packages/e2e/src/explore/index.ts index 299a2fc85..84e73dea0 100644 --- a/packages/e2e/src/explore/index.ts +++ b/packages/e2e/src/explore/index.ts @@ -13,12 +13,13 @@ import type { ModuleRegistration, RegisteredTest } from '../collect/registry.ts' import { selectTargets } from '../collect/select.ts'; import { discoverConfig, loadConfigModule, missingConfigError } from '../config/load.ts'; import { missingModelError, resolveAgentConfig } from '../config/agent.ts'; -import { resolveConfig, type ResolvedCredential, type ResolvedTarget } from '../config/resolve.ts'; +import { resolveConfig, type ResolvedConfig, type ResolvedCredential, type ResolvedTarget } from '../config/resolve.ts'; import { ConfigurationError } from '../internal/errors.ts'; import type { ReportExplore } from '../report/build.ts'; import { labelSegment } from '../run/artifacts.ts'; import type { RunEventSink } from '../run/events.ts'; import { run, type RunOutcome } from '../run/runner.ts'; +import { staticSecretLedger } from '../run/secrecy.ts'; import type { AgentConfig, BuiltinReporter, E2EConfig, RecordingMode } from '../types.ts'; import { createExploreBody } from './body.ts'; import { explorerAgent, withFindingTool } from './executor.ts'; @@ -120,7 +121,7 @@ export async function explore(options: ExploreOptions = {}): Promise MAX_GOAL_CHARS) { + throw new ConfigurationError( + 'INVALID_CONFIG', + `the goal must be at most ${MAX_GOAL_CHARS} characters with its secret values redacted, got ${redacted.length}`, + ); + } + return redacted; +} + function resolveBounded( value: number | undefined, bounds: { readonly min: number; readonly max: number; readonly default: number }, diff --git a/packages/e2e/src/run/execute.ts b/packages/e2e/src/run/execute.ts index 6622dde09..096a7d12b 100644 --- a/packages/e2e/src/run/execute.ts +++ b/packages/e2e/src/run/execute.ts @@ -49,7 +49,7 @@ import { isFailedStatus } from './records.ts'; import { runWithRetries } from './retry.ts'; import { runSerialUnit, type SerialHost, type SharedSerialSession } from './serial.ts'; import { interruptedSkip, pairKey, pairResult, repeatSegment, unstartedResult } from './units.ts'; -import { adoptSecrecy, carriedSecrecy, processSecrets, registerDerivedSecrets, resolveSecretValue, sessionSecrecy } from './secrecy.ts'; +import { adoptSecrecy, carriedSecrecy, processSecrets, registerDerivedSecrets, redactForSession, resolveSecretValue, sessionSecrecy, staticSecretLedger } from './secrecy.ts'; import { isSecret } from '../secrets.ts'; import { SessionStaging, SessionStore, targetIdentity, type SessionIdentity } from './sessions.ts'; import { redactTraceArchives } from './trace-redaction.ts'; @@ -184,6 +184,7 @@ export class TargetExecutor implements SerialHost { timeout: options.config.timeout, cleanupTimeout: options.config.cleanupTimeout, runErrors: this.runErrors, + redactTitle: staticSecretLedger(options.config.allSecrets).redact, debug: this.debug, }); this.sessionIdentity = targetIdentity(options.target); @@ -803,10 +804,10 @@ export class TargetExecutor implements SerialHost { const shared = context.kind === 'serial' ? context.shared : undefined; const onProgress = this.options.events?.onProgress; let openSession: TargetSession | null = null; - // Secret values the session has seen never enter an error record; the - // ledger is live, so a value resolved mid-attempt is covered too. - const redact = (text: string): string => - openSession === null ? text : sessionSecrecy(openSession, this.config.allSecrets).ledger.redact(text); + // Secret values the session has seen never enter an error record or a + // step label; the ledger is live, so a value resolved mid-attempt is + // covered too. Before a session opens, the static values still are. + const redact = (text: string): string => redactForSession(openSession ?? undefined, this.config.allSecrets, text); const steps = new StepRecorder(attemptId, { attempt: { id: shared?.attemptId ?? attemptId, index: attemptIndex }, maxEventsPerStep: this.config.limits.maxEventsPerStep, diff --git a/packages/e2e/src/run/realm.ts b/packages/e2e/src/run/realm.ts index 46f67a57d..e997a912a 100644 --- a/packages/e2e/src/run/realm.ts +++ b/packages/e2e/src/run/realm.ts @@ -84,6 +84,8 @@ export interface RealmManagerOptions { readonly cleanupTimeout: number; /** Run-level error sink for hook failures. */ readonly runErrors: RunError[]; + /** What collection rewrites titles with (`collectModule`), so a realm's tests match the collected ids. */ + readonly redactTitle: (title: string) => string; readonly debug?: DebugTrace; } @@ -103,6 +105,7 @@ export class RealmManager { collectModule( () => importModule(file.absolutePath, `${this.options.targetName}-${this.realmCounter}`), file.absolutePath, + this.options.redactTitle, ), ); return this.adopt(registration, file); diff --git a/packages/e2e/src/run/secrecy.ts b/packages/e2e/src/run/secrecy.ts index 27066d537..b7eddba40 100644 --- a/packages/e2e/src/run/secrecy.ts +++ b/packages/e2e/src/run/secrecy.ts @@ -70,6 +70,28 @@ export class SecretExposure { */ export const processSecrets = new SecretLedger(); +/** The static ledger of each secrets map, built once. */ +const staticLedgers = new WeakMap(); + +/** + * A ledger of the static values of `secrets` alone, built once per map. What + * text produced before any session exists is redacted with, such as a test + * title at collection: every process resolving the same config redacts it + * alike, so a test's id agrees between the runner and its workers. A + * provider-backed value is not known yet and is not in it. Shared, so it is + * handed out read-only: nothing registers into it. + */ +export function staticSecretLedger(secrets: ResolvedConfig['allSecrets']): Pick { + let ledger = staticLedgers.get(secrets); + if (ledger === undefined) { + ledger = new SecretLedger( + [...secrets].flatMap(([name, { value }]) => (typeof value === 'string' ? [[name, value] as const] : [])), + ); + staticLedgers.set(secrets, ledger); + } + return ledger; +} + /** Seeds `processSecrets` with the static values of `secrets`, so output before any session opens is covered too. */ export function registerStaticSecrets(secrets: ResolvedConfig['allSecrets']): void { for (const [name, { value }] of secrets) { @@ -136,11 +158,7 @@ export function sessionSecrecy( if (secrecy === undefined) { registerStaticSecrets(secrets); secrecy = { - ledger: new SecretLedger( - [...secrets].flatMap(([name, { value }]) => - typeof value === 'string' ? [[name, value] as const] : [], - ), - ), + ledger: new SecretLedger(staticSecretLedger(secrets).entries()), exposure: new SecretExposure(), }; secrecyBySession.set(session, secrecy); @@ -148,6 +166,19 @@ export function sessionSecrecy( return secrecy; } +/** + * Redacts `text` with every value `session` has seen, or with the static + * values while no session is open yet, so text an attempt produces before + * its session exists (a step label, a fixture error) is covered too. + */ +export function redactForSession( + session: TargetSession | undefined, + secrets: ResolvedConfig['allSecrets'], + text: string, +): string { + return (session === undefined ? staticSecretLedger(secrets) : sessionSecrecy(session, secrets).ledger).redact(text); +} + /** What a saved session carries of the secrecy of the attempt that saved it. */ export interface SavedSecrecy { /** Name and value pairs the saving session learned beyond the static config values; travels only inside the encrypted payload. */ diff --git a/packages/e2e/src/run/standalone.ts b/packages/e2e/src/run/standalone.ts index 8c19a8b3b..de82705ec 100644 --- a/packages/e2e/src/run/standalone.ts +++ b/packages/e2e/src/run/standalone.ts @@ -27,7 +27,7 @@ import type { ProcessPool } from './process-pool.ts'; import type { AppProcesses } from './managed-process.ts'; import { PreparedEngines, recordingNotices, startDeclaredProcesses, validateEngine } from './provision.ts'; import { attemptRecording, type AttemptRecording, type ResolvedRecording } from '../internal/recording-modes.ts'; -import { sessionSecrecy } from './secrecy.ts'; +import { redactForSession, sessionSecrecy } from './secrecy.ts'; import { SessionStore } from './sessions.ts'; import { outputLayout } from './output.ts'; import { StepRecorder, type StepProgress } from './steps.ts'; @@ -136,12 +136,13 @@ export async function openStandaloneAttempt(options: StandaloneAttemptOptions): ...(options.onProgress === undefined ? {} : { onProgress: (_testId: unknown, progress: StepProgress) => options.onProgress?.(progress) }), }, }); + let session: TargetSession | undefined; const steps = new StepRecorder(attemptId, { maxEventsPerStep: config.limits.maxEventsPerStep, projectRoot: config.projectRoot, + redact: (text) => redactForSession(session, config.allSecrets, text), ...(options.onProgress === undefined ? {} : { onProgress: options.onProgress }), }); - let session: TargetSession | undefined; const artifacts = createAttemptArtifacts({ artifactsRoot: layout.artifacts, segments: [target.name, 'sessions', attemptId], diff --git a/packages/e2e/src/run/steps.ts b/packages/e2e/src/run/steps.ts index aece6bd0e..ac8d4d992 100644 --- a/packages/e2e/src/run/steps.ts +++ b/packages/e2e/src/run/steps.ts @@ -266,7 +266,7 @@ export interface StepRecorderOptions { readonly onProgress?: (progress: StepProgress) => void; /** The project root; with it, every step and step error names the test line it came from. */ readonly projectRoot?: string; - /** Replaces secret values in a step error's message and details before the record keeps them. */ + /** Replaces secret values in a step's label, and in its error's message and details, before the record keeps them. */ readonly redact?: (text: string) => string; } @@ -368,10 +368,13 @@ export class StepRecorder { run( kind: StepKind, api: string, - label: string, + rawLabel: string, body: () => Promise, options: StepRunOptions = {}, ): Promise { + // The label is published, reported, and traced as is: a secret the test + // spelled into an instruction or a locator stops here, once. + const label = this.redact?.(rawLabel) ?? rawLabel; const index = this.steps.length; const startedAt = timestamp(); const stack = stepStack(this.projectRoot); diff --git a/packages/e2e/src/run/worker/entry.ts b/packages/e2e/src/run/worker/entry.ts index da26ce84d..6ce691c62 100644 --- a/packages/e2e/src/run/worker/entry.ts +++ b/packages/e2e/src/run/worker/entry.ts @@ -16,7 +16,7 @@ import { AiTraceRecorder, registerAiTraceRecorder } from '../../internal/ai-trac import { DebugTrace } from '../../internal/debug.ts'; import { classifyError, ConfigurationError, serializeError } from '../../internal/errors.ts'; import { StreamRedactor } from '../../internal/redact.ts'; -import { processSecrets, registerStaticSecrets } from '../secrecy.ts'; +import { processSecrets, registerStaticSecrets, staticSecretLedger } from '../secrecy.ts'; import { SessionStore } from '../sessions.ts'; import type { ChildProcessInbound, @@ -161,6 +161,7 @@ async function bootstrap( const registration = await collectModule( () => importModule(unit.absolutePath, `worker-collect-${collectCounter}`), unit.absolutePath, + staticSecretLedger(config.allSecrets).redact, ); const collected = collectFromRegistration(config.projectRoot, unit.absolutePath, registration); const byId = new Map(collected.tests.map((test) => [test.id, test])); diff --git a/packages/e2e/tests/integration/explore.test.ts b/packages/e2e/tests/integration/explore.test.ts index fe034fc6c..05834ecc9 100644 --- a/packages/e2e/tests/integration/explore.test.ts +++ b/packages/e2e/tests/integration/explore.test.ts @@ -224,6 +224,60 @@ describe('e2e explore', () => { expect((outcome as unknown as { notices: string[] }).notices).toEqual([]); }, 120_000); + it('redacts a registered secret value the goal spells out from the title, artifact names, report, and model input', async () => { + const value = 'goal-Secret-5Tq8Wz'; + const goal = `Explore the home page with key ${value}`; + let plans = 0; + const model = installExploreModel({ + plan: () => { + plans += 1; + return plans === 1 + ? { decision: 'step', title: 'Counter', instruction: 'Look at the counter' } + : { decision: 'finish', summary: 'One issue.' }; + }, + loop: (call) => + call.turn === 1 + ? [{ toolName: FINDING_TOOL_NAME, input: COUNTER_FINDING }] + : [{ toolName: 'complete_step', input: { status: 'passed', summary: 'Looked' } }], + }); + const outcome = await explore({ + cwd: project.dir, + rawConfig: { + targets: [{ name: 'web', engine: web(), app: { url: app.url } }] as never, + agents: { default: { model } }, + secrets: { probe: value }, + }, + goal, + maxSteps: 1, + timeoutMs: 180_000, + }); + const result = outcome.report.run.results[0]!; + expect(result.titlePath).toEqual(['Explore the home page with key ']); + expect(outcome.report.run.explore!.goal).toBe('Explore the home page with key '); + expect(fakeCalls[0]!.system).toContain('Exploration goal: Explore the home page with key '); + // The explorer's loop ran, carrying the goal as context. + expect(loopCalls.length).toBeGreaterThan(0); + expect(loopCalls[0]!.system).toContain('Exploration goal: Explore the home page with key '); + const paths = result.attempts[0]!.artifacts.map((artifact) => artifact.path ?? ''); + expect(paths.some((entry) => entry.startsWith('web/explore-the-home-page-with-key-secret-'))).toBe(true); + expect(JSON.stringify([outcome.report, fakeCalls, loopCalls])).not.toContain(value); + expect(readFileSync(path.join(project.dir, '.e2e', 'report.json'), 'utf8')).not.toContain(value); + + // A goal at the ceiling that the markers push past it is refused before anything starts. + const long = `${'a'.repeat(2_000 - value.length - 1)} ${value}`; + await expect( + explore({ + cwd: project.dir, + rawConfig: { + targets: [{ name: 'web', engine: web(), app: { url: app.url } }] as never, + agents: { default: { model } }, + secrets: { 'a-long-secret-name': value }, + }, + goal: long, + }), + ).rejects.toThrow('the goal must be at most 2000 characters with its secret values redacted, got 2009'); + }, 120_000); + it('says to pass --trace on and --video on when a retry mode would record nothing, explore running once', async () => { const model = installExploreModel({ plan: () => ({ decision: 'finish', summary: 'Nothing here.' }), diff --git a/packages/e2e/tests/integration/plain-string-secrets.test.ts b/packages/e2e/tests/integration/plain-string-secrets.test.ts new file mode 100644 index 000000000..7b3b9639a --- /dev/null +++ b/packages/e2e/tests/integration/plain-string-secrets.test.ts @@ -0,0 +1,192 @@ +/** + * A registered secret value the test passes as a plain string, not as a + * `secrets.get()` handle: spelled into test and describe titles, an + * `app.open` URL, a locator, an `agent.act` instruction and its params, and + * the test's agent context, and logged by the test itself. Through the + * worker path, so the runner's collection and the worker's agree on the + * redacted ids: it reaches no executor input, no step label, no report, + * reporter file, or file name under `.e2e`, and no output event. A secret + * handle in the same run still types the real value. Through the built-in + * agent in process: neither the act loop's prompt nor a judgment's carries it. + */ + +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { RunEvent } from '../../src/run/events.ts'; +import type { RunOutcome } from '../../src/run/runner.ts'; +import { fakeCalls, installFakeModel, judgment } from '../helpers/fake-model.ts'; +import { installFakeLoopModel, loopCalls } from '../helpers/fake-loop-model.ts'; +import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts'; +import { contentsUnder, resultByTitle, runProject, runProjectWithConfigFile, type FixtureProject } from '../helpers/run-project.ts'; + +const SECRET = 'plain-Zr8Kq2Wm5Tx9'; +const MARKER = ''; + +const CONFIG = `import type { E2EConfig, StepExecutor } from 'e2e'; +import { appendFileSync } from 'node:fs'; +import { web } from '@e2e-dev/web'; + +const recording: StepExecutor = { + name: 'recording', + version: '1', + cache: 'off', + async runStep(context) { + const { instruction, params } = context.step; + appendFileSync(new URL('./executor-input.jsonl', import.meta.url), JSON.stringify({ instruction, params, agentContext: context.agentContext }) + '\\n'); + if (context.step.secrets.length > 0) { + const observation = await context.observe(); + const id = /#(\\S+) textbox "Focus target"/.exec(observation.text)?.[1]; + if (id === undefined) return { status: 'failed', summary: 'no focus target' }; + await context.actions.typeSecret({ id }, context.step.secrets[0]!.name); + } + return { status: 'passed', summary: 'done' }; + }, +}; + +export default { + targets: [{ name: 'web', engine: web(), app: { url: process.env.APP_URL! } }], + workers: 1, + reporters: ['markdown', 'junit'], + secrets: { probe: ${JSON.stringify(SECRET)} }, + agents: { default: { executor: recording } }, +} satisfies E2EConfig; +`; + +const SUITE = `import { describe, expect, secrets, test } from 'e2e'; +const S = ${JSON.stringify(SECRET)}; + +describe(\`suite \${S}\`, () => { + test(\`title holds \${S}\`, { agentContext: \`context \${S}\` }, async ({ app, agent, screen }) => { + console.log(\`stdout carries \${S}\`); + console.error(\`stderr carries \${S}\`); + await app.open(\`/?token=\${S}\`); + await agent.act(\`inspect \${S} with {note}\`, { params: { note: S, [\`key \${S}\`]: [S] } }); + await expect(screen.getByText(\`missing \${S}\`)).toBeVisible({ timeout: 500 }); + }); +}); + +test('types the secret through its handle', async ({ app, agent, screen }) => { + await app.open(); + await agent.act('paste the key into the focus target', { params: { key: secrets.get('probe') } }); + await expect(screen.getByLabel('Focus target')).toHaveValue(S); +}); +`; + +/** Every path under `dir`, files and directories, relative to it. */ +function pathsUnder(dir: string): string[] { + return readdirSync(dir).flatMap((name) => { + const entry = path.join(dir, name); + return statSync(entry).isDirectory() ? [name, ...pathsUnder(entry).map((inner) => path.join(name, inner))] : [name]; + }); +} + +describe('a secret value passed as a plain string', () => { + let app: FixtureApp; + let outcome: RunOutcome; + let project: FixtureProject; + const events: RunEvent[] = []; + + beforeAll(async () => { + app = await startFixtureApp(); + ({ outcome, project } = await runProjectWithConfigFile( + { 'tests/plain.e2e.ts': SUITE }, + { appUrl: app.url, configSource: CONFIG, runOptions: { onEvent: (event: RunEvent) => { events.push(event); } } }, + )); + }, 120_000); + + afterAll(async () => { + project?.cleanup(); + await app?.close(); + }); + + it('runs the test under its redacted title and id, its error still masked', () => { + const result = resultByTitle(outcome, `title holds ${MARKER}`); + expect(result.status).toBe('failed'); + expect(result.test.titlePath).toEqual([`suite ${MARKER}`, `title holds ${MARKER}`]); + expect(result.test.id).toBe('tests/plain.e2e.ts::suite%20%3Csecret%3Aprobe%3E::title%20holds%20%3Csecret%3Aprobe%3E'); + const attempt = result.attempts.at(-1)!; + expect(attempt.error?.code).toBe('ASSERTION_FAILED'); + expect(attempt.error?.message).toContain(`getByText("missing ${MARKER}")`); + expect(attempt.steps.map((step) => step.label)).toContain(`inspect ${MARKER} with {note}`); + }); + + it('hands the executor the instruction, params, and agent context redacted', () => { + const inputs = readFileSync(path.join(project.dir, 'executor-input.jsonl'), 'utf8').trim().split('\n').map((line) => JSON.parse(line) as unknown); + expect(inputs).toEqual([ + { + instruction: `inspect ${MARKER} with {note}`, + params: { note: MARKER, [`key ${MARKER}`]: [MARKER] }, + agentContext: `context ${MARKER}`, + }, + { instruction: 'paste the key into the focus target', params: { key: { kind: 'secret', name: 'probe', purpose: 'generic-secret' } } }, + ]); + }); + + it('still types the real value through the secret handle', () => { + expect(resultByTitle(outcome, 'types the secret through its handle').status).toBe('passed'); + }); + + it('keeps the value out of every output event, report, reporter file, and file name under .e2e', () => { + const output = events.filter((event) => event.type === 'output'); + expect(output.map((event) => event.text).join('')).toContain(`stdout carries ${MARKER}`); + expect(output.map((event) => event.text).join('')).toContain(`stderr carries ${MARKER}`); + expect(JSON.stringify(events)).not.toContain(SECRET); + expect(JSON.stringify(outcome.report)).not.toContain(SECRET); + const root = path.join(project.dir, '.e2e'); + const names = pathsUnder(root); + expect(names).toContain('junit.xml'); + expect(names).toContain('summary.md'); + expect(names.some((name) => name.startsWith('failures'))).toBe(true); + for (const name of names) expect(name.includes(SECRET), name).toBe(false); + // A trace of a session no secret reached is kept as recorded; the URL the test opened is in it. + for (const [file, text] of contentsUnder(root)) { + if (!file.includes('.zip!')) expect(text.includes(SECRET), file).toBe(false); + } + expect(readFileSync(path.join(root, 'junit.xml'), 'utf8')).toContain('title holds <secret:probe>'); + }); +}); + +const BUILT_IN_SUITE = `import { test, expect } from 'e2e'; +const S = ${JSON.stringify(SECRET)}; + +test('built-in agent', { agentContext: \`context \${S}\` }, async ({ app, agent }) => { + await app.open(); + await agent.act(\`inspect \${S} with {note}\`, { params: { note: S } }); + await agent.assert(\`the page does not show \${S}\`); +}); +`; + +describe('a secret value passed as a plain string to the built-in agent', () => { + let app: FixtureApp; + let outcome: RunOutcome; + let project: FixtureProject; + + beforeAll(async () => { + app = await startFixtureApp(); + const model = installFakeLoopModel(() => [{ toolName: 'complete_step', input: { status: 'passed', summary: 'inspected' } }]); + const judge = installFakeModel(() => judgment(true, 'nothing secret is shown')); + ({ outcome, project } = await runProject( + { 'tests/built-in.e2e.ts': BUILT_IN_SUITE }, + { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model, judge } }, secrets: { probe: SECRET } } }, + )); + }, 120_000); + + afterAll(async () => { + project?.cleanup(); + await app?.close(); + }); + + it('sends the act loop and the judgment the redacted instruction, params, and context', () => { + expect(resultByTitle(outcome, 'built-in agent').status).toBe('passed'); + expect(loopCalls.length).toBeGreaterThan(0); + const [act] = loopCalls; + expect(act!.prompt).toContain(`Execute this test step: inspect ${MARKER} with {note}`); + expect(act!.prompt).toContain(`{"note":"${MARKER}"}`); + expect(act!.system).toContain(`context ${MARKER}`); + expect(fakeCalls).toHaveLength(1); + expect(fakeCalls[0]!.instruction).toBe(`the page does not show ${MARKER}`); + expect(fakeCalls[0]!.system).toContain(`context ${MARKER}`); + expect(JSON.stringify([loopCalls, fakeCalls])).not.toContain(SECRET); + }); +}); diff --git a/packages/e2e/tests/unit/act-validation.test.ts b/packages/e2e/tests/unit/act-validation.test.ts index 2ae93ff6c..853530ae2 100644 --- a/packages/e2e/tests/unit/act-validation.test.ts +++ b/packages/e2e/tests/unit/act-validation.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { validateActOptions } from '../../src/agent/act-validation.ts'; +import { redactParams, validateActOptions, validateParams } from '../../src/agent/act-validation.ts'; +import { templateParams } from '../../src/cache/template.ts'; +import { unique } from '../../src/params.ts'; import { ConfigurationError, TestError } from '../../src/internal/errors.ts'; describe('validateActOptions', () => { @@ -49,3 +51,43 @@ describe('validateActOptions', () => { expect(() => validateActOptions({ vision: true } as never, 0)).toThrow(ConfigurationError); }); }); + +describe('redactParams', () => { + const SECRET = 'sk_param_4Rt9Zq'; + const redact = (text: string) => text.replaceAll(SECRET, ''); + + it('redacts keys and leaves and moves unique() templates with their leaves, so the cache key slots them', () => { + const { projected, templates } = validateParams({ + note: SECRET, + [`account ${SECRET}`]: { name: unique(`E2E ${SECRET} 42`), tags: [SECRET, 7] }, + }); + expect(templates).toEqual([{ pointer: `/account ${SECRET}/name`, value: `E2E ${SECRET} 42` }]); + const redacted = redactParams(projected!, templates, redact); + expect(redacted.params).toEqual({ + note: '', + 'account ': { name: 'E2E 42', tags: ['', 7] }, + }); + expect(redacted.templates).toEqual([{ pointer: '/account /name', value: 'E2E 42' }]); + expect(templateParams(redacted.params, redacted.templates)).toEqual({ + note: '', + 'account ': { name: '{{param:/account /name}}', tags: ['', 7] }, + }); + }); + + it('keeps a __proto__ key as an own property', () => { + const { projected, templates } = validateParams(JSON.parse(`{"__proto__":"${SECRET}","note":"x"}`) as Record); + const redacted = redactParams(projected!, templates, redact); + expect(Object.hasOwn(redacted.params, '__proto__')).toBe(true); + expect(JSON.stringify(redacted.params)).toBe('{"__proto__":"","note":"x"}'); + }); + + it('refuses two keys that redact alike instead of dropping one', () => { + const { projected, templates } = validateParams({ nested: { [`id ${SECRET}`]: 1, 'id ': 2 } }); + expect(() => redactParams(projected!, templates, redact)).toThrow( + expect.objectContaining({ + code: 'INVALID_ARGUMENT', + message: 'agent.act params has two keys that read "id " once secret values are redacted; a key cannot be told apart by a secret', + }), + ); + }); +}); diff --git a/packages/e2e/tests/unit/realm.test.ts b/packages/e2e/tests/unit/realm.test.ts index ae21d3830..daabc8076 100644 --- a/packages/e2e/tests/unit/realm.test.ts +++ b/packages/e2e/tests/unit/realm.test.ts @@ -17,7 +17,7 @@ const noop = async () => {}; const FILE = { file: 'tests/realm.e2e.ts', absolutePath: '/project/tests/realm.e2e.ts' }; function manager(runErrors: RunError[] = [], timeouts = { timeout: 1000, cleanupTimeout: 1000 }) { - return new RealmManager({ targetName: 'web', platform: 'web', runErrors, ...timeouts }); + return new RealmManager({ targetName: 'web', platform: 'web', runErrors, redactTitle: (title) => title, ...timeouts }); } /** The pairs still to run, as `leaveFinished` sees them: only the title path matters here. */ diff --git a/packages/e2e/tests/unit/registry.test.ts b/packages/e2e/tests/unit/registry.test.ts index d5d5dbbf8..935fd5054 100644 --- a/packages/e2e/tests/unit/registry.test.ts +++ b/packages/e2e/tests/unit/registry.test.ts @@ -363,6 +363,54 @@ describe('collectFromRegistration', () => { ); }); + it('redacts registered secret values in test and describe titles before ids are derived, and detects a collision after', async () => { + const value = 'sk_title_9f3Kq2'; + const redact = (title: string) => title.replaceAll(value, ''); + const registration = await collectModule(async () => { + test.describe(`suite ${value}`, { serial: true }, () => { + test(`holds ${value}`, noop); + }); + }, undefined, redact); + const collected = collectFromRegistration('/root', '/root/tests/a.e2e.ts', registration); + const [only] = collected.tests; + expect(only!.title).toBe('holds '); + expect(only!.titlePath).toEqual(['suite ', 'holds ']); + expect(only!.group?.title).toBe('suite '); + expect(only!.id).toBe('tests/a.e2e.ts::suite%20%3Csecret%3Aprobe%3E::holds%20%3Csecret%3Aprobe%3E'); + expect(only!.serialId).toBe('serial::tests/a.e2e.ts::suite%20%3Csecret%3Aprobe%3E'); + expect(JSON.stringify(collected.tests.map(({ fn: _fn, group: _group, serialRoot: _root, ...rest }) => rest))).not.toContain(value); + + const colliding = await collectModule(async () => { + test(`token ${value}`, noop); + test('token ', noop); + }, undefined, redact); + expect(() => collectFromRegistration('/root', '/root/tests/b.e2e.ts', colliding)).toThrow( + 'duplicate title path token in tests/b.e2e.ts', + ); + }); + + it('redacts a title before normalizing it, so a value spelled in decomposed form still matches', async () => { + const decomposed = 'café-Kq7Zr2'; + const registration = await collectModule(async () => { + test.describe(`group ${decomposed}`, () => { + test(`holds ${decomposed}`, noop); + }); + }, undefined, (title) => title.replaceAll(decomposed, '')); + expect(registration.tests[0]!.titlePath).toEqual(['group ', 'holds ']); + }); + + it('checks the title limit again once a marker longer than the value replaces it', async () => { + const value = 'Kq7Zr2'; + const title = `${'a'.repeat(512 - value.length)}${value}`; + const redact = (text: string) => text.replaceAll(value, ''); + await expect(collectModule(async () => test(title, noop), undefined, redact)).rejects.toThrow( + 'title must be 1 through 512 UTF-8 bytes after NFC, got 520 once secret values are redacted', + ); + await expect(collectModule(async () => test.describe(title, () => undefined), undefined, redact)).rejects.toThrow( + 'once secret values are redacted', + ); + }); + it('marks serial members with the unit source ID', async () => { const registration = await collectModule(async () => { test.describe('wizard', { serial: true }, () => { diff --git a/packages/e2e/tests/unit/session-secrecy.test.ts b/packages/e2e/tests/unit/session-secrecy.test.ts index c9d2dc48c..3f3feb28c 100644 --- a/packages/e2e/tests/unit/session-secrecy.test.ts +++ b/packages/e2e/tests/unit/session-secrecy.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import type { ResolvedConfig } from '../../src/config/resolve.ts'; import type { TargetSession } from '../../src/engine/surface.ts'; -import { adoptSecrecy, carriedSecrecy, processSecrets, sessionSecrecy } from '../../src/run/secrecy.ts'; +import { adoptSecrecy, carriedSecrecy, processSecrets, redactForSession, sessionSecrecy, staticSecretLedger } from '../../src/run/secrecy.ts'; const STATIC_VALUE = 'static-config-password-5521'; const PROVIDER_VALUE = 'provider-minted-token-8804'; @@ -62,3 +62,21 @@ describe('session exposure', () => { expect(carriedSecrecy(engineHeld, secrets).tainted).toBe(true); }); }); + +describe('redaction before and after a session opens', () => { + it('redacts the static values with no session, and every value the session learned once one is open', () => { + expect(redactForSession(undefined, secrets, `title ${STATIC_VALUE}`)).toBe('title '); + // A provider value is not known before it resolves. + expect(redactForSession(undefined, secrets, `title ${PROVIDER_VALUE}`)).toBe(`title ${PROVIDER_VALUE}`); + const session = newSession(); + sessionSecrecy(session, secrets).ledger.register('token', PROVIDER_VALUE); + expect(redactForSession(session, secrets, `${STATIC_VALUE} ${PROVIDER_VALUE}`)).toBe(' '); + }); + + it('keeps one static ledger per secrets map, untouched by what a session learns', () => { + const ledger = staticSecretLedger(secrets); + expect(staticSecretLedger(secrets)).toBe(ledger); + sessionSecrecy(newSession(), secrets).ledger.register('token', PROVIDER_VALUE); + expect(ledger.entries()).toEqual([['password', STATIC_VALUE]]); + }); +}); diff --git a/packages/e2e/tests/unit/step-progress.test.ts b/packages/e2e/tests/unit/step-progress.test.ts index 8b8666355..7b5b4db31 100644 --- a/packages/e2e/tests/unit/step-progress.test.ts +++ b/packages/e2e/tests/unit/step-progress.test.ts @@ -128,4 +128,18 @@ describe('StepRecorder progress identity', () => { expect(JSON.stringify(heard)).not.toContain('private-value'); expect(heard.at(-1)).toHaveProperty('error', steps.all()[0]!.error); }); + + it('records and publishes a label with a secret the test spelled into it redacted', async () => { + const heard: StepProgress[] = []; + const steps = new StepRecorder('attempt', { + onProgress: (progress) => heard.push(progress), + redact: (text) => text.replaceAll('private-value', '[REDACTED]'), + }); + await steps.run('agent', 'agent.act', 'inspect private-value', async () => undefined); + expect(steps.all()[0]?.label).toBe('inspect [REDACTED]'); + expect(heard).toMatchObject([ + { phase: 'start', label: 'inspect [REDACTED]' }, + { phase: 'end', label: 'inspect [REDACTED]' }, + ]); + }); }); diff --git a/skills/e2e/references/writing-tests.md b/skills/e2e/references/writing-tests.md index 9d6d11567..385107dbd 100644 --- a/skills/e2e/references/writing-tests.md +++ b/skills/e2e/references/writing-tests.md @@ -334,6 +334,11 @@ credentials: { `E2E_SECRET_STRIPE_KEY`; `secrets.get('stripe-key')` is the same kind of handle, fills any editable input, and is redacted by name everywhere the runner writes. +- A registered value passed as a plain string (`process.env.STRIPE_KEY` in a + title, a URL, a locator, an `agent.act` instruction or param) is redacted + too: reports, step labels, the executor, and the model see + `` (and so does the test id, for a value in a title), + so the agent cannot type it. Pass the handle. ## The browser fixture (browser only) From 7fd610b0c22b58ddbb0c08557685e3a677dd3ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Fri, 2 Oct 2026 18:33:09 +0200 Subject: [PATCH 03/50] fix(web): list an inline svg as an image, named by its title, as Playwright does (#770) * fix(web): list an inline svg as an image, named by its title, as Playwright does Fixes #749 * test(web): count only compared crosscheck nodes, fix svg fixture comment --- .changeset/svg-image-role.md | 5 +++ packages/web/src/in-page/read-semantics.ts | 34 +++++++++++++++---- packages/web/tests/crosscheck/crosscheck.ts | 14 ++++++++ .../tests/integration/read-node-roles.test.ts | 28 +++++++++++++-- 4 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 .changeset/svg-image-role.md diff --git a/.changeset/svg-image-role.md b/.changeset/svg-image-role.md new file mode 100644 index 000000000..f54413d21 --- /dev/null +++ b/.changeset/svg-image-role.md @@ -0,0 +1,5 @@ +--- +'@e2e-dev/web': patch +--- + +The web semantic tree lists an inline `` as an `image`, named by its `` child, as Playwright's `getByRole('img')` and aria snapshot read it. An icon-only control built from an unnamed wrapper and an svg no longer disappears from what the agent sees. diff --git a/packages/web/src/in-page/read-semantics.ts b/packages/web/src/in-page/read-semantics.ts index cd4019266..779e1f6a8 100644 --- a/packages/web/src/in-page/read-semantics.ts +++ b/packages/web/src/in-page/read-semantics.ts @@ -167,6 +167,10 @@ export const readSemanticsFunction = ( return 'textbox'; case 'img': return el.getAttribute('alt') === '' ? 'presentation' : 'image'; + // An inline icon is a picture whether or not anything names it, as + // Playwright reads it; Chrome ignores an unnamed one. + case 'svg': + return 'image'; case 'nav': return 'navigation'; case 'main': @@ -395,6 +399,25 @@ export const readSemanticsFunction = ( return alt !== null && alt.trim() !== '' ? alt.trim() : null; }; + /** + * The `` child an SVG element is named by (SVG-AAM), as Playwright + * reads it: `<svg><title>Close` is the image "Close". An svg + * marked `presentation` or `none` takes no name of its own from it, though + * the title still names a link or button around it. + */ + const svgTitleOf = (el: Element): string | null => { + if (!(el instanceof SVGElement)) return null; + const title = Array.from(el.children).find((child) => child instanceof SVGTitleElement); + const text = (title?.textContent ?? '').replace(/\s+/g, ' ').trim(); + return text === '' ? null : text; + }; + + /** True for an element whose role removes its semantics: `presentation` or `none`. */ + const isPresentational = (el: Element): boolean => { + const role = implicitRole(el); + return role === 'presentation' || role === 'none'; + }; + /** * True for a referenced target accname 2A reads whole: hidden itself, or * under an `aria-hidden` ancestor, which excludes it from the tree as @@ -465,8 +488,8 @@ export const readSemanticsFunction = ( if (referenced !== null) return referenced.join(' '); const ariaLabel = el.getAttribute('aria-label'); if (ariaLabel !== null && ariaLabel.trim() !== '') return ariaLabel.trim(); - const alt = altOf(el); - if (alt !== null) return alt; + const alternative = altOf(el) ?? svgTitleOf(el); + if (alternative !== null) return alternative; if (NAME_OPAQUE_TAGS.has(el.tagName)) return ''; const content = childrenNameOf(el, walk); if (content !== '') return content; @@ -615,8 +638,8 @@ export const readSemanticsFunction = ( .trim(); if (joined !== '') return joined; } - const alt = altOf(el); - if (alt !== null) return alt; + const alternative = altOf(el) ?? (isPresentational(el) ? null : svgTitleOf(el)); + if (alternative !== null) return alternative; const captionTag = NAMING_CHILD_TAGS[el.tagName.toLowerCase()]; if (captionTag !== undefined) { const caption = Array.from(el.children).find((child) => child.tagName.toLowerCase() === captionTag); @@ -1187,8 +1210,7 @@ export const readSemanticsFunction = ( /** True when a node carries semantics worth sending to a model. */ const isInteresting = (el: Element): boolean => { if (el.hasAttribute(options.testIdAttribute)) return true; - const role = implicitRole(el); - if (role !== null && role !== 'presentation' && role !== 'none') return true; + if (implicitRole(el) !== null && !isPresentational(el)) return true; if (accessibleName(el) !== null) return true; return directTextOf(el) !== ''; }; diff --git a/packages/web/tests/crosscheck/crosscheck.ts b/packages/web/tests/crosscheck/crosscheck.ts index 9c048b84e..6964bc9af 100644 --- a/packages/web/tests/crosscheck/crosscheck.ts +++ b/packages/web/tests/crosscheck/crosscheck.ts @@ -77,6 +77,16 @@ const CHROME_IGNORES = new Set(['rowgroup']); */ const ENGINE_ONLY_ROLES = new Set(['box']); +/** + * An unnamed inline svg: Chrome's tree ignores it or leaves it out, while + * Playwright reads it as `img` and its aria snapshot lists it. The tree + * lists it as Playwright does, so an icon-only control a person sees reaches + * the model even when nothing names it. + */ +function isUnnamedSvg(node: SemanticNode, facts: ElementFacts): boolean { + return facts.svgRoot && node.role === 'image' && (node.name ?? '') === ''; +} + /** Engine roles ARIA does not define, so `getByRole` cannot be asked for them. */ const NON_ARIA_ROLES = new Set(['box', 'iframe']); @@ -155,6 +165,7 @@ export async function captureTagged(page: Page): Promise<{ inChildFrame: view !== null && view !== view.top, inClosedShadow: root instanceof ShadowRoot && root.mode === 'closed', editingHost: el instanceof HTMLElement && el.isContentEditable, + svgRoot: el instanceof SVGSVGElement, }; }, [MARKER, id])); } @@ -169,6 +180,8 @@ interface ElementFacts { readonly inClosedShadow: boolean; /** An editor's host, which the tree reads as the `textbox` it is. Chrome calls it `generic`. */ readonly editingHost: boolean; + /** An inline ``, which Chrome ignores unless something names it. */ + readonly svgRoot: boolean; } /** The fields of CDP's `Accessibility.AXNode` the check reads. */ @@ -280,6 +293,7 @@ export async function crossCheck(page: Page): Promise { for (const node of nodes) { const fact = facts.get(node.ref.id); if (node.role === undefined || STRUCTURAL_ROLES.has(node.role) || fact === undefined || fact.inChildFrame) continue; + if (isUnnamedSvg(node, fact)) continue; compared += 1; const ax = chrome.byId.get(node.ref.id); if (ax === undefined) disagreements.push({ oracle: 'chrome', field: 'unknown', ours: node.role, theirs: 'no AX node', node: describe(node.role, node.name ?? '') }); diff --git a/packages/web/tests/integration/read-node-roles.test.ts b/packages/web/tests/integration/read-node-roles.test.ts index a3c9fa2a7..f3c71ae8b 100644 --- a/packages/web/tests/integration/read-node-roles.test.ts +++ b/packages/web/tests/integration/read-node-roles.test.ts @@ -75,6 +75,30 @@ describe('role mapping', () => { expect(flatten(tree).some((node) => node.role === 'presentation')).toBe(false); }); + it('reports an inline svg as an image, named by its title child, as Playwright does', async () => { + await page.setContent(` +
+ Close + Ignored + + Decor + Home + `); + const nodes = await rolesByTestId(); + expect(nodes.get('bare')).toMatchObject({ role: 'image' }); + expect(nodes.get('bare')?.name).toBeUndefined(); + expect(nodes.get('titled')).toMatchObject({ role: 'image', name: 'Close' }); + expect(nodes.get('labelled')).toMatchObject({ role: 'image', name: 'Labelled' }); + expect(nodes.has('decorative')).toBe(false); + expect(nodes.get('home')).toMatchObject({ role: 'link', name: 'Home' }); + const { tree } = await capture(); + expect(flatten(tree).filter((node) => node.role === 'presentation' || node.role === 'none')).toEqual([]); + expect(await page.getByRole('img').evaluateAll((els) => els.map((el) => el.getAttribute('data-testid')))) + .toEqual(['bare', 'titled', 'labelled']); + expect(await page.getByRole('img', { name: 'Close', exact: true }).getAttribute('data-testid')).toBe('titled'); + }); + it('passes the composite widget roles through from role attributes and names their items from content', async () => { await page.setContent(` @@ -310,8 +334,8 @@ describe('contenteditable editing hosts', () => { await page.frames()[1]?.waitForLoadState('domcontentloaded'); const { tree } = await capture(); const roles = Object.fromEntries(flatten(tree).filter((node) => node.testId !== undefined).map((node) => [node.testId, node.role])); - // A pixel-only surface stays unlisted: with no role, name, or text it is not a node at all. - expect(roles).toEqual({ frame: 'iframe', host: 'textbox' }); + // A canvas, with no role, name, or text, is not a node at all; an svg has the image role, so it is listed. + expect(roles).toEqual({ svg: 'image', frame: 'iframe', host: 'textbox' }); }); it('drops a name taken from a block placeholder once the editor has content, and keeps one from the host', async () => { From a0ee3e9061b666fa3dd43e7dcc8e1a5da47362d6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:15:15 +0200 Subject: [PATCH 04/50] chore: version packages (#761) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .changeset/atomic-pinned-locate.md | 7 ------ .changeset/codex-client-version.md | 5 ----- .changeset/copilot-unusable-models.md | 5 ----- .changeset/github-long-key-marker.md | 5 ----- .../matcher-options-honored-or-rejected.md | 7 ------ .changeset/mcp-without-ai.md | 5 ----- .changeset/plain-string-secret-redaction.md | 5 ----- .changeset/slim-install.md | 10 --------- .changeset/svg-image-role.md | 5 ----- .changeset/telemetry-mcp-explore-config.md | 5 ----- .changeset/web-matcher-options-rejected.md | 5 ----- packages/e2e/CHANGELOG.md | 22 +++++++++++++++++++ packages/e2e/package.json | 2 +- packages/eas/CHANGELOG.md | 6 +++++ packages/eas/package.json | 2 +- packages/github/CHANGELOG.md | 8 +++++++ packages/github/package.json | 2 +- packages/kernel/CHANGELOG.md | 6 +++++ packages/kernel/package.json | 2 +- packages/mobile/CHANGELOG.md | 6 +++++ packages/mobile/package.json | 2 +- packages/web/CHANGELOG.md | 14 ++++++++++++ packages/web/package.json | 2 +- 23 files changed, 68 insertions(+), 70 deletions(-) delete mode 100644 .changeset/atomic-pinned-locate.md delete mode 100644 .changeset/codex-client-version.md delete mode 100644 .changeset/copilot-unusable-models.md delete mode 100644 .changeset/github-long-key-marker.md delete mode 100644 .changeset/matcher-options-honored-or-rejected.md delete mode 100644 .changeset/mcp-without-ai.md delete mode 100644 .changeset/plain-string-secret-redaction.md delete mode 100644 .changeset/slim-install.md delete mode 100644 .changeset/svg-image-role.md delete mode 100644 .changeset/telemetry-mcp-explore-config.md delete mode 100644 .changeset/web-matcher-options-rejected.md diff --git a/.changeset/atomic-pinned-locate.md b/.changeset/atomic-pinned-locate.md deleted file mode 100644 index 7e7f53864..000000000 --- a/.changeset/atomic-pinned-locate.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@e2e-dev/web": patch ---- - -A locator whose match the page replaces on every animation frame (a field re-rendered each frame) no longer fails with `LOCATOR_NOT_FOUND` "node became stale" on a loaded machine. Locate read each match in a protocol call after the one that found it, so when a frame passed between the two, which a busy runner makes likely every time, the read found the match detached and the locate retried until the action timeout. Each match is now read in the same in-page task that finds it, so the read never sees a replaced element. A `getByLabel` or `getByDisplayValue` match is still pinned to the element that was read. - -A `browser.locator` selector with a `*` capture before its last part (`*css=article >> text=Hello`) now fails with `INVALID_LOCATOR` instead of resolving; use `filter({ has })`. A capture rewrites the chain so the elements it returns are never the ones the rest of the locate reads, and composing anything onto such a selector already matched the captured element instead of its descendants. diff --git a/.changeset/codex-client-version.md b/.changeset/codex-client-version.md deleted file mode 100644 index 928f80757..000000000 --- a/.changeset/codex-client-version.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'e2e': patch ---- - -`e2e models openai` asks the ChatGPT backend for its model list as Codex CLI 0.160.0 (was 0.155.1). The backend hides models from clients it considers too old, so models unlocked since 0.155.1 now show up and can be passed to `chatgpt()`. diff --git a/.changeset/copilot-unusable-models.md b/.changeset/copilot-unusable-models.md deleted file mode 100644 index 8263c6245..000000000 --- a/.changeset/copilot-unusable-models.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'e2e': patch ---- - -`e2e models github-copilot` marks the models `copilot()` cannot call: `not enabled` for a model your plan has not turned on, and `no chat completions` for a model Copilot serves only over other APIs such as Responses. These used to be listed like any other model and then failed the first agent step with "The requested model is not supported". diff --git a/.changeset/github-long-key-marker.md b/.changeset/github-long-key-marker.md deleted file mode 100644 index 4ca6da926..000000000 --- a/.changeset/github-long-key-marker.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@e2e-dev/github": patch ---- - -Two matrix jobs whose `key`, workflow, or job differ only after the first 200 encoded characters now keep their own pull request comments instead of overwriting one. A long field's marker ends in a digest of the whole value; markers of shorter fields are unchanged, so existing comments are still found. A job with a field that long posts one new comment after upgrading and leaves its old one behind. The reporter only takes a comment whose first line is its marker, so a reply quoting the marker is never edited. diff --git a/.changeset/matcher-options-honored-or-rejected.md b/.changeset/matcher-options-honored-or-rejected.md deleted file mode 100644 index 209e481d9..000000000 --- a/.changeset/matcher-options-honored-or-rejected.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'e2e': patch ---- - -Locator matchers honor Playwright's `{ checked: false }`, `{ enabled: false }`, `{ visible: false }`, and `{ attached: false }`, which flip the matcher, and `{ ignoreCase }` on `toHaveText`, `toContainText`, `toHaveAccessibleName`, and `toHaveAttribute(name, value)`. Before, these were ignored at runtime, so `toBeChecked({ checked: false })` passed on a checked box and `not.toContainText('error', { ignoreCase: true })` passed on `Error`. Any other option key, or a flag that is not a boolean, is now `INVALID_ARGUMENT` before the first read, from JavaScript too. - -`e2e/engine` exports `rejectUnknownOptions(api, options, known, code?)`, the check the runner's own fixtures use, so an engine refuses an option its fixture methods and matchers do not take the same way. diff --git a/.changeset/mcp-without-ai.md b/.changeset/mcp-without-ai.md deleted file mode 100644 index 803307ffe..000000000 --- a/.changeset/mcp-without-ai.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'e2e': patch ---- - -`e2e mcp` opens a session in a project without the optional `ai` package. Without the AI SDK, the catalog and the argument checks read each tool's Standard Schema (zod's), so a deterministic-only project (no `agents`, `e2e init` with no model gateway) can `open_session`, `observe`, `locate`, and drive the page. diff --git a/.changeset/plain-string-secret-redaction.md b/.changeset/plain-string-secret-redaction.md deleted file mode 100644 index 69aa70abf..000000000 --- a/.changeset/plain-string-secret-redaction.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"e2e": patch ---- - -A registered secret value passed as a plain string is now redacted in more places: test and describe titles (and the test ids, reporter output, and artifact and failure-page file names derived from them), step labels, `agent.act` and `agent.assert` instructions and params as executors and the model receive them, `agentContext`, judgment prompts, and an `e2e explore` goal. Error messages, observations, and worker console output already masked it. A test whose title spells out a secret gets a new id, so `--last-failed` from an earlier run does not select it once. Not covered: a value only a secret provider returns stays in titles, and a Playwright trace of a session no secret reached is kept as recorded. diff --git a/.changeset/slim-install.md b/.changeset/slim-install.md deleted file mode 100644 index d21572e7d..000000000 --- a/.changeset/slim-install.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -'e2e': patch -'@e2e-dev/web': patch -'@e2e-dev/mobile': patch -'@e2e-dev/github': patch -'@e2e-dev/kernel': patch -'@e2e-dev/eas': patch ---- - -Installing `e2e` pulls in 29 packages instead of 117 and takes about 31MB instead of 36MB. `e2e mcp` now runs on the split MCP SDK (`@modelcontextprotocol/server` 2.2.0) in place of `@modelcontextprotocol/sdk`, which brought in express, hono, and the rest of an HTTP server stack that stdio never used. The server keeps the same protocol version, so existing clients connect as before. Packages are built and published without sourcemaps, which pointed at a `src/` that was never shipped. Stack traces show `dist/` positions. diff --git a/.changeset/svg-image-role.md b/.changeset/svg-image-role.md deleted file mode 100644 index f54413d21..000000000 --- a/.changeset/svg-image-role.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@e2e-dev/web': patch ---- - -The web semantic tree lists an inline `` as an `image`, named by its `` child, as Playwright's `getByRole('img')` and aria snapshot read it. An icon-only control built from an unnamed wrapper and an svg no longer disappears from what the agent sees. diff --git a/.changeset/telemetry-mcp-explore-config.md b/.changeset/telemetry-mcp-explore-config.md deleted file mode 100644 index 1775d22dd..000000000 --- a/.changeset/telemetry-mcp-explore-config.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'e2e': minor ---- - -Anonymous telemetry covers `e2e mcp` and `e2e explore`. Each MCP session sends one `e2e_mcp_session` event when it closes or fails to open: the client's self-reported name and version, the platform and engine, how it ended, and its tool calls counted by the runner's own tool names with the error codes they failed with. `e2e_run_completed` adds `command`, counts and option ids of the config features the run used, and for `e2e explore` why it stopped and its steps and findings by kind and severity. Beyond the client's own name, no project names, paths, tool arguments or results, goals, or findings are sent. The notice shows again once; `E2E_TELEMETRY_DEBUG=1` prints every event, and the opt-outs are unchanged. diff --git a/.changeset/web-matcher-options-rejected.md b/.changeset/web-matcher-options-rejected.md deleted file mode 100644 index 69c70e823..000000000 --- a/.changeset/web-matcher-options-rejected.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@e2e-dev/web': patch ---- - -`expect(browser).toHaveURL`, `toHaveTitle`, `toHaveClass`, and `browser.waitForURL` refuse an option other than `timeout` with `INVALID_ARGUMENT` instead of ignoring it, so Playwright's `ignoreCase` on `toHaveURL` or `waitUntil` on `waitForURL` no longer runs as if absent. diff --git a/packages/e2e/CHANGELOG.md b/packages/e2e/CHANGELOG.md index 1419c30f0..c9b2ccb34 100644 --- a/packages/e2e/CHANGELOG.md +++ b/packages/e2e/CHANGELOG.md @@ -1,5 +1,27 @@ # e2e +## 0.16.0 + +### Minor Changes + +- [#762](https://github.com/tester-army/e2e/pull/762) [`83c17e2`](https://github.com/tester-army/e2e/commit/83c17e261e1e6de043c6ae96b9251fecf05356d6) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Anonymous telemetry covers `e2e mcp` and `e2e explore`. Each MCP session sends one `e2e_mcp_session` event when it closes or fails to open: the client's self-reported name and version, the platform and engine, how it ended, and its tool calls counted by the runner's own tool names with the error codes they failed with. `e2e_run_completed` adds `command`, counts and option ids of the config features the run used, and for `e2e explore` why it stopped and its steps and findings by kind and severity. Beyond the client's own name, no project names, paths, tool arguments or results, goals, or findings are sent. The notice shows again once; `E2E_TELEMETRY_DEBUG=1` prints every event, and the opt-outs are unchanged. + +### Patch Changes + +- [#760](https://github.com/tester-army/e2e/pull/760) [`b3c9832`](https://github.com/tester-army/e2e/commit/b3c9832aa78ac073ceeb4f11e21c833b8ed1e6f7) Thanks [@devin-ai-integration](https://github.com/apps/devin-ai-integration)! - `e2e models openai` asks the ChatGPT backend for its model list as Codex CLI 0.160.0 (was 0.155.1). The backend hides models from clients it considers too old, so models unlocked since 0.155.1 now show up and can be passed to `chatgpt()`. + +- [#767](https://github.com/tester-army/e2e/pull/767) [`5609f5e`](https://github.com/tester-army/e2e/commit/5609f5e5f0e1a72d632c9cce185135f337d1b86d) Thanks [@okwasniewski](https://github.com/okwasniewski)! - `e2e models github-copilot` marks the models `copilot()` cannot call: `not enabled` for a model your plan has not turned on, and `no chat completions` for a model Copilot serves only over other APIs such as Responses. These used to be listed like any other model and then failed the first agent step with "The requested model is not supported". + +- [#579](https://github.com/tester-army/e2e/pull/579) [`c6c39ad`](https://github.com/tester-army/e2e/commit/c6c39ad3f631cc0a29d788af7d8e6b798ef5307c) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Locator matchers honor Playwright's `{ checked: false }`, `{ enabled: false }`, `{ visible: false }`, and `{ attached: false }`, which flip the matcher, and `{ ignoreCase }` on `toHaveText`, `toContainText`, `toHaveAccessibleName`, and `toHaveAttribute(name, value)`. Before, these were ignored at runtime, so `toBeChecked({ checked: false })` passed on a checked box and `not.toContainText('error', { ignoreCase: true })` passed on `Error`. Any other option key, or a flag that is not a boolean, is now `INVALID_ARGUMENT` before the first read, from JavaScript too. + + `e2e/engine` exports `rejectUnknownOptions(api, options, known, code?)`, the check the runner's own fixtures use, so an engine refuses an option its fixture methods and matchers do not take the same way. + +- [#759](https://github.com/tester-army/e2e/pull/759) [`0cda95d`](https://github.com/tester-army/e2e/commit/0cda95daa72aa88874e2348942913f075f5df5a1) Thanks [@devin-ai-integration](https://github.com/apps/devin-ai-integration)! - `e2e mcp` opens a session in a project without the optional `ai` package. Without the AI SDK, the catalog and the argument checks read each tool's Standard Schema (zod's), so a deterministic-only project (no `agents`, `e2e init` with no model gateway) can `open_session`, `observe`, `locate`, and drive the page. + +- [#727](https://github.com/tester-army/e2e/pull/727) [`dec903b`](https://github.com/tester-army/e2e/commit/dec903bd4c20a58cd01a4c09e7ab55ecae6a1cce) Thanks [@okwasniewski](https://github.com/okwasniewski)! - A registered secret value passed as a plain string is now redacted in more places: test and describe titles (and the test ids, reporter output, and artifact and failure-page file names derived from them), step labels, `agent.act` and `agent.assert` instructions and params as executors and the model receive them, `agentContext`, judgment prompts, and an `e2e explore` goal. Error messages, observations, and worker console output already masked it. A test whose title spells out a secret gets a new id, so `--last-failed` from an earlier run does not select it once. Not covered: a value only a secret provider returns stays in titles, and a Playwright trace of a session no secret reached is kept as recorded. + +- [#763](https://github.com/tester-army/e2e/pull/763) [`b573756`](https://github.com/tester-army/e2e/commit/b573756818d7d04088142d29e0e730cfbaf21b45) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Installing `e2e` pulls in 29 packages instead of 117 and takes about 31MB instead of 36MB. `e2e mcp` now runs on the split MCP SDK (`@modelcontextprotocol/server` 2.2.0) in place of `@modelcontextprotocol/sdk`, which brought in express, hono, and the rest of an HTTP server stack that stdio never used. The server keeps the same protocol version, so existing clients connect as before. Packages are built and published without sourcemaps, which pointed at a `src/` that was never shipped. Stack traces show `dist/` positions. + ## 0.15.2 ### Patch Changes diff --git a/packages/e2e/package.json b/packages/e2e/package.json index c78ded411..9f5021597 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -1,6 +1,6 @@ { "name": "e2e", - "version": "0.15.2", + "version": "0.16.0", "description": "SDK, runner, and CLI for agentic end-to-end testing", "keywords": [ "e2e", diff --git a/packages/eas/CHANGELOG.md b/packages/eas/CHANGELOG.md index a53788dc1..6e7f27e2c 100644 --- a/packages/eas/CHANGELOG.md +++ b/packages/eas/CHANGELOG.md @@ -1,5 +1,11 @@ # @e2e-dev/eas +## 0.2.1 + +### Patch Changes + +- [#763](https://github.com/tester-army/e2e/pull/763) [`b573756`](https://github.com/tester-army/e2e/commit/b573756818d7d04088142d29e0e730cfbaf21b45) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Installing `e2e` pulls in 29 packages instead of 117 and takes about 31MB instead of 36MB. `e2e mcp` now runs on the split MCP SDK (`@modelcontextprotocol/server` 2.2.0) in place of `@modelcontextprotocol/sdk`, which brought in express, hono, and the rest of an HTTP server stack that stdio never used. The server keeps the same protocol version, so existing clients connect as before. Packages are built and published without sourcemaps, which pointed at a `src/` that was never shipped. Stack traces show `dist/` positions. + ## 0.2.0 ### Minor Changes diff --git a/packages/eas/package.json b/packages/eas/package.json index 3c2fd7666..a846aba40 100644 --- a/packages/eas/package.json +++ b/packages/eas/package.json @@ -1,6 +1,6 @@ { "name": "@e2e-dev/eas", - "version": "0.2.0", + "version": "0.2.1", "description": "EAS Simulators for e2e: run mobile targets on Expo's hosted iOS simulators and Android emulators", "keywords": [ "e2e", diff --git a/packages/github/CHANGELOG.md b/packages/github/CHANGELOG.md index 4066c3d12..aa6e27d47 100644 --- a/packages/github/CHANGELOG.md +++ b/packages/github/CHANGELOG.md @@ -1,5 +1,13 @@ # @e2e-dev/github +## 0.3.2 + +### Patch Changes + +- [#720](https://github.com/tester-army/e2e/pull/720) [`62bfe0a`](https://github.com/tester-army/e2e/commit/62bfe0a896a50f501f071d31c8935e0e0bdabfcf) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Two matrix jobs whose `key`, workflow, or job differ only after the first 200 encoded characters now keep their own pull request comments instead of overwriting one. A long field's marker ends in a digest of the whole value; markers of shorter fields are unchanged, so existing comments are still found. A job with a field that long posts one new comment after upgrading and leaves its old one behind. The reporter only takes a comment whose first line is its marker, so a reply quoting the marker is never edited. + +- [#763](https://github.com/tester-army/e2e/pull/763) [`b573756`](https://github.com/tester-army/e2e/commit/b573756818d7d04088142d29e0e730cfbaf21b45) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Installing `e2e` pulls in 29 packages instead of 117 and takes about 31MB instead of 36MB. `e2e mcp` now runs on the split MCP SDK (`@modelcontextprotocol/server` 2.2.0) in place of `@modelcontextprotocol/sdk`, which brought in express, hono, and the rest of an HTTP server stack that stdio never used. The server keeps the same protocol version, so existing clients connect as before. Packages are built and published without sourcemaps, which pointed at a `src/` that was never shipped. Stack traces show `dist/` positions. + ## 0.3.1 ### Patch Changes diff --git a/packages/github/package.json b/packages/github/package.json index ed747a02d..b4d68f0a2 100644 --- a/packages/github/package.json +++ b/packages/github/package.json @@ -1,6 +1,6 @@ { "name": "@e2e-dev/github", - "version": "0.3.1", + "version": "0.3.2", "description": "GitHub reporter for e2e: one pull request comment per run, from GitHub Actions", "keywords": [ "e2e", diff --git a/packages/kernel/CHANGELOG.md b/packages/kernel/CHANGELOG.md index 2b749981b..d2e174566 100644 --- a/packages/kernel/CHANGELOG.md +++ b/packages/kernel/CHANGELOG.md @@ -1,5 +1,11 @@ # @e2e-dev/integrations +## 0.1.2 + +### Patch Changes + +- [#763](https://github.com/tester-army/e2e/pull/763) [`b573756`](https://github.com/tester-army/e2e/commit/b573756818d7d04088142d29e0e730cfbaf21b45) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Installing `e2e` pulls in 29 packages instead of 117 and takes about 31MB instead of 36MB. `e2e mcp` now runs on the split MCP SDK (`@modelcontextprotocol/server` 2.2.0) in place of `@modelcontextprotocol/sdk`, which brought in express, hono, and the rest of an HTTP server stack that stdio never used. The server keeps the same protocol version, so existing clients connect as before. Packages are built and published without sourcemaps, which pointed at a `src/` that was never shipped. Stack traces show `dist/` positions. + ## 0.1.1 ### Patch Changes diff --git a/packages/kernel/package.json b/packages/kernel/package.json index 9d6d86a75..455d477e8 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -1,6 +1,6 @@ { "name": "@e2e-dev/kernel", - "version": "0.1.1", + "version": "0.1.2", "description": "Kernel hosted browsers for e2e: run web targets in Kernel's hosted Chromium", "keywords": [ "e2e", diff --git a/packages/mobile/CHANGELOG.md b/packages/mobile/CHANGELOG.md index c44ae8ef4..545a6439a 100644 --- a/packages/mobile/CHANGELOG.md +++ b/packages/mobile/CHANGELOG.md @@ -1,5 +1,11 @@ # @e2e-dev/mobile +## 0.9.1 + +### Patch Changes + +- [#763](https://github.com/tester-army/e2e/pull/763) [`b573756`](https://github.com/tester-army/e2e/commit/b573756818d7d04088142d29e0e730cfbaf21b45) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Installing `e2e` pulls in 29 packages instead of 117 and takes about 31MB instead of 36MB. `e2e mcp` now runs on the split MCP SDK (`@modelcontextprotocol/server` 2.2.0) in place of `@modelcontextprotocol/sdk`, which brought in express, hono, and the rest of an HTTP server stack that stdio never used. The server keeps the same protocol version, so existing clients connect as before. Packages are built and published without sourcemaps, which pointed at a `src/` that was never shipped. Stack traces show `dist/` positions. + ## 0.9.0 ### Minor Changes diff --git a/packages/mobile/package.json b/packages/mobile/package.json index 8f9a5fc28..8f57a8d74 100644 --- a/packages/mobile/package.json +++ b/packages/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@e2e-dev/mobile", - "version": "0.9.0", + "version": "0.9.1", "description": "Mobile engine (iOS and Android) for e2e agentic end-to-end testing, built on agent-device", "keywords": [ "e2e", diff --git a/packages/web/CHANGELOG.md b/packages/web/CHANGELOG.md index 0a8cd4457..d727f9b78 100644 --- a/packages/web/CHANGELOG.md +++ b/packages/web/CHANGELOG.md @@ -1,5 +1,19 @@ # @e2e-dev/web +## 0.11.2 + +### Patch Changes + +- [#748](https://github.com/tester-army/e2e/pull/748) [`8415579`](https://github.com/tester-army/e2e/commit/84155791f7bc970ac3eb74d1d6f59e9c54a6e5c4) Thanks [@okwasniewski](https://github.com/okwasniewski)! - A locator whose match the page replaces on every animation frame (a field re-rendered each frame) no longer fails with `LOCATOR_NOT_FOUND` "node became stale" on a loaded machine. Locate read each match in a protocol call after the one that found it, so when a frame passed between the two, which a busy runner makes likely every time, the read found the match detached and the locate retried until the action timeout. Each match is now read in the same in-page task that finds it, so the read never sees a replaced element. A `getByLabel` or `getByDisplayValue` match is still pinned to the element that was read. + + A `browser.locator` selector with a `*` capture before its last part (`*css=article >> text=Hello`) now fails with `INVALID_LOCATOR` instead of resolving; use `filter({ has })`. A capture rewrites the chain so the elements it returns are never the ones the rest of the locate reads, and composing anything onto such a selector already matched the captured element instead of its descendants. + +- [#763](https://github.com/tester-army/e2e/pull/763) [`b573756`](https://github.com/tester-army/e2e/commit/b573756818d7d04088142d29e0e730cfbaf21b45) Thanks [@okwasniewski](https://github.com/okwasniewski)! - Installing `e2e` pulls in 29 packages instead of 117 and takes about 31MB instead of 36MB. `e2e mcp` now runs on the split MCP SDK (`@modelcontextprotocol/server` 2.2.0) in place of `@modelcontextprotocol/sdk`, which brought in express, hono, and the rest of an HTTP server stack that stdio never used. The server keeps the same protocol version, so existing clients connect as before. Packages are built and published without sourcemaps, which pointed at a `src/` that was never shipped. Stack traces show `dist/` positions. + +- [#770](https://github.com/tester-army/e2e/pull/770) [`7fd610b`](https://github.com/tester-army/e2e/commit/7fd610b0c22b58ddbb0c08557685e3a677dd3ba7) Thanks [@okwasniewski](https://github.com/okwasniewski)! - The web semantic tree lists an inline `` as an `image`, named by its `` child, as Playwright's `getByRole('img')` and aria snapshot read it. An icon-only control built from an unnamed wrapper and an svg no longer disappears from what the agent sees. + +- [#579](https://github.com/tester-army/e2e/pull/579) [`c6c39ad`](https://github.com/tester-army/e2e/commit/c6c39ad3f631cc0a29d788af7d8e6b798ef5307c) Thanks [@okwasniewski](https://github.com/okwasniewski)! - `expect(browser).toHaveURL`, `toHaveTitle`, `toHaveClass`, and `browser.waitForURL` refuse an option other than `timeout` with `INVALID_ARGUMENT` instead of ignoring it, so Playwright's `ignoreCase` on `toHaveURL` or `waitUntil` on `waitForURL` no longer runs as if absent. + ## 0.11.1 ### Patch Changes diff --git a/packages/web/package.json b/packages/web/package.json index c8dd94b42..a4c7a3073 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,6 +1,6 @@ { "name": "@e2e-dev/web", - "version": "0.11.1", + "version": "0.11.2", "description": "Web engine for e2e agentic end-to-end testing, built on Playwright", "keywords": [ "e2e", From 9d61e2130f4d1ad286a9a82b1328951f28935547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Fri, 2 Oct 2026 19:43:00 +0200 Subject: [PATCH 05/50] docs(mobile): document physical devices (#768) * docs(mobile): document physical devices Closes #766 * docs(mobile): phone requirements in setup and README * docs(mobile): move physical devices to its own page --- docs/docs.json | 3 +- docs/mobile.mdx | 10 ++++- docs/physical-devices.mdx | 76 ++++++++++++++++++++++++++++++++++ docs/reference/mobile.mdx | 8 ++-- packages/mobile/README.md | 9 ++-- packages/mobile/src/options.ts | 5 ++- skills/e2e/SKILL.md | 4 +- skills/e2e/references/setup.md | 14 ++++--- 8 files changed, 110 insertions(+), 19 deletions(-) create mode 100644 docs/physical-devices.mdx diff --git a/docs/docs.json b/docs/docs.json index fbb675fa0..a41744320 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -150,7 +150,8 @@ "group": "Mobile", "icon": "mobile-screen", "pages": [ - "mobile" + "mobile", + "physical-devices" ] }, { diff --git a/docs/mobile.mdx b/docs/mobile.mdx index deff670a1..a867013a8 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -3,8 +3,8 @@ title: Testing iOS and Android description: Set up a simulator or emulator target, write a device test, and know what differs from the web. --- -`@e2e-dev/mobile` runs tests on iOS simulators and Android emulators. It is -built on [agent-device](https://github.com/callstack/agent-device). +`@e2e-dev/mobile` runs tests on iOS simulators, Android emulators, and +phones connected to your machine. It is built on [agent-device](https://github.com/callstack/agent-device). Tests use the same `agent`, `screen`, and `expect` APIs as browser tests. A test launches the configured app with `app.open()`; between tests the app stays where the last one left it. @@ -178,6 +178,12 @@ mobile({ platform: 'ios', device: ['iPhone 17', 'iPhone 17 Pro'] }) Without `device`, the engine uses booted devices up to the worker limit. If none are booted, it boots one. Two workers never drive the same device. +## Physical devices + +A phone connected to your machine works as a device too: name it in +`device`. Setup and the controls that need a simulator are on +[Physical devices](/physical-devices). + ## Hosted devices A `device` can also be a `DeviceProvider`: an object that leases one hosted diff --git a/docs/physical-devices.mdx b/docs/physical-devices.mdx new file mode 100644 index 000000000..c4d560437 --- /dev/null +++ b/docs/physical-devices.mdx @@ -0,0 +1,76 @@ +--- +title: Physical devices +description: Run mobile tests on an iPhone or Android phone connected to your machine. +--- + +agent-device drives a phone plugged into your machine the same way it +drives a simulator, so a physical device needs no other engine or option. +List what agent-device sees, then name the phone in `device` as it is +listed: + + +```bash npm +npx agent-device devices --platform ios +``` + +```bash pnpm +pnpm exec agent-device devices --platform ios +``` + +```bash bun +bunx agent-device devices --platform ios +``` + + +```ts +mobile({ platform: 'ios', device: 'QA iPhone' }) +``` + +Name a phone by its name, not its id. The engine reads a `device` string as +an id only when it is a simulator UDID or an `emulator-` serial, so a +phone's UDID or adb serial is looked up as a name and not found. The first +device with a matching name wins, so give each phone a unique name: two +phones both called `iPhone` or `Pixel 8` would bind one phone to two +workers. Without `device`, a connected phone counts as booted and joins the +pool beside any booted simulators, so name it to be sure a run lands on it. + +Set the phone up once: + +- **iOS**: the phone is paired and trusted, unlocked, with Developer Mode + on. agent-device builds and signs its XCTest runner onto the phone, so set + `AGENT_DEVICE_IOS_TEAM_ID` to your team id and `AGENT_DEVICE_IOS_BUNDLE_ID` + to a unique reverse-DNS id (`com.example.agentdevice.runner`); the + signing profile must allow that id and `.uitests`. Set them before the + first agent-device command, or run `npx agent-device daemon stop` after: + a running daemon keeps the environment it started with. The first run + builds the runner and takes longer. `app.appPath` must be a signed device + build: a simulator `.app` does not install on a phone. +- **Android**: USB debugging is on and the phone is authorized, listed as + `device` in `adb devices`. `app.appPath` is the same `.apk`. + +`npx agent-device doctor` checks the setup, and +`npx agent-device help physical-device` covers iOS signing in detail. + +Some device controls need a simulator or emulator, and fail on a phone with +agent-device's message: + +- On an iPhone, every device setting is simulator-only: `app.permissions` + and `device.setPermission`, `app.clearState()`, `device.setNetwork`, + `device.setAirplaneMode`, `device.setLocation`, `device.clearLocation`, + `device.setAppearance`, the biometric methods, and `device.clearKeychain`. + So are `device.clipboard`, `device.setClipboard`, and `device.fold`. For a + fresh start, reinstall with `device.installApp(path, { reinstall: true })`. +- An older iPhone that `xcrun devicectl list devices` does not list is + driven without CoreDevice: taps, typing, and the screen work, but + installs, `--video`, `device.openLink`, and `app.launchArguments` fail. + Install the build from Xcode and pin `app.bundleId`. +- On an Android phone, `device.setLocation` is emulator-only, and + `device.setBiometrics('fingerprint', ...)` works only where the phone + exposes `cmd fingerprint`. +- `--video` on an iPhone is built from screenshots at about 15 frames a + second, and drops frames. +- A phone is not this machine, so `localhost` in the app is the phone + itself. Point the app at your machine's network address, or on Android + forward the port with `adb reverse tcp: tcp:`. + +To run on phones in a device farm, see [Hosted devices](/mobile#hosted-devices). diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 7b72bf8fa..60efb5fbf 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -3,7 +3,8 @@ title: Mobile description: Mobile engine options and the device fixture for iOS and Android. --- -`@e2e-dev/mobile` drives iOS simulators and Android emulators through +`@e2e-dev/mobile` drives iOS simulators, Android emulators, and connected +phones through [agent-device](https://github.com/callstack/agent-device). `agent`, `app`, `screen`, and `expect` work as they do everywhere; this page covers what is specific to a device target. Import `test` from the package to have `device` @@ -79,8 +80,9 @@ export interface MobileOptions { /** Platform the target's device runs. */ readonly platform: MobilePlatform; /** - * Simulator or emulator to use, by name, simulator UDID, or emulator - * serial (`emulator-5554`). A list is a pool: the + * Device to use: a simulator or emulator by name, simulator UDID, or + * emulator serial (`emulator-5554`), or a connected phone by name. A list + * is a pool: the * engine declares one worker per entry and worker slot `n` drives the * `n`th, so `workers` at or above the pool size runs the target's files * across every device at once. Omitted, the pool is every booted device of diff --git a/packages/mobile/README.md b/packages/mobile/README.md index 3ea55e882..4b962a10d 100644 --- a/packages/mobile/README.md +++ b/packages/mobile/README.md @@ -1,8 +1,8 @@ # @e2e-dev/mobile The mobile engine for [`e2e`](https://www.npmjs.com/package/e2e), built on -[agent-device](https://github.com/callstack/agent-device): iOS simulators and -Android emulators through the same `e2e/engine` contract the browser engine +[agent-device](https://github.com/callstack/agent-device): iOS simulators, +Android emulators, and connected phones through the same `e2e/engine` contract the browser engine implements. A test written against `screen`, `expect`, `app`, and `agent` runs on a device target unchanged; nothing in `e2e` core knows this package exists. @@ -18,7 +18,8 @@ npm install --save-dev e2e @e2e-dev/mobile ``` `agent-device` needs Xcode with an iOS simulator runtime, or the Android SDK -with an emulator. Run `npx agent-device doctor` once before handing the target +with an emulator. A phone needs Developer Mode and runner signing on iOS, or +USB debugging authorized on Android (see [Physical devices](https://e2e.tester.army/docs/physical-devices)). Run `npx agent-device doctor` once before handing the target to the runner. ```ts title="e2e.config.ts" @@ -68,7 +69,7 @@ the other old app options are `INVALID_CONFIG` naming their key under `app`. | Option | Meaning | | --- | --- | | `platform` | `'ios'` or `'android'`. | -| `device` | Simulator or emulator name or UDID. A list is a pool: one worker per entry, worker slot `n` driving the `n`th. Omitted, every booted device of the platform is the pool, as many as the run has slots. | +| `device` | Simulator or emulator name or UDID, or a connected phone's name. A list is a pool: one worker per entry, worker slot `n` driving the `n`th. Omitted, every booted device of the platform is the pool, as many as the run has slots. | | `session` | agent-device session name, before the worker slot: slot `n` drives its device under `-`, `e2e--` by default. One run per session at a time. | | `snapshot` | `'full'` (default, includes static text) or `'interactive'` (actionable nodes only). | | `settle` | For agent actions: milliseconds the UI must hold still after an action before the agent observes again, default `150`; `false` skips the wait. A test's own steps never settle; `expect` verifies their outcome. | diff --git a/packages/mobile/src/options.ts b/packages/mobile/src/options.ts index 344dc40f8..0a39eef4c 100644 --- a/packages/mobile/src/options.ts +++ b/packages/mobile/src/options.ts @@ -54,8 +54,9 @@ export interface MobileOptions { /** Platform the target's device runs. */ readonly platform: MobilePlatform; /** - * Simulator or emulator to use, by name, simulator UDID, or emulator - * serial (`emulator-5554`). A list is a pool: the + * Device to use: a simulator or emulator by name, simulator UDID, or + * emulator serial (`emulator-5554`), or a connected phone by name. A list + * is a pool: the * engine declares one worker per entry and worker slot `n` drives the * `n`th, so `workers` at or above the pool size runs the target's files * across every device at once. Omitted, the pool is every booted device of diff --git a/skills/e2e/SKILL.md b/skills/e2e/SKILL.md index 10aea08b6..0067ff17e 100644 --- a/skills/e2e/SKILL.md +++ b/skills/e2e/SKILL.md @@ -10,8 +10,8 @@ one goal; `agent.assert`, `agent.waitFor`, and `agent.extract` judge the screen. `screen`, `app`, `browser`, and `expect` make exact interactions and checks. The replay cache reruns verified actions and checks their recorded end state without a model call; agent judgments still run live. UI targets use -`@e2e-dev/web` for browsers or `@e2e-dev/mobile` for iOS simulators and -Android emulators. A test that takes only `app` can check an API with `fetch` +`@e2e-dev/web` for browsers or `@e2e-dev/mobile` for iOS simulators, +Android emulators, and connected phones. A test that takes only `app` can check an API with `fetch` and `expect` (topic `writing-tests`). Model sign-in commands are in [setup](references/setup.md#subscriptions-and-api-keys). diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index e63cad741..1eedcbdbe 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -252,10 +252,14 @@ For an app started elsewhere, point `app.url` at it, literally or via ## Mobile targets -`@e2e-dev/mobile` drives iOS simulators and Android emulators through -[agent-device](https://github.com/callstack/agent-device); needs Xcode with a -simulator runtime or the Android SDK with an emulator; run -`npx agent-device doctor` once. +`@e2e-dev/mobile` drives iOS simulators, Android emulators, and connected +phones through [agent-device](https://github.com/callstack/agent-device); +needs Xcode with a simulator runtime or the Android SDK with an emulator (a +phone needs Xcode with Developer Mode and runner signing, or adb with USB +debugging authorized); run `npx agent-device doctor` once. Name a phone in `device` by the name +`npx agent-device devices` lists, not its UDID or serial. On an iPhone, device +settings (permissions, `clearState`, network, location, appearance, +biometrics, keychain) and the clipboard are simulator-only. ```ts import type { E2EConfig } from 'e2e'; @@ -292,7 +296,7 @@ export default { `app.command` (`npx expo start --port 8081`) with `readyUrl: 'http://localhost:8081/status'` starts it. - One worker per device. No `device`: every booted simulator or emulator of - the platform is the pool, up to `workers` (none booted: agent-device boots + the platform, and every connected phone, is the pool, up to `workers` (none booted: agent-device boots one); one `device`: one worker whatever `workers` says; a list (`device: ['iPhone 17', 'iPhone 17 Pro']`): an explicit pool. Devices boot in `prepare`, before the run's clock. Two sessions on one device fight over From f7c075666672d128f79ef2ec347fda67ea9e42ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Fri, 2 Oct 2026 20:00:51 +0200 Subject: [PATCH 06/50] fix(expect): waitFor attached/detached, absent-frame absence, toHaveProperty primitives, toHaveURL ignoreCase (#726) Rebuilt on top of #579, which already landed matcher option flags, ignoreCase, and unknown-option refusal. --- .changeset/matcher-option-strictness.md | 6 + docs/migrate/playwright.mdx | 7 +- docs/reference/engine.mdx | 2 +- docs/reference/expect.mdx | 25 ++-- docs/reference/screen.mdx | 14 ++- packages/e2e/src/expect/async.ts | 9 +- packages/e2e/src/expect/values.ts | 15 ++- packages/e2e/src/internal/urls.ts | 18 ++- packages/e2e/src/locator/engine.ts | 10 +- packages/e2e/src/locator/screen.ts | 42 ++++++- packages/e2e/src/types.ts | 8 +- .../tests/integration/matcher-options.test.ts | 6 +- .../integration/scripted-polling.test.ts | 14 +++ .../integration/web-class-assertions.test.ts | 25 +++- .../tests/integration/web-platform.test.ts | 13 ++ packages/e2e/tests/types/sdk-types.ts | 4 + packages/e2e/tests/unit/expect-values.test.ts | 19 +++ .../e2e/tests/unit/locator-wait-for.test.ts | 116 ++++++++++++++++++ packages/e2e/tests/unit/urls.test.ts | 9 ++ packages/web/src/browser.ts | 18 +-- skills/e2e/references/writing-tests.md | 6 +- 21 files changed, 335 insertions(+), 51 deletions(-) create mode 100644 .changeset/matcher-option-strictness.md create mode 100644 packages/e2e/tests/unit/locator-wait-for.test.ts diff --git a/.changeset/matcher-option-strictness.md b/.changeset/matcher-option-strictness.md new file mode 100644 index 000000000..03e0812f0 --- /dev/null +++ b/.changeset/matcher-option-strictness.md @@ -0,0 +1,6 @@ +--- +"e2e": patch +"@e2e-dev/web": patch +--- + +`locator.waitFor` adds Playwright's `attached` and `detached` states and refuses any other state, or an option key besides `state` and `timeout`, with `INVALID_ARGUMENT`. Before, every state but `visible` ran as `hidden`, so `waitFor({ state: 'attached' })` and a misspelled state passed at once on an absent node. `toBeHidden`, `toBeVisible({ visible: false })`, `toBeAttached({ attached: false })`, and `waitFor` `detached` and `hidden` read a locator under a frame missing from the document as zero matches instead of timing out waiting for the frame. `toHaveProperty` reads a primitive on the path through its wrapper, as Jest does, so `toHaveProperty('label.length', 3)` passes. `expect(browser).toHaveURL` takes `ignoreCase`, and `urlMatches` in `e2e/engine` takes it as an optional fourth argument. diff --git a/docs/migrate/playwright.mdx b/docs/migrate/playwright.mdx index 1c2beeebe..1797a3b19 100644 --- a/docs/migrate/playwright.mdx +++ b/docs/migrate/playwright.mdx @@ -149,8 +149,7 @@ what to do today. | `isChecked()` | the same on a checkbox, radio, or `aria-checked` node; `false` on any other node, where Playwright throws | by design | | `isHidden()`, `isDisabled()` | the same | same | | `isEditable()` | not yet | missing | -| `waitFor({ state: 'visible' })`, `'hidden'` | the same | same | -| `waitFor({ state: 'attached' })`, `'detached'` | not yet; `expect(locator).toBeAttached()` and `.not.toBeAttached()` | missing | +| `waitFor({ state: 'visible' })`, `'hidden'`, `'attached'`, `'detached'` | the same | same | | `locator.evaluate(fn)`, `evaluateAll(fn)` | not yet; `browser.evaluate(fn, selector)` with a CSS selector argument | missing | | `locator.screenshot()` | not yet; `app.screenshot(label)` captures the viewport | missing | | `innerHTML()`, `dispatchEvent()`, `selectText()`, `blur()`, `highlight()` | not yet; `browser.evaluate` | missing | @@ -173,14 +172,14 @@ what to do today. | `toHaveText(['a', 'b'])`, `toContainText([...])` on several matches | the same: `toHaveText` needs the count to match, `toContainText` an ordered subsequence | same | | `toBeChecked({ checked: false })`, `toBeEnabled({ enabled })`, `toBeVisible({ visible })`, `toBeAttached({ attached })` | the same | same | | `{ ignoreCase }` on `toHaveText()`, `toContainText()`, `toHaveAccessibleName()`, `toHaveAttribute(name, value)` | the same | same | -| `toBeChecked({ indeterminate })`, `{ useInnerText }`, `toHaveURL(url, { ignoreCase })` | not yet; `INVALID_ARGUMENT`, so the matcher never passes on its default | missing | +| `toBeChecked({ indeterminate })`, `{ useInnerText }` | not yet; `INVALID_ARGUMENT`, so the matcher never passes on its default | missing | | `toHaveAttribute('aria-selected', 'true')`, `'aria-expanded'` | `toBeSelected()`, `toBeExpanded()` | renamed | | a positive matcher on several matches | `LOCATOR_AMBIGUOUS` at once; narrow with `filter`, `first`, or `{ visible: true }` | by design | | `toBeAttached()` | the same; passes on a hidden match too | same | | `toBeEmpty()`, `toBeInViewport()`, `toBeEditable()`, `toHaveRole()`, `toHaveAccessibleDescription()` | not yet | missing | | `expect(locator).toHaveClass(cls)` | `expect(browser).toHaveClass(locator, cls)`; a DOM fact is asserted through `browser`; reads the `class` attribute, so `''` matches `class=""` and not a missing attribute | web only | | `toHaveCSS()`, `toHaveId()`, `toHaveJSProperty()`, `toContainClass()`, `toHaveValues()` | not yet; `browser.evaluate` plus a value matcher | missing | -| `expect(page).toHaveURL()`, `toHaveTitle()` | `expect(browser).toHaveURL()`, `toHaveTitle()` | web only | +| `expect(page).toHaveURL()`, `toHaveTitle()` | `expect(browser).toHaveURL()`, `toHaveTitle()`; `toHaveURL` takes `ignoreCase` | web only | | `toHaveScreenshot()`, `toMatchSnapshot()` | not yet; `agent.assert('...', { vision: true })` is a judgment, not a pixel diff | missing | | `toMatchAriaSnapshot()` | not yet | missing | | `expect(value).toBe()`, `toEqual()`, `toMatch()`, `toBeTruthy()`, `toBeCloseTo()` and the rest | the same, except `toContain`, below, see [value matchers](/reference/expect#value-matchers) | same | diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index d243bb21b..994ab50e9 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -960,7 +960,7 @@ Use these helpers to keep behavior consistent across engines: | --- | --- | | `matchesText(actual, pattern)`, `toTextPattern(match, { exact? })`, `describePattern(pattern)` | `TextPattern` matching for `locate` and text matchers; `toTextPattern` throws `INVALID_ARGUMENT` for a `match` that is neither a string nor a `RegExp` | | `resolveExpression(expression, nodes, { selector? })` | The reference `locate`. Resolves one `LocatorExpression` over a semantic tree in document order, with `text` and `label` answering the innermost match when a container echoes a descendant's text. An engine whose tree is the whole truth (a device) calls it; a browser engine reproduces it. A `selector` needs the hook, a `frame` is `FRAME_NOT_FOUND` | -| `urlMatches(current, expected, baseHref)` | `toHaveURL`-style comparison; an `expected` that is neither a string nor a `RegExp` throws `INVALID_ARGUMENT`, as in `toTextPattern` | +| `urlMatches(current, expected, baseHref, ignoreCase?)` | `toHaveURL`-style comparison, `ignoreCase` as on the locator matchers; an `expected` that is neither a string nor a `RegExp` throws `INVALID_ARGUMENT`, as in `toTextPattern` | | `pollCondition({ deadline, signal, negated, evaluate, onTimeout })`, `Deadline` | Assertion polling and the negation grace window | | `validateJsonValue(value, label, rules?)` | Checks a value is plain JSON: a top-level `undefined` passes; a non-finite number, a cycle, an object that is not a plain object or array, or a function, symbol, bigint, or `undefined` inside is `INVALID_ARGUMENT` naming `label`. A `Secret` is `POLICY_DENIED` unless `rules.onSecret` throws its own; `rules.maxDepth` caps the nesting | | `rejectUnknownKeys(label, value, keys, code?)` | Throws `INVALID_CONFIG` for the first own key of `value` outside `keys`, naming the nearest known key when one is a plausible typo (`web() has unknown key "viewprt"; did you mean "viewport"?`) and the known keys otherwise: the check the runner runs on its own config objects, for an engine factory's options. Pass `'INVALID_ARGUMENT'` as `code` for an object a test passes to a fixture method | diff --git a/docs/reference/expect.mdx b/docs/reference/expect.mdx index 9b836c6fe..a219c779f 100644 --- a/docs/reference/expect.mdx +++ b/docs/reference/expect.mdx @@ -58,8 +58,11 @@ before passing its result to `expect`. Locator strictness: except for `toHaveCount` and the list form of `toHaveText` and `toContainText`, a positive matcher requires exactly one match; zero matches keep polling until the timeout, and multiple matches fail -immediately with `LOCATOR_AMBIGUOUS`. `toBeHidden`, negated visibility, and -negated `toBeAttached` accept zero matches. +immediately with `LOCATOR_AMBIGUOUS`. `toBeHidden`, `toBeVisible({ visible: +false })`, `toBeAttached({ attached: false })`, negated visibility, and +negated `toBeAttached` accept zero matches; for them, and for +`waitFor({ state: 'detached' })` and `'hidden'`, a locator under a frame that +is not in the document is zero matches rather than a wait for the frame. Text comparison is normalized: leading and trailing whitespace is trimmed and each run of whitespace collapses to one space. `toHaveText` and @@ -201,8 +204,8 @@ Every matcher call is a recorded `expect.` assertion step. export interface BrowserExpectation { /** Inverts the matcher. */ readonly not: BrowserExpectation; - /** Waits for the current URL to match. */ - toHaveURL(expected: string | RegExp, options?: { timeout?: number }): Promise; + /** Waits for the current URL to match; `ignoreCase` compares a string case-insensitively and adds the `i` flag to a RegExp, `false` removes it. */ + toHaveURL(expected: string | RegExp, options?: { ignoreCase?: boolean; timeout?: number }): Promise; /** Waits for the current title to match. */ toHaveTitle(expected: TextMatch, options?: { timeout?: number }): Promise; /** Waits for the target's `class` attribute to match: a string is the whole normalized class list, a RegExp is tested against it. */ @@ -212,9 +215,10 @@ export interface BrowserExpectation { A relative `toHaveURL` string resolves against the base URL and matches exactly after normalization; a `RegExp` tests the full URL. Anything else, -such as a predicate function, is `INVALID_ARGUMENT`. Each `browser` matcher takes -`timeout` only; any other key, such as Playwright's `ignoreCase` on -`toHaveURL`, is `INVALID_ARGUMENT`. +such as a predicate function, is `INVALID_ARGUMENT`. `toHaveURL` also takes +Playwright's `ignoreCase`, with the locator matchers' semantics. Otherwise each +`browser` matcher takes `timeout` only; any other key, such as `ignoreCase` on +`toHaveTitle`, which Playwright does not take either, is `INVALID_ARGUMENT`. `toHaveClass` polls for the node too: a target that matches nothing yet is one more poll, not a failure, until the assertion timeout runs out. An ambiguous @@ -326,8 +330,11 @@ side is refused. `toHaveProperty` walks a dotted path or an array of keys. Use the array form when a key contains a dot, or to index an array by number (`['lines', 0, 'sku']`). Inherited properties count; a property holding -`undefined` exists. With a second argument the value at the path must equal it -by `toEqual` rules. +`undefined` exists. A string, number, or boolean on the path is read through +its wrapper object, as in Jest, so `toHaveProperty('label.length', 3)` and +`toHaveProperty(['label', 0], 'a')` reach into a string; `null` and +`undefined` have no properties. With a second argument the value at the path +must equal it by `toEqual` rules. ### Matching a schema diff --git a/docs/reference/screen.mdx b/docs/reference/screen.mdx index 39772b02e..10f33adda 100644 --- a/docs/reference/screen.mdx +++ b/docs/reference/screen.mdx @@ -292,8 +292,12 @@ export interface Locator extends Screen { all(): Promise; /** Reads the normalized text of every current match, without auto-waiting; empty when nothing matches. */ allTextContents(): Promise; - /** Waits for the requested locator state. */ - waitFor(options?: { state?: 'visible' | 'hidden'; timeout?: number }): Promise; + /** + * Waits for the requested locator state, `visible` by default: `attached` + * for one match, visible or not; `detached` for none; `hidden` for none or + * a hidden one. + */ + waitFor(options?: { state?: 'attached' | 'detached' | 'visible' | 'hidden'; timeout?: number }): Promise; /** Adds deterministic locator filters. */ filter(options: { hasText?: TextMatch; has?: Locator }): Locator; /** Selects the first current match. */ @@ -382,7 +386,11 @@ await screen.getByLabel('City').pressSequentially('War', { delay: 50 }); ## Locator reads `waitFor` defaults `state` to `'visible'` and `timeout` to -`config.actionTimeout`, and throws `LOCATOR_NOT_FOUND` on timeout. +`config.actionTimeout`, and throws `LOCATOR_NOT_FOUND` on timeout. `'attached'` +waits for one match, hidden or shown, `'detached'` for none, and `'hidden'` for +a hidden match or none. Several matches are `LOCATOR_AMBIGUOUS` in every +state. Any other state, or an option key besides `state` and `timeout`, is +`INVALID_ARGUMENT` before anything is resolved. `textContent` and `allTextContents` return the node's rendered text with whitespace collapsed to single spaces. On the web that is what `innerText` diff --git a/packages/e2e/src/expect/async.ts b/packages/e2e/src/expect/async.ts index 33383488f..f8f3f1c93 100644 --- a/packages/e2e/src/expect/async.ts +++ b/packages/e2e/src/expect/async.ts @@ -34,7 +34,11 @@ interface MatcherSpec { * the matches, `'read'` also reads each of them. */ readonly wholeSet?: boolean | 'read'; - /** The predicate is meaningful even when zero nodes match. */ + /** + * The predicate is meaningful even when zero nodes match, so a locator + * under a frame that is not in the document samples as zero matches + * instead of waiting for the frame. + */ readonly evaluableWithoutNode?: boolean; /** * Whether the one matched node can answer the predicate at all. A node it @@ -220,7 +224,8 @@ class AsyncExpectationImpl implements AsyncExpectation { const refs = await engine.resolveAll(this.internals.expression, deadline); return { count: refs.length, node: null, nodes: [] }; } - const { node, count } = await engine.tryRead(this.internals.expression, deadline); + const missingFrame = spec.evaluableWithoutNode === true ? 'empty' : 'wait'; + const { node, count } = await engine.tryRead(this.internals.expression, deadline, missingFrame); return { count, node, nodes: [] }; } diff --git a/packages/e2e/src/expect/values.ts b/packages/e2e/src/expect/values.ts index aa7572682..62cb6dc8a 100644 --- a/packages/e2e/src/expect/values.ts +++ b/packages/e2e/src/expect/values.ts @@ -287,14 +287,19 @@ class ValueExpectationImpl implements ValueExpectation { } } -/** Walks `keys` into `value`; `present` says whether the whole path resolved. */ +/** + * Walks `keys` into `value`; `present` says whether the whole path resolved. + * A primitive on the way is read through its wrapper object, as Jest does, + * so a string has its `length` and indices; `null` and `undefined` have no + * properties. + */ function lookup(value: unknown, keys: readonly (string | number)[]): { present: boolean; value: unknown } { let current: unknown = value; for (const key of keys) { - if ((typeof current !== 'object' && typeof current !== 'function') || current === null || !(key in current)) { - return { present: false, value: undefined }; - } - current = (current as Record)[key]; + if (current === null || current === undefined) return { present: false, value: undefined }; + const holder: object = Object(current); + if (!(key in holder)) return { present: false, value: undefined }; + current = (holder as Record)[key]; } return { present: true, value: current }; } diff --git a/packages/e2e/src/internal/urls.ts b/packages/e2e/src/internal/urls.ts index 7fb5530b2..d2bd0f902 100644 --- a/packages/e2e/src/internal/urls.ts +++ b/packages/e2e/src/internal/urls.ts @@ -2,7 +2,7 @@ import { ConfigurationError } from './errors.ts'; import { testPattern } from './regexp.ts'; -import { toTextPattern } from './text.ts'; +import { toTextPattern, withIgnoreCase } from './text.ts'; export interface NormalizedBaseUrl { /** Serialized base URL without trailing artifacts beyond the normalized path. */ @@ -179,9 +179,16 @@ export function sameSite(url: string | URL, site: string): boolean { * Relative expected strings resolve against the base URL; string comparison is * exact after WHATWG serialization; regexps test the complete serialized URL. * Anything else is `INVALID_ARGUMENT`, the `toTextPattern` rule. + * `ignoreCase` is Playwright's: `true` compares a string case-insensitively + * and adds the `i` flag to a regexp, `false` removes it. */ -export function urlMatches(current: string, expected: string | RegExp, baseHref: string): boolean { - const pattern = toTextPattern(expected); +export function urlMatches( + current: string, + expected: string | RegExp, + baseHref: string, + ignoreCase?: boolean, +): boolean { + const pattern = withIgnoreCase(toTextPattern(expected), ignoreCase); if (pattern.kind === 'regexp') { return testPattern(pattern.source, pattern.flags, serializeForComparison(current)); } @@ -191,7 +198,10 @@ export function urlMatches(current: string, expected: string | RegExp, baseHref: } catch { return false; } - return serializeForComparison(current) === expectedUrl.href; + const actual = serializeForComparison(current); + return ignoreCase === true + ? actual.toLowerCase() === expectedUrl.href.toLowerCase() + : actual === expectedUrl.href; } function serializeForComparison(url: string): string { diff --git a/packages/e2e/src/locator/engine.ts b/packages/e2e/src/locator/engine.ts index 362f22d83..512cf6139 100644 --- a/packages/e2e/src/locator/engine.ts +++ b/packages/e2e/src/locator/engine.ts @@ -286,12 +286,18 @@ export class LocatorEngine { } } - /** Reads one node for polling surfaces; returns null while zero matches. */ + /** + * Reads one node for polling surfaces; returns null while zero matches. + * A poll that waits for absence passes `missingFrame: 'empty'`: nothing + * under a frame that is not in the document is attached, so the sample is + * zero matches rather than a wait for the frame. + */ async tryRead( expression: LocatorExpression, deadline: Deadline, + missingFrame: MissingFrame = 'wait', ): Promise<{ node: SemanticNode | null; count: number }> { - return this.readOne(expression, deadline, 'wait'); + return this.readOne(expression, deadline, missingFrame); } /** diff --git a/packages/e2e/src/locator/screen.ts b/packages/e2e/src/locator/screen.ts index 32ee55b1c..90fd0cdaa 100644 --- a/packages/e2e/src/locator/screen.ts +++ b/packages/e2e/src/locator/screen.ts @@ -119,6 +119,32 @@ const ROLE_FILTER_KEYS = [...TEXT_OPTION_KEYS, 'checked', 'disabled', 'selected' /** The keys of `RoleOptions`: the accessible name and the filters. */ const ROLE_OPTION_KEYS = ['name', ...ROLE_FILTER_KEYS] as const; +/** The states `locator.waitFor` waits for, Playwright's four. */ +const WAIT_FOR_STATES: readonly string[] = ['attached', 'detached', 'visible', 'hidden']; + +/** The states absence satisfies: a frame missing from the document reads as zero matches for them. */ +const ABSENCE_STATES: ReadonlySet = new Set(['detached', 'hidden']); + +type WaitForState = NonNullable[0]>['state']>; + +/** + * Whether the one match, or its absence, is in `state`: `attached` is any + * match, `detached` none, `visible` a match that is not hidden, and `hidden` + * a hidden match or none. + */ +function inWaitForState(node: SemanticNode | null, state: WaitForState): boolean { + switch (state) { + case 'attached': + return node !== null; + case 'detached': + return node === null; + case 'visible': + return isNodeVisible(node); + case 'hidden': + return !isNodeVisible(node); + } +} + class ScreenImpl implements Screen { constructor( protected readonly context: ScreenContext, @@ -528,8 +554,15 @@ class LocatorImpl extends ScreenImpl implements Locator { return nodes.map((node) => normalizeText(node.text ?? '')); } - async waitFor(options?: { state?: 'visible' | 'hidden'; timeout?: number }): Promise { - const state = options?.state ?? 'visible'; + async waitFor(options?: { state?: WaitForState; timeout?: number }): Promise { + rejectUnknownOptions('waitFor', options, ['state', 'timeout']); + const state = options?.state === undefined ? 'visible' : options.state; + if (!WAIT_FOR_STATES.includes(state)) { + throw new TestError( + 'INVALID_ARGUMENT', + `waitFor state must be one of ${WAIT_FOR_STATES.join(', ')}, got ${JSON.stringify(state)}`, + ); + } await this.context.steps.run('locator', 'locator.waitFor', `${this.label} → ${state}`, async () => { const { engine } = this.context; const deadline = engine.deadline(options?.timeout); @@ -539,9 +572,8 @@ class LocatorImpl extends ScreenImpl implements Locator { signal: engine.signal, negated: false, evaluate: async () => { - const { node } = await engine.tryRead(this.expression, deadline); - const visible = isNodeVisible(node); - return state === 'visible' ? visible : !visible; + const { node } = await engine.tryRead(this.expression, deadline, ABSENCE_STATES.has(state) ? 'empty' : 'wait'); + return inWaitForState(node, state); }, onTimeout: () => new TestError('LOCATOR_NOT_FOUND', `locator did not become ${state}: ${this.label}`, { diff --git a/packages/e2e/src/types.ts b/packages/e2e/src/types.ts index f86e71a4e..a34ea77eb 100644 --- a/packages/e2e/src/types.ts +++ b/packages/e2e/src/types.ts @@ -515,8 +515,12 @@ export interface Locator extends Screen { all(): Promise; /** Reads the normalized text of every current match, without auto-waiting; empty when nothing matches. */ allTextContents(): Promise; - /** Waits for the requested locator state. */ - waitFor(options?: { state?: 'visible' | 'hidden'; timeout?: number }): Promise; + /** + * Waits for the requested locator state, `visible` by default: `attached` + * for one match, visible or not; `detached` for none; `hidden` for none or + * a hidden one. + */ + waitFor(options?: { state?: 'attached' | 'detached' | 'visible' | 'hidden'; timeout?: number }): Promise; /** Adds deterministic locator filters. */ filter(options: { hasText?: TextMatch; has?: Locator }): Locator; /** Selects the first current match. */ diff --git a/packages/e2e/tests/integration/matcher-options.test.ts b/packages/e2e/tests/integration/matcher-options.test.ts index 97d96f2da..47f7e8a31 100644 --- a/packages/e2e/tests/integration/matcher-options.test.ts +++ b/packages/e2e/tests/integration/matcher-options.test.ts @@ -66,9 +66,9 @@ test('an option toBeChecked does not take is refused', async ({ app, screen }) = await expect(screen.getByLabel('Notifications')).toBeChecked(flag('indeterminate', true)); }); -test('an option toHaveURL does not take is refused', async ({ app, browser }) => { +test('an option toHaveTitle does not take is refused', async ({ app, browser }) => { await app.open(); - await expect(browser).toHaveURL('/', flag('ignoreCase', true)); + await expect(browser).toHaveTitle('', flag('ignoreCase', true)); }); test('an option waitForURL does not take is refused', async ({ app, browser }) => { @@ -114,7 +114,7 @@ describe('matcher options in a browser', () => { it.each([ ['an option toBeChecked does not take is refused', 'expect.toBeChecked options has no key "indeterminate"'], - ['an option toHaveURL does not take is refused', 'expect.toHaveURL options has no key "ignoreCase"'], + ['an option toHaveTitle does not take is refused', 'expect.toHaveTitle options has no key "ignoreCase"'], ['an option waitForURL does not take is refused', 'browser.waitForURL options has no key "waitUntil"'], ])('%s', (title, message) => { const result = resultByTitle(outcome, title); diff --git a/packages/e2e/tests/integration/scripted-polling.test.ts b/packages/e2e/tests/integration/scripted-polling.test.ts index 9ecf814b0..62bab05e7 100644 --- a/packages/e2e/tests/integration/scripted-polling.test.ts +++ b/packages/e2e/tests/integration/scripted-polling.test.ts @@ -49,6 +49,12 @@ test('a negated matcher waits for the flip, then the grace window', async ({ app await screen.getByRole('status', { name: 'Loading' }).waitFor({ state: 'hidden' }); }); +test('waitFor attached and detached read presence, not visibility', async ({ app, screen }) => { + await app.open('/'); + await screen.getByText('Hidden content').waitFor({ state: 'attached' }); + await screen.getByText('Never rendered').waitFor({ state: 'detached' }); +}); + test('a missing node is LOCATOR_NOT_FOUND after the deadline', async ({ app, screen }) => { await app.open('/'); await screen.getByRole('button', { name: 'Never exists' }).tap({ timeout: 250 }); @@ -107,6 +113,14 @@ describe('scripted engine: polling, staleness, failure mapping, and scrolling', expect(step(flipped, 'expect.not.toBeVisible').durationMs).toBeGreaterThanOrEqual(1_150); }); + it('waits for presence with the attached and detached states', () => { + const attempt = passed(run.outcome, 'waitFor attached and detached read presence, not visibility'); + expect(attempt.steps.filter((entry) => entry.api === 'locator.waitFor').map((entry) => entry.label)).toEqual([ + 'getByText("Hidden content") → attached', + 'getByText("Never rendered") → detached', + ]); + }); + it('reports a missing node after the deadline and an ambiguous one at once', () => { const missing = failed(run.outcome, 'a missing node is LOCATOR_NOT_FOUND after the deadline', 'LOCATOR_NOT_FOUND', 'matched no nodes within'); expect(missing.error?.details).toMatchObject({ role: 'button', name: 'Never exists', waitedMs: expect.any(Number) }); diff --git a/packages/e2e/tests/integration/web-class-assertions.test.ts b/packages/e2e/tests/integration/web-class-assertions.test.ts index 626f5dd9f..51a68c6b8 100644 --- a/packages/e2e/tests/integration/web-class-assertions.test.ts +++ b/packages/e2e/tests/integration/web-class-assertions.test.ts @@ -3,7 +3,8 @@ * loads it: from a config file, so the engine's `e2e/engine` and the runner's * core are two module copies and `instanceof` cannot tell a runner error apart. * A node that is not there yet must keep the matcher polling; an ambiguous - * locator must still fail at once. + * locator must still fail at once. `toHaveURL` takes Playwright's + * `ignoreCase`. */ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; @@ -38,6 +39,18 @@ test('a missing class attribute is not an empty class list', async ({ app, brows await app.open('/classes'); await expect(browser).toHaveClass(screen.getByTestId('items'), '', { timeout: 300 }); }); + +test('toHaveURL ignoreCase folds the comparison', async ({ app, browser }) => { + await app.open('/classes'); + await expect(browser).toHaveURL('/CLASSES', { ignoreCase: true }); + await expect(browser).toHaveURL(/CLASSES$/, { ignoreCase: true }); + await expect(browser).not.toHaveURL('/CLASSES', { timeout: 300 }); +}); + +test('a negated toHaveURL ignoreCase fails on a case-folded match', async ({ app, browser }) => { + await app.open('/classes'); + await expect(browser).not.toHaveURL('/CLASSES', { ignoreCase: true, timeout: 300 }); +}); `; describe('web class assertions', () => { @@ -80,4 +93,14 @@ describe('web class assertions', () => { expect(missing.attempts[0]!.error?.code).toBe('ASSERTION_FAILED'); expect(missing.attempts[0]!.error?.message).toContain('observed: no class attribute'); }); + + it('honors ignoreCase on toHaveURL, negated too', () => { + const folded = resultByTitle(outcome, 'toHaveURL ignoreCase folds the comparison'); + expect(folded.status, JSON.stringify(folded.attempts[0]?.error)).toBe('passed'); + const negated = resultByTitle(outcome, 'a negated toHaveURL ignoreCase fails on a case-folded match'); + expect(negated.status).toBe('failed'); + expect(negated.attempts[0]!.error?.code).toBe('ASSERTION_FAILED'); + expect(negated.attempts[0]!.error?.message).toContain('expected: not URL /CLASSES (ignoring case)'); + }); + }); diff --git a/packages/e2e/tests/integration/web-platform.test.ts b/packages/e2e/tests/integration/web-platform.test.ts index 197e7afa9..4ef7f6027 100644 --- a/packages/e2e/tests/integration/web-platform.test.ts +++ b/packages/e2e/tests/integration/web-platform.test.ts @@ -422,6 +422,18 @@ test('a nested absent frame counts as zero matches at once', async ({ app, brows expect(Date.now() - started).toBeLessThan(2_000); }); +test('absence matchers and waits under an absent frame pass at once', async ({ app, browser }) => { + await app.open('/frame'); + const absent = browser.frameLocator('#absent').getByRole('button'); + const started = Date.now(); + await expect(absent).toBeAttached({ attached: false, timeout: 3000 }); + await expect(absent).toBeVisible({ visible: false, timeout: 3000 }); + await expect(absent).toBeHidden({ timeout: 3000 }); + await absent.waitFor({ state: 'detached', timeout: 3000 }); + await absent.waitFor({ state: 'hidden', timeout: 3000 }); + expect(Date.now() - started).toBeLessThan(2_000); +}); + test('an action under an absent frame polls for the frame until its timeout', async ({ app, browser }) => { await app.open('/frame'); await browser.frameLocator('#absent').getByRole('button').tap({ timeout: 800 }); @@ -643,6 +655,7 @@ describe('web platform integration', () => { 'dialogs are handled by registered handlers', 'frame locators scope queries into iframes', 'a nested absent frame counts as zero matches at once', + 'absence matchers and waits under an absent frame pass at once', 'nested frame locators resolve each frame inside the one before it', 'css selectors via browser.locator', 'contenteditable hosts are textboxes: reached by label, filled, and read as a value', diff --git a/packages/e2e/tests/types/sdk-types.ts b/packages/e2e/tests/types/sdk-types.ts index 0c9dc75ec..1ee74be8d 100644 --- a/packages/e2e/tests/types/sdk-types.ts +++ b/packages/e2e/tests/types/sdk-types.ts @@ -266,6 +266,10 @@ void asyncExpectation.toContainText(['a', /b/]); void asyncExpectation.toHaveText(['a', 1]); // @ts-expect-error only toHaveText and toContainText take a list; a value is one string void asyncExpectation.toHaveValue(['a']); +void screen.getByRole('button').waitFor({ state: 'attached' }); +void screen.getByRole('button').waitFor({ state: 'detached', timeout: 1000 }); +// @ts-expect-error waitFor takes Playwright's four states +void screen.getByRole('button').waitFor({ state: 'gone' }); // Playwright's state flags and ignoreCase are taken where Playwright takes them, and nowhere else. void asyncExpectation.toBeChecked({ checked: false }); void asyncExpectation.toBeVisible({ visible: false, timeout: 1000 }); diff --git a/packages/e2e/tests/unit/expect-values.test.ts b/packages/e2e/tests/unit/expect-values.test.ts index 1060f71a0..45050ae32 100644 --- a/packages/e2e/tests/unit/expect-values.test.ts +++ b/packages/e2e/tests/unit/expect-values.test.ts @@ -252,6 +252,25 @@ describe('value matchers', () => { failsWith(() => e2eExpect(user).toHaveProperty(''), /dotted path/); failsWith(() => e2eExpect(null).toHaveProperty('a'), /to have property "a"/); }); + + it('toHaveProperty reads a primitive on the path through its wrapper, as Jest does', () => { + const tag = { label: 'abc', count: 3, flag: true, none: null, missing: undefined }; + e2eExpect(tag).toHaveProperty('label.length', 3); + e2eExpect(tag).toHaveProperty(['label', 0], 'a'); + e2eExpect(tag).toHaveProperty('label.2', 'c'); + e2eExpect(tag).not.toHaveProperty('label.3'); + e2eExpect(tag).not.toHaveProperty('label.length', 4); + e2eExpect(tag).toHaveProperty('count.toFixed'); + e2eExpect(tag).toHaveProperty('flag.valueOf'); + e2eExpect('abc').toHaveProperty('length', 3); + e2eExpect(new String('abc')).toHaveProperty('length', 3); + e2eExpect(tag).toHaveProperty('none', null); + e2eExpect(tag).not.toHaveProperty('none.length'); + e2eExpect(tag).not.toHaveProperty('missing.length'); + failsWith(() => e2eExpect(tag).toHaveProperty('label.length', 4), /expected property "label.length" of .* to equal 4, got 3/); + failsWith(() => e2eExpect(tag).toHaveProperty('none.length'), /to have property "none.length"$/); + failsWith(() => e2eExpect(undefined).toHaveProperty('length'), /to have property "length"/); + }); }); describe('asymmetric matchers', () => { diff --git a/packages/e2e/tests/unit/locator-wait-for.test.ts b/packages/e2e/tests/unit/locator-wait-for.test.ts new file mode 100644 index 000000000..326a4b38a --- /dev/null +++ b/packages/e2e/tests/unit/locator-wait-for.test.ts @@ -0,0 +1,116 @@ +/** + * `locator.waitFor` waits for each of Playwright's four states against a + * screen whose nodes arrive and leave on a clock, and refuses a state or an + * option key it does not know before resolving anything: a typo or an + * unimplemented state read as `hidden` would pass at once on an absent node. + */ + +import { afterEach, describe, expect, it } from 'vitest'; +import { resolveExpression, type SemanticNode } from '../../src/engine/index.ts'; +import type { Locator } from '../../src/types.ts'; +import { invalid } from '../helpers/invalid.ts'; +import { screenOver } from '../helpers/screen-over.ts'; +import { snapshot } from '../helpers/snapshot.ts'; + +const CHANGE_AFTER_MS = 1_200; + +const banner: SemanticNode = { ref: { id: 'banner', revision: '' }, role: 'alert', name: 'Saved', testId: 'banner' }; +const hiddenBanner: SemanticNode = { ...banner, states: { hidden: true } }; + +const timers: NodeJS.Timeout[] = []; +afterEach(() => { + for (const timer of timers.splice(0)) clearTimeout(timer); +}); + +/** A screen whose node list starts as `initial` and becomes `later` after 1200 ms, counting every lookup. */ +function changingScreen(initial: readonly SemanticNode[], later: readonly SemanticNode[]) { + let nodes = initial; + let lookups = 0; + timers.push(setTimeout(() => (nodes = later), CHANGE_AFTER_MS)); + const { screen, steps } = screenOver({ + locate: (expression) => { + lookups += 1; + return resolveExpression(expression, nodes); + }, + observe: () => snapshot(nodes), + timeoutMs: 4_000, + }); + return { banner: screen.getByTestId('banner'), screen, steps, lookups: () => lookups }; +} + +/** Milliseconds `locator.waitFor(options)` took to resolve. */ +async function timed(locator: Locator, options: Parameters[0]): Promise { + const started = performance.now(); + await locator.waitFor(options); + return performance.now() - started; +} + +describe('locator.waitFor states', () => { + it.each([ + ['attached', [], [hiddenBanner]], + ['detached', [hiddenBanner], []], + ['visible', [hiddenBanner], [banner]], + ['hidden', [banner], [hiddenBanner]], + ['hidden', [banner], []], + ] as const)('waits for %s until the node changes after 1200 ms', async (state, initial, later) => { + const { banner: locator, steps } = changingScreen(initial, later); + expect(await timed(locator, { state })).toBeGreaterThanOrEqual(CHANGE_AFTER_MS - 50); + expect(steps.all()).toEqual([ + expect.objectContaining({ api: 'locator.waitFor', label: `getByTestId("banner") → ${state}`, status: 'passed' }), + ]); + }); + + it.each([ + ['attached', [hiddenBanner]], + ['detached', []], + ['visible', [banner]], + ['hidden', []], + ] as const)('passes %s at once when the state already holds', async (state, nodes) => { + const { banner: locator } = changingScreen(nodes, nodes); + expect(await timed(locator, { state })).toBeLessThan(CHANGE_AFTER_MS); + }); + + it('attached accepts a hidden node that visible keeps waiting for', async () => { + const { banner: locator } = changingScreen([hiddenBanner], [hiddenBanner]); + await locator.waitFor({ state: 'attached' }); + await expect(locator.waitFor({ state: 'visible', timeout: 100 })).rejects.toMatchObject({ + code: 'LOCATOR_NOT_FOUND', + message: expect.stringContaining('did not become visible'), + }); + }); + + it.each([ + ['attached', []], + ['detached', [hiddenBanner]], + ] as const)('times out %s with LOCATOR_NOT_FOUND when the state never holds', async (state, nodes) => { + const { banner: locator } = changingScreen(nodes, nodes); + await expect(locator.waitFor({ state, timeout: 100 })).rejects.toMatchObject({ + code: 'LOCATOR_NOT_FOUND', + message: `locator did not become ${state}: getByTestId("banner")`, + }); + }); + + it('fails every state at once on several matches', async () => { + const twin: SemanticNode = { ...banner, ref: { id: 'twin', revision: '' } }; + const { banner: locator } = changingScreen([banner, twin], [banner, twin]); + for (const state of ['attached', 'detached', 'visible', 'hidden'] as const) { + await expect(locator.waitFor({ state, timeout: 1_000 })).rejects.toMatchObject({ code: 'LOCATOR_AMBIGUOUS' }); + } + }); +}); + +describe('locator.waitFor validation', () => { + it.each([ + [{ state: 'atached' }, 'waitFor state must be one of attached, detached, visible, hidden, got "atached"'], + [{ state: 'Visible' }, 'waitFor state must be one of attached, detached, visible, hidden, got "Visible"'], + [{ state: null }, 'waitFor state must be one of attached, detached, visible, hidden, got null'], + [{ state: 'visible', strict: true }, 'waitFor options has no key "strict"; it takes state, timeout'], + [{ timeoutMs: 100 }, 'waitFor options has no key "timeoutMs" (now "timeout"); it takes state, timeout'], + ['visible', 'waitFor options must be a plain object'], + ])('rejects %j before any lookup or step', async (options, message) => { + const { banner: locator, steps, lookups } = changingScreen([], []); + await expect(locator.waitFor(invalid(options))).rejects.toMatchObject({ code: 'INVALID_ARGUMENT', message }); + expect(lookups()).toBe(0); + expect(steps.all()).toEqual([]); + }); +}); diff --git a/packages/e2e/tests/unit/urls.test.ts b/packages/e2e/tests/unit/urls.test.ts index cb994edb9..2a57709ac 100644 --- a/packages/e2e/tests/unit/urls.test.ts +++ b/packages/e2e/tests/unit/urls.test.ts @@ -140,6 +140,15 @@ describe('urlMatches', () => { expect(urlMatches('http://localhost:3000/alpha', /beta/, base.href)).toBe(false); }); + it('ignoreCase folds a string comparison and sets or clears the i flag of a regexp', () => { + expect(urlMatches('http://localhost:3000/Billing', '/billing', base.href)).toBe(false); + expect(urlMatches('http://localhost:3000/Billing', '/billing', base.href, true)).toBe(true); + expect(urlMatches('http://localhost:3000/Billing', '/billing', base.href, false)).toBe(false); + expect(urlMatches('http://localhost:3000/Billing', /billing$/, base.href, true)).toBe(true); + expect(urlMatches('http://localhost:3000/Billing', /billing$/i, base.href)).toBe(true); + expect(urlMatches('http://localhost:3000/Billing', /billing$/i, base.href, false)).toBe(false); + }); + it('rejects a predicate instead of matching every URL', () => { const predicate = ((url: URL) => url.pathname === '/other') as unknown as RegExp; expect(() => urlMatches('http://localhost:3000/items', predicate, base.href)).toThrow( diff --git a/packages/web/src/browser.ts b/packages/web/src/browser.ts index 5074f7b38..4f9757bfc 100644 --- a/packages/web/src/browser.ts +++ b/packages/web/src/browser.ts @@ -128,8 +128,8 @@ export type Cookie = CookieFields & export interface BrowserExpectation { /** Inverts the matcher. */ readonly not: BrowserExpectation; - /** Waits for the current URL to match. */ - toHaveURL(expected: string | RegExp, options?: { timeout?: number }): Promise; + /** Waits for the current URL to match; `ignoreCase` compares a string case-insensitively and adds the `i` flag to a RegExp, `false` removes it. */ + toHaveURL(expected: string | RegExp, options?: { ignoreCase?: boolean; timeout?: number }): Promise; /** Waits for the current title to match. */ toHaveTitle(expected: TextMatch, options?: { timeout?: number }): Promise; /** Waits for the target's `class` attribute to match: a string is the whole normalized class list, a RegExp is tested against it. */ @@ -600,20 +600,24 @@ function createBrowserExpectation(deps: ExpectationDeps, negated = false): Brows ), }); }; - const validate = (api: string, options: object | undefined): void => - rejectUnknownOptions(`expect.${negated ? 'not.' : ''}${api}`, options, ['timeout']); + const validate = (api: string, options: object | undefined, keys: readonly string[] = ['timeout']): void => + rejectUnknownOptions(`expect.${negated ? 'not.' : ''}${api}`, options, keys); return { get not() { return createBrowserExpectation(deps, !negated); }, toHaveURL(expected, options) { - validate('toHaveURL', options); + validate('toHaveURL', options, ['ignoreCase', 'timeout']); + const ignoreCase = options?.ignoreCase; + if (ignoreCase !== undefined && typeof ignoreCase !== 'boolean') { + throw new TestError('INVALID_ARGUMENT', `expect.${negated ? 'not.' : ''}toHaveURL option ignoreCase must be a boolean`); + } const label = typeof expected === 'string' ? expected : String(expected); const target = deps.baseHref(); return poll( 'toHaveURL', - `URL ${label}`, - async () => urlMatches(await deps.currentUrl(), expected, target), + `URL ${label}${ignoreCase === true ? ' (ignoring case)' : ''}`, + async () => urlMatches(await deps.currentUrl(), expected, target, ignoreCase), async () => `URL ${await deps.currentUrl()}`, options?.timeout, ); diff --git a/skills/e2e/references/writing-tests.md b/skills/e2e/references/writing-tests.md index 385107dbd..d56dd9849 100644 --- a/skills/e2e/references/writing-tests.md +++ b/skills/e2e/references/writing-tests.md @@ -215,7 +215,7 @@ text, both `[]` at zero. Text is the rendered text, whitespace collapsed: on the web what `innerText` reads (`text-transform` applies, `display: none` drops out, `
` is a space); `toHaveText` reads the same. `isChecked()` is `false`, not an error, on a node with no checked state, so query checkable -controls by role. `waitFor({ state?: 'visible' | 'hidden', timeout? })` waits +controls by role. `waitFor({ state?: 'attached' | 'detached' | 'visible' | 'hidden', timeout? })` waits within `actionTimeout`, else `LOCATOR_NOT_FOUND`. For a value that has to settle use `expect`, not a read. Reading a password field's value or attributes is `POLICY_DENIED`, as is `toHaveAttribute` on one, negated too. @@ -265,8 +265,8 @@ list or tests a RegExp. A failed matcher is `ASSERTION_FAILED`, exit code 1. Playwright's `{ checked: false }`, `{ enabled: false }`, `{ visible: false }`, and `{ attached: false }` flip their matchers, and `{ ignoreCase: true }` -works on `toHaveText`, `toContainText`, `toHaveAccessibleName`, and -`toHaveAttribute(name, value)`. Any other option, `indeterminate` or +works on `toHaveText`, `toContainText`, `toHaveAccessibleName`, +`toHaveAttribute(name, value)`, and `expect(browser).toHaveURL`. Any other option, `indeterminate` or `useInnerText` included, is `INVALID_ARGUMENT`, in JavaScript too. ## Sign-in sessions From 7ab80bc61001dd102006ea020419b74b72ddf0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Sat, 3 Oct 2026 09:30:44 +0200 Subject: [PATCH 07/50] fix(report)!: count interrupted tests apart from failures (#713) * fix(report)!: count interrupted tests apart from failures * fix(report): tell a failed test from its last failed attempt, whatever the cut retry left --- .changeset/forced-interrupt-file-reporters.md | 5 ++ .changeset/report-interrupted-count.md | 6 ++ docs/ci.mdx | 2 +- docs/github.mdx | 6 +- docs/reference/cli.mdx | 19 ++++- docs/reference/reporters.mdx | 2 +- docs/telemetry.mdx | 2 +- .../schema/fixtures/report-v1.invalid.json | 12 ++- .../e2e/schema/fixtures/report-v1.valid.json | 1 + packages/e2e/schema/report-v1.schema.json | 3 +- packages/e2e/src/report/build.ts | 20 ++++- packages/e2e/src/report/failure-text.ts | 17 +++- packages/e2e/src/report/format.ts | 21 +++-- packages/e2e/src/report/junit.ts | 22 ++--- packages/e2e/src/report/list.ts | 31 ++++--- packages/e2e/src/report/markdown.ts | 51 +++++++---- packages/e2e/src/report/repeats.ts | 20 ++++- packages/e2e/src/run/retry.ts | 6 +- packages/e2e/src/run/runner.ts | 25 ++++-- packages/e2e/src/run/serial.ts | 4 +- packages/e2e/src/telemetry/events.ts | 1 + packages/e2e/tests/helpers/report.ts | 2 +- packages/e2e/tests/helpers/sample-report.ts | 2 +- .../integration/runner-lifecycle.test.ts | 84 ++++++++++++++++++- packages/e2e/tests/types/sdk-types.ts | 2 + packages/e2e/tests/unit/report-junit.test.ts | 41 ++++++++- packages/e2e/tests/unit/report-list.test.ts | 27 ++++-- .../e2e/tests/unit/report-markdown.test.ts | 82 ++++++++++++++++++ .../e2e/tests/unit/report-repeats.test.ts | 18 +++- packages/e2e/tests/unit/retry.test.ts | 9 ++ .../e2e/tests/unit/schema-fixtures.test.ts | 8 ++ .../e2e/tests/unit/telemetry-events.test.ts | 3 +- packages/github/src/merge.ts | 18 ++-- packages/github/tests/unit/merge.test.ts | 35 +++++++- skills/e2e/references/running.md | 12 ++- 35 files changed, 519 insertions(+), 100 deletions(-) create mode 100644 .changeset/forced-interrupt-file-reporters.md create mode 100644 .changeset/report-interrupted-count.md diff --git a/.changeset/forced-interrupt-file-reporters.md b/.changeset/forced-interrupt-file-reporters.md new file mode 100644 index 000000000..10a18cab0 --- /dev/null +++ b/.changeset/forced-interrupt-file-reporters.md @@ -0,0 +1,5 @@ +--- +'e2e': patch +--- + +A forced interrupt (a second Ctrl-C, or a CI cancel that sends SIGINT and then SIGTERM) still writes `junit.xml` and `summary.md`. The runner used to abandon them along with every other reporter, which left the previous run's files beside the new `report.json`. Custom reporters are still abandoned on force. diff --git a/.changeset/report-interrupted-count.md b/.changeset/report-interrupted-count.md new file mode 100644 index 000000000..d978f0dcb --- /dev/null +++ b/.changeset/report-interrupted-count.md @@ -0,0 +1,6 @@ +--- +'e2e': minor +'@e2e-dev/github': patch +--- + +Breaking: an interrupted test is no longer counted as failed. `report.json` gains `run.summary.interrupted`, and `run.summary.failed` counts only failed and timed-out tests. `run.summary.skipped` now counts only selected tests, so `passed + failed + interrupted + flaky + skipped` equals `selected`; the tests a filter left out are `discovered - selected`. The `list` reporter prints `3 interrupted` in its own column, `summary.md` shows them with ⏹️ and gives them no failure block or page, and `junit.xml` writes each one as a `` whose message starts `interrupted:`. A test that failed and whose retry an interrupt cut short stays failed, and its failure is the one reported. A `--repeat-each` run the interrupt stopped is listed as interrupted, not as a flake. The telemetry event gains `tests_interrupted`. In `@e2e-dev/github`, a test that was interrupted and then passed on a `--last-failed` rerun shows as passed, not flaky. diff --git a/docs/ci.mdx b/docs/ci.mdx index 6d64efbcc..f6fc883d2 100644 --- a/docs/ci.mdx +++ b/docs/ci.mdx @@ -193,7 +193,7 @@ flaky, so you can identify flows that need investigation. `--max-failures ` stops the run once that many tests have failed, so a broken deploy does not spend a model call on every remaining agent step. The tests not started are skipped with cause `failure-limit`, the ones running -end as `interrupted`, and the job exits 1. +end as `interrupted`, counted apart from the failures, and the job exits 1. ## Exit codes diff --git a/docs/github.mdx b/docs/github.mdx index f7570579a..242eac887 100644 --- a/docs/github.mdx +++ b/docs/github.mdx @@ -143,9 +143,11 @@ passes: The comment covers the whole suite, not only the tests the rerun ran again. A test that failed and then passed shows as flaky, a test that failed twice -stays failed, and the rest keep their first-pass results. The comment +stays failed, one that was interrupted and then passed shows as passed, and +the rest keep their first-pass results. The comment matches the job's status: it stays red when another filter kept a failed -test out of the rerun. +test out of the rerun, and shows ⏹️ when it kept out a test the first pass +never finished. ## Matrix jobs diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx index df50aa29b..6e2c398f0 100644 --- a/docs/reference/cli.mdx +++ b/docs/reference/cli.mdx @@ -479,8 +479,8 @@ bunx e2e run --reporter list,junit | --- | --- | | `list` | Terminal progress, failures with code frames, and a run summary. On a TTY, live progress includes the current agent step, model turns, tool calls, and reasoning excerpts. Untrusted fields are capped at 8 KiB and stripped of control characters. | | `json` | The report document to stdout and nothing else there, so `e2e run --reporter json \| jq` works. Same document as `/report.json`, described by `schema/report-v1.schema.json` in the package. The shape changes only with the `schemaVersion` field (`report-1` today). Cannot be combined with `list`. | -| `junit` | `junit.xml` beside the report. One `` per file, one `` per test-target pair named `title > path [target]`. A test-class error is a ``, any other an ``, a skip a `` with its reason. Run-level errors form a `run` suite. | -| `markdown` | `summary.md` beside the report with counts, cost, errors, and failure details. Failed and flaky tests also get a page under `failures/` there, with steps, model turns, and the screen at failure. Exploration runs show findings. [`@e2e-dev/github`](/github) uses this summary for pull request comments. | +| `junit` | `junit.xml` beside the report. One `` per file, one `` per test-target pair named `title > path [target]`. A test-class error is a ``, any other an ``, a skip a `` with its reason. An interrupted test reached no verdict, so it is a `` whose message starts `interrupted:`. Run-level errors form a `run` suite. | +| `markdown` | `summary.md` beside the report with counts, cost, errors, and failure details. Failed and flaky tests also get a page under `failures/` there, with steps, model turns, and the screen at failure. Interrupted tests are counted on their own and get no page. Exploration runs show findings. [`@e2e-dev/github`](/github) uses this summary for pull request comments. | Every run that gets to its tests writes `report.json` under the [output directory](/reference/config#output) (`.e2e` by default) whatever the reporters are, and the reporters' files @@ -490,6 +490,16 @@ fails to start, an interrupt) writes none of them and leaves the previous run's files in place; its errors are on the terminal and in the exit code. A `junit.xml` or `summary.md` that could not be written is a line on stderr, never a run error. +`run.summary` in the report counts the results. `discovered` is every +test-target pair the runner collected and `selected` the ones this run +chose; the selection's left-out pairs are in `results` as `selected: false` +but in no other count. `passed`, `failed` (timed out included), +`interrupted`, `flaky`, and `skipped` count the selected results and add up +to `selected`. `interrupted` is a test the run stopped before its verdict +(a signal, `--max-failures`, or a run-level error such as +`MODEL_UNAVAILABLE`), never a failure. `executed` is the selected +results that ran at least one attempt. + With `--ai-trace`, the run also writes `/ai-trace.json`: one entry per model round trip with the exact prompt, tool definitions, response, usage, and provider metadata. Observations in it are already redacted. Inline image @@ -510,7 +520,10 @@ Signals escalate. any test starts (during collection, an engine's `prepare`, or app startup) writes no report and leaves the previous one in place. 2. A second signal forces. Workers dispose their engines at once and are - killed when the cleanup budget is spent. The report is still written. + killed when the cleanup budget is spent. The report is still written, + and so are `junit.xml` and `summary.md` when the `junit` and + `markdown` reporters are on; a custom reporter's + `onRunFinished` is abandoned. 3. A third signal exits on the spot, after killing the process groups of every app command the run started. A process attached through `reuseExisting` is left alone. diff --git a/docs/reference/reporters.mdx b/docs/reference/reporters.mdx index 392516234..b53f7ce12 100644 --- a/docs/reference/reporters.mdx +++ b/docs/reference/reporters.mdx @@ -171,7 +171,7 @@ prints. All reporters run concurrently, each with a one-minute deadline. A timeout, exception, or invalid return value is reported on stderr without changing the run's status, exit code, or report. -One Ctrl-C still lets reporters finish. A second Ctrl-C stops waiting for them. +One Ctrl-C still lets reporters finish. A second Ctrl-C stops waiting for them, except the built-in `junit` and `markdown`, which still write their files so none of the previous run's is left beside the new report. The second argument is an `AbortSignal` that aborts when the budget runs out or the run is forced to stop. Hand it to every request and timer, or an diff --git a/docs/telemetry.mdx b/docs/telemetry.mdx index 3b295a7b4..ea7d1ea71 100644 --- a/docs/telemetry.mdx +++ b/docs/telemetry.mdx @@ -119,7 +119,7 @@ Versions: | `command` | `run` or `explore` | | `status`, `exit_code`, `duration_ms` | The run's outcome | | `flags` | The names of the flags given to the run command, never their values | -| `tests_discovered`, `tests_selected`, `tests_executed`, `tests_passed`, `tests_failed`, `tests_flaky`, `tests_skipped` | The summary counts | +| `tests_discovered`, `tests_selected`, `tests_executed`, `tests_passed`, `tests_failed`, `tests_interrupted`, `tests_flaky`, `tests_skipped` | The summary counts | | `attempts_total`, `tests_retried` | How many attempts the run made across tests and serial groups, and how many of them needed more than one | | `targets`, `platforms`, `engines` | How many targets, on which platforms, and with which engines and versions, as your config declares them: `web`, `ios`, `web@0.6.1`, or your own engine's name and version. A name that is not a plain token of letters, digits, dots, dashes, and underscores is reported as `other` | | `steps_total`, `steps_agent`, `steps_locator`, `steps_assertion`, `steps_screen`, `steps_app`, `steps_session`, `steps_resource` | Step counts by kind | diff --git a/packages/e2e/schema/fixtures/report-v1.invalid.json b/packages/e2e/schema/fixtures/report-v1.invalid.json index a5aa79db0..846f86937 100644 --- a/packages/e2e/schema/fixtures/report-v1.invalid.json +++ b/packages/e2e/schema/fixtures/report-v1.invalid.json @@ -81,6 +81,16 @@ "targetId": "web" } } - ] + ], + "summary": { + "discovered": 1, + "selected": 1, + "executed": 1, + "passed": 0, + "failed": 0, + "interrupted": -1, + "flaky": 0, + "skipped": 0 + } } } diff --git a/packages/e2e/schema/fixtures/report-v1.valid.json b/packages/e2e/schema/fixtures/report-v1.valid.json index 1e17aa491..2dab3aca8 100644 --- a/packages/e2e/schema/fixtures/report-v1.valid.json +++ b/packages/e2e/schema/fixtures/report-v1.valid.json @@ -369,6 +369,7 @@ "executed": 2, "passed": 1, "failed": 0, + "interrupted": 0, "flaky": 0, "skipped": 1 }, diff --git a/packages/e2e/schema/report-v1.schema.json b/packages/e2e/schema/report-v1.schema.json index ed414137b..527839bf4 100644 --- a/packages/e2e/schema/report-v1.schema.json +++ b/packages/e2e/schema/report-v1.schema.json @@ -570,13 +570,14 @@ "summary": { "type": "object", "additionalProperties": false, - "required": ["discovered", "selected", "executed", "passed", "failed", "flaky", "skipped"], + "required": ["discovered", "selected", "executed", "passed", "failed", "interrupted", "flaky", "skipped"], "properties": { "discovered": { "type": "integer", "minimum": 0 }, "selected": { "type": "integer", "minimum": 0 }, "executed": { "type": "integer", "minimum": 0 }, "passed": { "type": "integer", "minimum": 0 }, "failed": { "type": "integer", "minimum": 0 }, + "interrupted": { "type": "integer", "minimum": 0 }, "flaky": { "type": "integer", "minimum": 0 }, "skipped": { "type": "integer", "minimum": 0 } } diff --git a/packages/e2e/src/report/build.ts b/packages/e2e/src/report/build.ts index 759d27314..ea7f2705e 100644 --- a/packages/e2e/src/report/build.ts +++ b/packages/e2e/src/report/build.ts @@ -319,12 +319,20 @@ export interface ReportUsage { estimatedCostUsd?: number; } +/** + * The run's counts by result. `passed`, `failed` (timed out included), + * `interrupted`, `flaky`, and `skipped` count selected results only and add + * up to `selected`; `discovered` minus `selected` is what the selection left + * out. + */ export interface ReportSummary { discovered: number; selected: number; executed: number; passed: number; failed: number; + /** Stopped by the run (a signal, `--max-failures`, a run-level error) before reaching a verdict; never counted as failed. */ + interrupted: number; flaky: number; skipped: number; } @@ -503,19 +511,21 @@ function serializeTarget( }; } -/** Computes report-1 summary counts from results. */ +/** Computes report-1 summary counts from results; the status counts are over the selected ones. */ function computeSummary(results: readonly ResultRecord[]): ReportSummary { let selected = 0; let executed = 0; let passed = 0; let failed = 0; + let interrupted = 0; let flaky = 0; let skipped = 0; for (const result of results) { - if (result.selected) selected += 1; if (result.attempts.length > 0 || (result.serialGroupId !== undefined && result.status !== 'skipped')) { executed += 1; } + if (!result.selected) continue; + selected += 1; switch (result.status) { case 'passed': passed += 1; @@ -525,15 +535,17 @@ function computeSummary(results: readonly ResultRecord[]): ReportSummary { break; case 'failed': case 'timed-out': - case 'interrupted': failed += 1; break; + case 'interrupted': + interrupted += 1; + break; case 'skipped': skipped += 1; break; } } - return { discovered: results.length, selected, executed, passed, failed, flaky, skipped }; + return { discovered: results.length, selected, executed, passed, failed, interrupted, flaky, skipped }; } /** diff --git a/packages/e2e/src/report/failure-text.ts b/packages/e2e/src/report/failure-text.ts index cb276401d..a7500ad7b 100644 --- a/packages/e2e/src/report/failure-text.ts +++ b/packages/e2e/src/report/failure-text.ts @@ -85,9 +85,18 @@ function screenUrl(url: string): string { } } -/** The attempt whose story a block tells: a flaky test's last failure, otherwise the final one. */ +/** + * The attempt whose story a block tells: a flaky test's last failure; for a + * failed or timed-out test, the last attempt that reached that verdict, so a + * retry the run interrupted (or a serial member that retry never reached) + * tells nothing; otherwise the final one. + */ export function toldAttempt(result: ReportResult, final: Outcome): AttemptView { - return result.status === 'flaky' ? (final.lastFailed ?? final.final) : final.final; + if (result.status === 'flaky') return final.lastFailed ?? final.final; + if (result.status === 'failed' || result.status === 'timed-out') { + return final.attempts.findLast((attempt) => attempt.status === 'failed' || attempt.status === 'timed-out') ?? final.final; + } + return final.final; } /** A step that did not pass. */ @@ -108,9 +117,9 @@ export function failedStepOf(steps: readonly ReportStep[]): { index: number; ste */ export function attemptsLine(result: ReportResult, final: Outcome): string | undefined { // An attempt that recorded nothing for this test (a serial group that never - // reached the member) says nothing about how the test fails. + // reached the member), or one the run interrupted, says nothing about how the test fails. const failed = final.attempts - .filter((attempt) => attempt.status !== 'passed' && attempt.status !== 'skipped' && (attempt.error !== undefined || attempt.steps.length > 0)) + .filter((attempt) => attempt.status !== 'passed' && attempt.status !== 'skipped' && attempt.status !== 'interrupted' && (attempt.error !== undefined || attempt.steps.length > 0)) .map((attempt) => ({ code: cell(attempt.error?.code ?? attempt.status, 128), step: failedStepOf(attempt.steps)?.index, diff --git a/packages/e2e/src/report/format.ts b/packages/e2e/src/report/format.ts index 28d4c98ba..87e75ec01 100644 --- a/packages/e2e/src/report/format.ts +++ b/packages/e2e/src/report/format.ts @@ -115,18 +115,23 @@ export function rule(text: string, align: 'center' | 'right'): { before: string; export interface Counters { passed: number; failed: number; + interrupted: number; flaky: number; skipped: number; total: number; } export function emptyCounters(): Counters { - return { passed: 0, failed: 0, flaky: 0, skipped: 0, total: 0 }; + return { passed: 0, failed: 0, interrupted: 0, flaky: 0, skipped: 0, total: 0 }; } -/** The counter a result status lands in: every non-success status is a failure. */ -export function statusBucket(status: ResultStatus): 'passed' | 'failed' | 'flaky' | 'skipped' { - return status === 'passed' || status === 'flaky' || status === 'skipped' ? status : 'failed'; +/** + * The counter a result status lands in. A timeout is a failure; an + * interrupted test was stopped by the run, not by its own verdict, so it + * counts on its own. + */ +export function statusBucket(status: ResultStatus): 'passed' | 'failed' | 'interrupted' | 'flaky' | 'skipped' { + return status === 'timed-out' ? 'failed' : status; } /** Counts results by bucket; `total` is the number of items. */ @@ -139,17 +144,19 @@ export function tally(items: readonly { readonly status: ResultStatus }[]): Coun return counters; } -/** A file's outcome from its tests: failed if any did, skipped if all did, else passed. */ -export function fileOutcome(counters: Counters): 'passed' | 'failed' | 'skipped' { +/** A file's outcome from its tests: failed if any did, else interrupted if any was, skipped if all were, else passed. */ +export function fileOutcome(counters: Counters): 'passed' | 'failed' | 'interrupted' | 'skipped' { if (counters.failed > 0) return 'failed'; + if (counters.interrupted > 0) return 'interrupted'; if (counters.total > 0 && counters.skipped === counters.total) return 'skipped'; return 'passed'; } -/** `2 failed | 10 passed | 1 flaky (13)` in vitest's colors and order. */ +/** `2 failed | 1 interrupted | 10 passed | 1 flaky (14)` in vitest's colors and order. */ export function stateString(pc: Colors, counters: Counters): string { const parts = [ counters.failed > 0 ? pc.bold(pc.red(`${counters.failed} failed`)) : undefined, + counters.interrupted > 0 ? pc.bold(pc.yellow(`${counters.interrupted} interrupted`)) : undefined, pc.bold(pc.green(`${counters.passed} passed`)), counters.flaky > 0 ? pc.yellow(`${counters.flaky} flaky`) : undefined, counters.skipped > 0 ? pc.gray(`${counters.skipped} skipped`) : undefined, diff --git a/packages/e2e/src/report/junit.ts b/packages/e2e/src/report/junit.ts index 89756a774..1bf44fcb4 100644 --- a/packages/e2e/src/report/junit.ts +++ b/packages/e2e/src/report/junit.ts @@ -3,6 +3,7 @@ import { sanitizeText } from '../internal/errors.ts'; import type { Report1Document, ReportError, ReportResult, ReportSerialGroup } from './build.ts'; import { repeatSuffix } from './format.ts'; +import { toldAttempt } from './failure-text.ts'; import { outcome } from './outcome.ts'; import { fileReporter } from './write.ts'; @@ -83,10 +84,9 @@ function addCounts(total: SuiteCounts, part: SuiteCounts): void { total.timeMs += part.timeMs; } -/** `` for a product verdict, `` for anything that prevented one. */ -function verdictElement(error: ReportError | undefined, status: ReportResult['status']): 'failure' | 'error' { - if (error !== undefined) return error.category === 'test' ? 'failure' : 'error'; - return status === 'interrupted' ? 'error' : 'failure'; +/** `` for a product verdict, `` for anything else that prevented one. */ +function verdictElement(error: ReportError | undefined): 'failure' | 'error' { + return error === undefined || error.category === 'test' ? 'failure' : 'error'; } function errorBody(error: ReportError): string { @@ -117,18 +117,18 @@ function renderResult(result: ReportResult, groups: ReadonlyMap`, - ` `, - '', - ], + lines: [`${open}>`, ` `, ''], outcome: 'skipped', durationMs: final.durationMs, }; + } default: { - const element = verdictElement(final.final.error, result.status); - const error = final.final.error; + const error = toldAttempt(result, final).error; + const element = verdictElement(error); const message = error?.message ?? result.status; const type = error?.code ?? result.status; const body = error === undefined ? result.status : errorBody(error); diff --git a/packages/e2e/src/report/list.ts b/packages/e2e/src/report/list.ts index 44e57ca35..49e8c2b08 100644 --- a/packages/e2e/src/report/list.ts +++ b/packages/e2e/src/report/list.ts @@ -116,14 +116,16 @@ interface ResultDetails { /** Details of an ordinary result: summed over its attempts, the error from the last. */ function attemptDetails(attempts: readonly AttemptRecord[]): ResultDetails { const steps = attempts.map((attempt) => attempt.steps); + // A retry the run interrupted tells nothing; the failure before it is the verdict. + const told = attempts.findLast((attempt) => attempt.status !== 'interrupted') ?? attempts.at(-1); return { durationMs: attempts.reduce((total, attempt) => total + attempt.durationMs, 0), usage: stepsUsage(steps), models: stepsModelTally(steps), cache: stepsCacheTally(steps), - error: attempts[attempts.length - 1]?.error, + error: told?.error, videos: videoPaths(attempts), - ...failureOf(attempts[attempts.length - 1]), + ...failureOf(told), }; } @@ -161,7 +163,7 @@ function serialMemberDetails(group: SerialGroupRecord, testId: string): ResultDe attempt, member: attempt.members.find((member) => member.testId === testId), })); - const last = runs[runs.length - 1]; + const last = runs.findLast((run) => run.attempt.status !== 'interrupted') ?? runs.at(-1); const own = last?.member; const neverRan = own === undefined || own.status === 'skipped'; const steps = runs.map((run) => run.member?.steps ?? []); @@ -654,9 +656,9 @@ export class ListReporter implements Reporter { addCacheTally(this.runCache, cache); const title = this.titledAs(result.test.titlePath.join(' > '), result.agent, result.repeat); if (this.explore !== undefined) { - // The exploration's verdict is its findings; any other error is a failure of its own. + // The exploration's verdict is its findings; any other error is a failure of its own, unless the run stopped it. this.explore.result(result.attempts.flatMap((attempt) => attempt.artifacts)); - if (error !== undefined && !this.explore.isVerdict(error)) { + if (error !== undefined && result.status !== 'interrupted' && !this.explore.isVerdict(error)) { this.failures.push({ group, title, status: result.status, error, videos, failure, screenPath }); } this.window.redraw(); @@ -721,12 +723,15 @@ export class ListReporter implements Reporter { const symbol = outcome === 'failed' ? pc.red(F_POINTER) - : outcome === 'skipped' - ? pc.dim(pc.gray(F_DOWN)) - : pc.green(F_CHECK); + : outcome === 'interrupted' + ? pc.yellow(F_POINTER) + : outcome === 'skipped' + ? pc.dim(pc.gray(F_DOWN)) + : pc.green(F_CHECK); const state = [ pc.dim(`${counts.total} test${counts.total === 1 ? '' : 's'}`), counts.failed > 0 ? pc.red(`${counts.failed} failed`) : undefined, + counts.interrupted > 0 ? pc.yellow(`${counts.interrupted} interrupted`) : undefined, counts.flaky > 0 ? pc.yellow(`${counts.flaky} flaky`) : undefined, counts.skipped > 0 ? pc.yellow(`${counts.skipped} skipped`) : undefined, ] @@ -744,6 +749,7 @@ export class ListReporter implements Reporter { this.print(parts.join(' ')); const verbose = counts.failed > 0 || + counts.interrupted > 0 || counts.flaky > 0 || this.groups.size === 1 || (this.live && group.lines.some((line) => line.steps.length > 0)); @@ -774,6 +780,8 @@ export class ListReporter implements Reporter { : pc.dim(pc.gray(` [${line.skipReason}]`)); return [` ${pc.dim(pc.gray(F_DOWN))} ${line.title}${reason}`]; } + case 'interrupted': + return [` ${pc.yellow(F_CROSS)} ${line.title} ${pc.yellow('(interrupted)')} ${duration}${ai}`]; default: { const status = line.status === 'failed' ? '' : pc.red(` (${line.status})`); const rows = [` ${pc.red(`${F_CROSS} ${line.title}`)}${status} ${duration}${ai}`]; @@ -832,11 +840,12 @@ export class ListReporter implements Reporter { const { pc } = this; const groups = repeatGroups(this.runs, (entry) => entry); if (groups.length === 0) return []; - const unstable = groups.filter((group) => group.passed < group.runs.length); + // A test is listed when it missed a run: a flake first, then one the run cut short. + const missed = groups.filter((group) => group.passed < group.runs.length).toSorted((a, b) => Number(b.unstable) - Number(a.unstable)); const summary = repeatSummary(groups); return [ - padTitle(pc, 'Repeats') + (unstable.length === 0 ? pc.green(summary) : pc.yellow(summary)), - ...unstable.map((group) => `${padTitle(pc, '')}${pc.yellow(F_CROSS)} ${group.label} ${pc.dim(repeatLine(group))}`), + padTitle(pc, 'Repeats') + (missed.some((group) => group.unstable) ? pc.yellow(summary) : pc.green(summary)), + ...missed.map((group) => `${padTitle(pc, '')}${group.unstable ? pc.yellow(F_CROSS) : pc.dim(pc.gray(F_DOWN))} ${group.label} ${pc.dim(repeatLine(group))}`), ]; } diff --git a/packages/e2e/src/report/markdown.ts b/packages/e2e/src/report/markdown.ts index 1d7ba3622..2e1775a63 100644 --- a/packages/e2e/src/report/markdown.ts +++ b/packages/e2e/src/report/markdown.ts @@ -23,7 +23,7 @@ import { collapseText } from '../internal/text.ts'; import type { Report1Document, ReportError, ReportExplore, ReportExploreFinding, ReportResult, ReportStep, ReportTarget } from './build.ts'; import { ENDED_TEXT, orderFindings, SEVERITY_WORDS, stepCountParts } from './explore-text.ts'; import { formatCost, formatTokens, repeatSuffix, statusBucket, tally, type Counters } from './format.ts'; -import { repeatGroups, repeatLine, repeatSummary } from './repeats.ts'; +import { repeatGroups, repeatLine, repeatSummary, type RepeatGroup } from './repeats.ts'; import { attemptsLine, detailLines, @@ -101,11 +101,11 @@ interface Entry { /** Screenshots first, then the recording, then the trace; the compiler fails when a kind is missing here. */ const KIND_RANK: Record = { screenshot: 0, video: 1, trace: 2, download: 3, log: 4 }; -const ICON: Record = { failed: '🔴', flaky: '⚠️', skipped: '⏭️', passed: '🟢' }; +const ICON: Record = { failed: '🔴', interrupted: '⏹️', flaky: '⚠️', skipped: '⏭️', passed: '🟢' }; /** Worst first: the order failures are listed, files are sorted, and a file's glyph is chosen in. */ -const WORST_FIRST: readonly Bucket[] = ['failed', 'flaky', 'skipped', 'passed']; -/** The order counts read in: `1 failed, 1 flaky, 3 passed, 1 skipped`. */ -const COUNT_ORDER: readonly Bucket[] = ['failed', 'flaky', 'passed', 'skipped']; +const WORST_FIRST: readonly Bucket[] = ['failed', 'interrupted', 'flaky', 'skipped', 'passed']; +/** The order counts read in: `1 failed, 2 interrupted, 1 flaky, 3 passed, 1 skipped`. */ +const COUNT_ORDER: readonly Bucket[] = ['failed', 'interrupted', 'flaky', 'passed', 'skipped']; function worstBucket(results: readonly ReportResult[]): Bucket { const present = new Set(results.map((result) => statusBucket(result.status))); @@ -125,9 +125,23 @@ function pageName(options: MarkdownReportOptions): string { return title === '' ? 'e2e' : `e2e ${cell(title, MAX_ID_CHARS)}`; } +/** + * The headline's glyph: red for a failed run or any failed test, the + * interrupted glyph for a run or a listed test the run stopped, else green. + * A `--last-failed` rerun that passed can still list a test the first pass + * never finished, and the page must not read as all green. + */ +function runIcon(run: ReportRun, results: readonly ReportResult[]): string { + if (run.status !== 'passed' && run.status !== 'interrupted') return ICON.failed; + const worst = worstBucket(results); + if (worst === 'failed') return ICON.failed; + if (worst === 'interrupted' || run.status === 'interrupted') return ICON.interrupted; + return ICON.passed; +} + function headline(run: ReportRun, results: readonly ReportResult[], options: MarkdownReportOptions): string { const summary = countText(tally(results)) || (run.errors.length > 0 ? 'no tests ran' : 'no tests selected'); - return `### ${run.status === 'passed' ? ICON.passed : ICON.failed} ${pageName(options)}: ${summary}`; + return `### ${runIcon(run, results)} ${pageName(options)}: ${summary}`; } /** The model calls an agent step made; the step's metrics are the source of truth. */ @@ -301,6 +315,12 @@ function flakyFold(entries: readonly Entry[], manyTargets: boolean, options: Mar return [['
', `${summary}`, '', shown.join('\n\n'), '
'].join('\n')]; } +/** A test that missed a run: flaky when a run failed, else interrupted when the run stopped one, else skipped. */ +function missIcon(group: RepeatGroup): string { + if (group.unstable) return ICON.flaky; + return group.runs.some((run) => run.status === 'interrupted') ? ICON.interrupted : ICON.skipped; +} + /** * A `--repeat-each` run's tally: how many tests passed every run, then each * test that did not, with the runs that failed. Empty for a run that @@ -318,10 +338,11 @@ function repeatsSection(entries: readonly Entry[], manyTargets: boolean): string run: { repeat: result.repeat, status: result.status, code: final.final.error?.code ?? final.lastFailed?.error?.code }, })); if (groups.length === 0) return []; - const unstable = groups.filter((group) => group.passed < group.runs.length); + // A test is listed when it missed a run: a flake first, then one the run cut short. + const missed = groups.filter((group) => group.passed < group.runs.length).toSorted((a, b) => Number(b.unstable) - Number(a.unstable)); return [ `**Repeats:** ${repeatSummary(groups)}`, - ...(unstable.length === 0 ? [] : ['', ...unstable.map((group) => `- ${ICON.flaky} ${group.label}: ${cell(repeatLine(group))}`)]), + ...(missed.length === 0 ? [] : ['', ...missed.map((group) => `- ${missIcon(group)} ${group.label}: ${cell(repeatLine(group))}`)]), ]; } @@ -426,13 +447,13 @@ function footer(run: ReportRun, targets: readonly ReportTarget[], options: Markd // --- the exploration record --- -function exploreHeadline(run: ReportRun, explore: ReportExplore, options: MarkdownReportOptions): string { +function exploreHeadline(run: ReportRun, results: readonly ReportResult[], explore: ReportExplore, options: MarkdownReportOptions): string { const issues = explore.findings.filter((finding) => finding.kind === 'issue').length; const warnings = explore.findings.length - issues; const counts = [...(issues > 0 ? [plural(issues, 'issue')] : []), ...(warnings > 0 ? [plural(warnings, 'warning')] : [])]; const summary = counts.length > 0 ? counts.join(', ') : run.status === 'blocked' ? 'explored nothing' : run.status === 'passed' ? 'no findings' : 'did not finish'; - return `### ${run.status === 'passed' ? ICON.passed : ICON.failed} ${pageName(options)} explore: ${summary}`; + return `### ${runIcon(run, results)} ${pageName(options)} explore: ${summary}`; } /** `3 of 8 steps · 2 passed · 1 failed · the agent covered the goal`. */ @@ -529,7 +550,7 @@ export function renderMarkdownReport(report: Report1Document, options: MarkdownR ); const spend = spendLine(run, entries); - const head = [explore === undefined ? headline(run, selected, options) : exploreHeadline(run, explore, options), ...(spend === undefined ? [] : [spend]), '']; + const head = [explore === undefined ? headline(run, selected, options) : exploreHeadline(run, selected, explore, options), ...(spend === undefined ? [] : [spend]), '']; const groups = fileGroups(entries); const sections = explore === undefined @@ -552,9 +573,9 @@ function failurePageName(result: ReportResult): string { } /** - * The results that get a page: every one that failed, timed out, was - * interrupted, or was flaky, except an exploration's own verdict, which its - * findings already tell. + * The results that get a page: every one that failed, timed out, or was + * flaky, except an exploration's own verdict, which its findings already + * tell. An interrupted test reached no verdict, so it has no failure to tell. */ function pagedResults(report: Report1Document): ReportResult[] { const explore = report.run.explore; @@ -571,7 +592,7 @@ function pagedResults(report: Report1Document): ReportResult[] { * `summary.md` beside `report.json`, with evidence listed as paths from the * project root, for a reader with the checkout in front of it: a pull * request description, a coding agent's handoff, a wiki page. Every test - * that did not pass gets a page of its own under `failures/`, with the + * that failed or was flaky gets a page of its own under `failures/`, with the * screen at failure inline; the run page links each block to its page. The * directory is the reporter's: what an earlier run left there is removed * first, so a stale page never describes a failure this run did not have. diff --git a/packages/e2e/src/report/repeats.ts b/packages/e2e/src/report/repeats.ts index dc6b8f192..7d4448c71 100644 --- a/packages/e2e/src/report/repeats.ts +++ b/packages/e2e/src/report/repeats.ts @@ -17,8 +17,16 @@ export interface RepeatGroup { readonly runs: readonly RepeatRun[]; /** Runs that passed on their first attempt; a flaky run needed a retry and does not count. */ readonly passed: number; + /** + * Whether a run that reached a verdict did not pass: a flake. A run the + * run interrupted, or never started, says nothing about the test. + */ + readonly unstable: boolean; } +/** Statuses of a run that reached no verdict: stopped by the run, or never started. */ +const UNDECIDED = new Set(['interrupted', 'skipped']); + /** * Groups runs by `key`, keeping only tests that ran more than once and did * not skip every run, in first-seen order. @@ -38,7 +46,12 @@ export function repeatGroups( .filter((group) => group.runs.length > 1 && group.runs.some((run) => run.status !== 'skipped')) .map((group) => { const runs = group.runs.toSorted((a, b) => a.repeat - b.repeat); - return { label: group.label, runs, passed: runs.filter((run) => run.status === 'passed').length }; + return { + label: group.label, + runs, + passed: runs.filter((run) => run.status === 'passed').length, + unstable: runs.some((run) => run.status !== 'passed' && !UNDECIDED.has(run.status)), + }; }); } @@ -49,13 +62,14 @@ export function repeatSummary(groups: readonly RepeatGroup[]): string { return `${stable} of ${groups.length} test${groups.length === 1 ? '' : 's'} passed all ${runs} runs`; } -/** A run that did not pass, in a word: its error code, or `flaky` with the code its retry recovered from. */ +/** A run that did not pass, in a word: its error code, `flaky` with the code its retry recovered from, or why it reached no verdict. */ function outcomeOf(run: RepeatRun): string { if (run.status === 'flaky') return run.code === undefined ? 'flaky' : `flaky (${run.code})`; + if (UNDECIDED.has(run.status)) return run.status; return run.code ?? run.status; } -/** `3/5 passed · repeat 0 ASSERTION_FAILED · repeat 2 flaky`, naming each run that did not pass. */ +/** `3/5 passed · repeat 0 ASSERTION_FAILED · repeat 2 flaky · repeat 4 interrupted`, naming each run that did not pass. */ export function repeatLine(group: RepeatGroup): string { const misses = group.runs .filter((run) => run.status !== 'passed') diff --git a/packages/e2e/src/run/retry.ts b/packages/e2e/src/run/retry.ts index d2990b97c..b363ee1f8 100644 --- a/packages/e2e/src/run/retry.ts +++ b/packages/e2e/src/run/retry.ts @@ -21,7 +21,9 @@ export function isRetryEligible(attempt: RetryAttempt): boolean { * Runs up to `maxAttempts` attempts, stopping on pass, interrupt, or a * failure that does not consume retry budget. `runOnce` returns the * finished attempt, or undefined when the attempt could not start (the - * loop stops and the last observed status stands). + * loop stops and the last observed status stands). An interrupt during a + * retry keeps the verdict the failed attempt before it reached: the test + * did fail, and only its second chance was cut short. */ export async function runWithRetries( maxAttempts: number, @@ -36,7 +38,7 @@ export async function runWithRetries( if (attempt === undefined) break; attemptCount += 1; if (attempt.status === 'passed') return attemptCount > 1 ? 'flaky' : 'passed'; - if (attempt.status === 'interrupted') return 'interrupted'; + if (attempt.status === 'interrupted') return attemptCount > 1 ? finalStatus : 'interrupted'; // A body that skipped itself has decided; a retry would only ask again. if (attempt.status === 'skipped') return 'skipped'; finalStatus = attempt.status; diff --git a/packages/e2e/src/run/runner.ts b/packages/e2e/src/run/runner.ts index 31b4f8379..f000bfc74 100644 --- a/packages/e2e/src/run/runner.ts +++ b/packages/e2e/src/run/runner.ts @@ -1044,13 +1044,22 @@ function resultExitCodes(results: readonly ResultRecord[]): number[] { /** Why a reporter's `onRunFinished` was given up on: the message is the stderr line. */ class ReporterAbandoned extends Error {} +/** + * The built-in reporters that write files beside `report.json`. A forced + * interrupt never abandons them: they are local writes, and giving one up + * would leave the previous run's `summary.md` or `junit.xml` beside this + * run's report, where CI reads it as this run's. + */ +const FILE_REPORTERS: ReadonlySet = new Set([STATELESS_REPORTERS.junit, STATELESS_REPORTERS.markdown]); + /** * Awaits every reporter's `onRunFinished` at once, each within `timeoutMs` - * and until a forced interrupt, and collects the summary rows they resolve - * with. Each reporter gets a signal that aborts on either, so a well-behaved - * one cancels its own work and leaves no handle holding the process. A - * reporter that throws, runs out of time, or returns something other than - * rows is one line on stderr; it can never change the run's outcome. + * and, except the built-in file reporters, until a forced interrupt, and + * collects the summary rows they resolve with. Each reporter gets a signal + * that aborts on either, so a well-behaved one cancels its own work and + * leaves no handle holding the process. A reporter that throws, runs out of + * time, or returns something other than rows is one line on stderr; it can + * never change the run's outcome. */ async function runReporters( reporters: readonly Reporter[], @@ -1072,8 +1081,10 @@ async function runReporters( timeoutMs, ); const onForce = (): void => abandon.abort(new ReporterAbandoned('abandoned: the run was forced to stop')); - if (force.aborted) onForce(); - else force.addEventListener('abort', onForce, { once: true }); + if (!FILE_REPORTERS.has(reporter)) { + if (force.aborted) onForce(); + else force.addEventListener('abort', onForce, { once: true }); + } try { const result = await withAbort( () => Promise.resolve(onRunFinished.call(reporter, finished, abandon.signal)), diff --git a/packages/e2e/src/run/serial.ts b/packages/e2e/src/run/serial.ts index ccdbb92bb..49936224e 100644 --- a/packages/e2e/src/run/serial.ts +++ b/packages/e2e/src/run/serial.ts @@ -125,7 +125,9 @@ export async function runSerialUnit( async (attemptIndex) => { const attempt = await runSerialAttempt(host, members, file, attemptIndex); group.attempts.push(attempt); - for (const member of attempt.members) memberFinalStatus.set(member.testId, member); + // A retry the run interrupted keeps the verdict before it, so each member keeps what that attempt said. + const cutRetry = attemptIndex > 0 && attempt.status === 'interrupted'; + if (!cutRetry) for (const member of attempt.members) memberFinalStatus.set(member.testId, member); // A beforeAll failure is not retry-eligible: the // attempt stands as recorded and the retry loop stops here. if (attempt.error?.code === 'HOOK_FAILED') return undefined; diff --git a/packages/e2e/src/telemetry/events.ts b/packages/e2e/src/telemetry/events.ts index 3f1dc0b42..3556f1dc4 100644 --- a/packages/e2e/src/telemetry/events.ts +++ b/packages/e2e/src/telemetry/events.ts @@ -291,6 +291,7 @@ export function runCompletedEvent(report: Report1Document, context: RunContext): tests_executed: run.summary.executed, tests_passed: run.summary.passed, tests_failed: run.summary.failed, + tests_interrupted: run.summary.interrupted, tests_flaky: run.summary.flaky, tests_skipped: run.summary.skipped, attempts_total: attempts.reduce((total, count) => total + count, 0), diff --git a/packages/e2e/tests/helpers/report.ts b/packages/e2e/tests/helpers/report.ts index 201e2c1f0..7cadb1afe 100644 --- a/packages/e2e/tests/helpers/report.ts +++ b/packages/e2e/tests/helpers/report.ts @@ -122,7 +122,7 @@ export function reportDocument(overrides: Partial = {}): serialGroups: [], results: [], errors: [], - summary: { discovered: 0, selected: 0, executed: 0, passed: 0, failed: 0, flaky: 0, skipped: 0 }, + summary: { discovered: 0, selected: 0, executed: 0, passed: 0, failed: 0, interrupted: 0, flaky: 0, skipped: 0 }, limits: LIMITS, usage: { discoveredResults: 0, diff --git a/packages/e2e/tests/helpers/sample-report.ts b/packages/e2e/tests/helpers/sample-report.ts index 0d152b781..9c87c211f 100644 --- a/packages/e2e/tests/helpers/sample-report.ts +++ b/packages/e2e/tests/helpers/sample-report.ts @@ -158,7 +158,7 @@ export function sampleReport(): Report1Document { phase: 'launch', }), ], - summary: { discovered: 3, selected: 2, executed: 2, passed: 1, failed: 1, flaky: 0, skipped: 1 }, + summary: { discovered: 3, selected: 2, executed: 2, passed: 1, failed: 1, interrupted: 0, flaky: 0, skipped: 0 }, usage: { discoveredResults: 3, maxAgentContextBytes: 1, diff --git a/packages/e2e/tests/integration/runner-lifecycle.test.ts b/packages/e2e/tests/integration/runner-lifecycle.test.ts index fc7fb5568..2b8a47e59 100644 --- a/packages/e2e/tests/integration/runner-lifecycle.test.ts +++ b/packages/e2e/tests/integration/runner-lifecycle.test.ts @@ -1,4 +1,4 @@ -import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts'; @@ -890,6 +890,8 @@ test('never started either', async () => {}); expect(interrupted.status).toBe('interrupted'); expect(planned).toBe(3); expect(interrupted.report.run.summary.discovered).toBe(planned); + // The test the interrupt stopped reached no verdict: it is not a failure. + expect(interrupted.report.run.summary).toMatchObject({ selected: 3, passed: 0, failed: 0, interrupted: 1, skipped: 2 }); for (const title of ['never started', 'never started either']) { const result = resultByTitle(interrupted, title); expect(result.status).toBe('skipped'); @@ -912,6 +914,86 @@ test('never started either', async () => {}); 120_000, ); + it( + 'an interrupt during a serial group retry keeps each member\'s verdict from the attempt before it', + async () => { + const project = createProject({}); + const marker = path.join(project.dir, 'failed-once'); + mkdirSync(path.join(project.dir, 'tests'), { recursive: true }); + writeFileSync( + path.join(project.dir, 'tests', 'flow.e2e.ts'), + `import { existsSync, writeFileSync } from 'node:fs'; +import { test } from 'e2e'; +test.describe('flow', { serial: true }, () => { + test('first step', async () => { + if (!existsSync(${JSON.stringify(marker)})) { + writeFileSync(${JSON.stringify(marker)}, ''); + throw new Error('first attempt fails'); + } + await new Promise((resolve) => setTimeout(resolve, 60_000)); + }); + test('second step', async () => {}); +}); +`, + ); + const controller = new AbortController(); + let starts = 0; + const outcome = await runExisting(project, { + appUrl: app.url, + config: { tests: 'tests/**/*.e2e.ts', retries: 1 }, + runOptions: { + interruptSignal: controller.signal, + onEvent: (event) => { + // A serial member starts once per group attempt: the second start is the retry. + if (event.type === 'test-started' && event.title === 'flow > first step' && (starts += 1) === 2) controller.abort(); + }, + }, + }); + expect(outcome.exitCode).toBe(130); + expect(outcome.report.run.serialGroups[0]?.attempts.map((attempt) => attempt.status)).toEqual(['failed', 'interrupted']); + expect(resultByTitle(outcome, 'first step').status).toBe('failed'); + expect(outcome.report.run.summary).toMatchObject({ failed: 1, interrupted: 0 }); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'a forced interrupt still writes the junit and markdown files, so none of the previous run is left beside the report', + async () => { + const project = createProject({ + 'tests/sleeps.e2e.ts': `import { test } from 'e2e'; +test('sleeps until interrupted', async () => { + await new Promise((resolve) => setTimeout(resolve, 60_000)); +}); +`, + }); + const output = path.join(project.dir, '.e2e'); + mkdirSync(output, { recursive: true }); + writeFileSync(path.join(output, 'summary.md'), '### 🟢 e2e: 3 passed\n'); + writeFileSync(path.join(output, 'junit.xml'), '\n'); + const force = new AbortController(); + const outcome = await runExisting(project, { + appUrl: app.url, + config: { tests: 'tests/**/*.e2e.ts', reporters: ['junit', 'markdown'] }, + runOptions: { + forceSignal: force.signal, + onEvent: (event) => { + if (event.type === 'test-started') force.abort(); + }, + }, + }); + expect(outcome.exitCode).toBe(130); + expect(outcome.report.run.summary).toMatchObject({ failed: 0, interrupted: 1 }); + expect(readFileSync(path.join(output, 'summary.md'), 'utf8')).toMatch(/^### ⏹️ e2e: 1 interrupted\n/u); + expect(readFileSync(path.join(output, 'junit.xml'), 'utf8')).toContain('timed-out'); - expect(xml).toContain('interrupted'); + expect(xml).toContain(''); + }); + + it('tells the failure an interrupted retry was cut short after', () => { + const interrupt = reportError({ category: 'interrupted', code: 'INTERRUPTED', message: 'run interrupted in phase body' }); + const xml = render( + reportDocument({ + results: [ + reportResult({ + status: 'failed', + attempts: [ + reportAttempt({ index: 0, status: 'failed', error: reportError() }), + reportAttempt({ id: 'attempt-2', index: 1, status: 'interrupted', error: interrupt }), + ], + }), + ], + }), + ); + expect(xml).toContain(''); + expect(rootAttributes(xml)).toMatchObject({ failures: '1', errors: '0', skipped: '0' }); + }); + + it('renders an interrupted result as skipped with why, never as a failure or an error', () => { + const xml = render( + reportDocument({ + results: [ + reportResult({ + status: 'interrupted', + attempts: [ + reportAttempt({ + status: 'interrupted', + error: { category: 'interrupted', code: 'INTERRUPTED', message: 'run interrupted in phase body', retryable: false }, + }), + ], + }), + ], + }), + ); + expect(xml).toContain(''); + expect(rootAttributes(xml)).toMatchObject({ tests: '1', failures: '0', errors: '0', skipped: '1' }); }); it('renders skipped results with their reason and no attempt time', () => { diff --git a/packages/e2e/tests/unit/report-list.test.ts b/packages/e2e/tests/unit/report-list.test.ts index e1daee32b..a9a1da396 100644 --- a/packages/e2e/tests/unit/report-list.test.ts +++ b/packages/e2e/tests/unit/report-list.test.ts @@ -572,9 +572,9 @@ describe('ListReporter', () => { it('names the status when a failure recorded no error', () => { const { lines, output } = capture(); const reporter = plainReporter(output); - reporter.handle(finished(result({ status: 'interrupted', attempts: [attempt({ status: 'interrupted' })] }))); + reporter.handle(finished(result({ status: 'timed-out', attempts: [attempt({ status: 'timed-out' })] }))); reporter.handle(runFinished({ status: 'failed', exitCode: 1 })); - expect(lines).toContain('interrupted: no error was recorded'); + expect(lines).toContain('timed-out: no error was recorded'); }); }); @@ -659,13 +659,30 @@ describe('ListReporter', () => { }); describe('summary', () => { - it('counts timed-out and interrupted results as failures', () => { + it('counts timed-out results as failures and interrupted ones on their own', () => { const { lines, output } = capture(); const reporter = plainReporter(output); reporter.handle(finished(result({ status: 'timed-out', id: 'a' }))); - reporter.handle(finished(result({ status: 'interrupted', id: 'b' }))); + reporter.handle(finished(result({ status: 'interrupted', id: 'b', file: 'tests/b.e2e.ts' }))); reporter.handle(runFinished({ status: 'failed', exitCode: 1, reportPath: '.e2e/report.json' })); - expect(lines).toContain(' Tests 2 failed | 0 passed (2)'); + expect(lines).toContain(' Test Files 1 failed | 1 interrupted | 0 passed (2)'); + expect(lines).toContain(' Tests 1 failed | 1 interrupted | 0 passed (2)'); + expect(lines.filter((line) => line.includes('Failed Tests'))).toHaveLength(1); + }); + + it('lists an interrupted test under its file but not among the failures', () => { + const { lines, output } = capture(); + const reporter = plainReporter(output); + reporter.handle(plan([{ file: 'tests/a.e2e.ts', tests: 2 }, { file: 'tests/b.e2e.ts', tests: 1 }])); + reporter.handle(finished(result({ status: 'passed', id: 'a', file: 'tests/a.e2e.ts', title: ['done'] }))); + reporter.handle(finished(result({ status: 'interrupted', id: 'b', file: 'tests/a.e2e.ts', title: ['cut'] }))); + reporter.handle(finished(result({ status: 'passed', id: 'c', file: 'tests/b.e2e.ts' }))); + reporter.handle(runFinished({ status: 'interrupted', exitCode: 130, reportPath: '.e2e/report.json' })); + expect(lines).toContain(' ❯ |chromium| tests/a.e2e.ts (2 tests | 1 interrupted) 240ms'); + expect(lines).toContain(' × cut (interrupted) 120ms'); + expect(lines.join('\n')).not.toContain('Failed Tests'); + expect(lines).toContain(' Test Files 1 interrupted | 1 passed (2)'); + expect(lines).toContain(' Tests 1 interrupted | 2 passed (3)'); }); it('summarizes files, tests, and the report path relative to the project', () => { diff --git a/packages/e2e/tests/unit/report-markdown.test.ts b/packages/e2e/tests/unit/report-markdown.test.ts index cd4aac2f9..7ab2603b1 100644 --- a/packages/e2e/tests/unit/report-markdown.test.ts +++ b/packages/e2e/tests/unit/report-markdown.test.ts @@ -506,6 +506,44 @@ describe('renderMarkdownReport', () => { expect(renderMarkdownReport(page({ status: 'failed', results: [member] }))).toContain('**🔴 step two** \n`tests/example.e2e.ts:3`\n\n**failed**\n'); }); + it('tells a serial member from the attempt that failed it when the interrupted retry skipped it', () => { + const member = named({ title: 'step two', status: 'failed', serialGroupId: 'g1' }); + const memberRecord = (status: 'failed' | 'skipped') => ({ + id: `m-${status}`, + index: 0, + testId: member.testId, + status, + startedAt: REPORT_AT, + durationMs: 40, + steps: status === 'failed' ? [step({ index: 0, label: 'tap Next', status: 'failed' })] : [], + ...(status === 'failed' + ? { error: { category: 'test' as const, code: 'ASSERTION_FAILED', message: 'nope', retryable: false } } + : { skip: { cause: 'serial-predecessor-failed' as const, reason: 'group attempt did not reach this member' } }), + secondaryErrors: [], + }); + const group: ReportSerialGroup = { + id: 'g1', + serialId: 'g1', + declarationIndex: 0, + file: member.file, + source: member.source, + titlePath: ['group'], + targetId: 'web', + platform: 'web', + agent: 'default', + repeat: 0, + memberTestIds: [member.testId], + status: 'failed', + attempts: [ + { ...attempt({ status: 'failed' }), members: [memberRecord('failed')] }, + { ...attempt({ status: 'interrupted', error: { code: 'INTERRUPTED', message: 'run interrupted in phase body' } }), members: [memberRecord('skipped')] }, + ], + }; + const body = renderMarkdownReport(page({ status: 'interrupted', results: [member], serialGroups: [group] })); + expect(body).toContain('**ASSERTION_FAILED** at step 1 of 1: `screen.tap tap Next`'); + expect(body).not.toContain('INTERRUPTED'); + }); + it('counts zero failed attempts for a flaky test a foreign document gives one attempt, never a negative', () => { const oneAttempt = named({ title: 'odd', status: 'flaky', attempts: [attempt({ status: 'passed' })] }); const body = renderMarkdownReport(page({ results: [oneAttempt] })); @@ -717,6 +755,41 @@ describe('renderMarkdownReport evidence paths', () => { }); }); +describe('an interrupted run', () => { + const cut = named({ + title: 'cut short', + status: 'interrupted', + attempts: [attempt({ status: 'interrupted', error: { code: 'INTERRUPTED', message: 'run interrupted in phase body' } })], + }); + + it('counts an interrupted test apart from failures and gives it no failure block', () => { + const body = renderMarkdownReport(page({ status: 'interrupted', results: [passing, cut] })); + expect(body.startsWith('### ⏹️ e2e: 1 interrupted, 1 passed\n')).toBe(true); + expect(body).not.toContain('INTERRUPTED'); + expect(body).toContain('| ⏹️ | cut short |'); + }); + + it('tells the failure an interrupted retry was cut short after', () => { + const retried = named({ + title: 'fails then gets cut', + status: 'failed', + attempts: [ + attempt({ status: 'failed', error: { code: 'ASSERTION_FAILED', message: 'no cart' } }), + attempt({ status: 'interrupted', error: { code: 'INTERRUPTED', message: 'run interrupted in phase body' } }), + ], + }); + const body = renderMarkdownReport(page({ status: 'interrupted', results: [retried] })); + expect(body.startsWith('### 🔴 e2e: 1 failed\n')).toBe(true); + expect(body).toContain('**ASSERTION_FAILED**'); + expect(body).not.toContain('**INTERRUPTED**'); + }); + + it('stays red when the run failed before it was stopped', () => { + const body = renderMarkdownReport(page({ status: 'interrupted', results: [failing, cut] })); + expect(body.startsWith('### 🔴 e2e: 1 failed, 1 interrupted\n')).toBe(true); + }); +}); + describe('markdownReporter', () => { const dirs: string[] = []; afterAll(() => { @@ -773,6 +846,15 @@ describe('markdownReporter', () => { expect(text).toContain('- screenshot `.e2e/artifacts/t/attempt-0/screenshot-1.bin`'); }); + it('writes no page for an interrupted test, which reached no verdict', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'e2e-markdown-')); + dirs.push(root); + const cut = named({ title: 'cut short', status: 'interrupted', attempts: [attempt({ status: 'interrupted' })] }); + const rows = await markdownReporter.onRunFinished!(finished(page({ status: 'interrupted', results: [passing, cut] }), root), new AbortController().signal); + expect(rows).toEqual([{ label: 'Markdown', text: path.join('.e2e', 'summary.md') }]); + expect(readdirSync(path.join(root, '.e2e'))).toEqual(['summary.md']); + }); + it('writes nothing when the report itself was not written', async () => { expect(await markdownReporter.onRunFinished!({ ...finished(page(), '/nowhere'), reportPath: undefined }, new AbortController().signal)).toBeUndefined(); }); diff --git a/packages/e2e/tests/unit/report-repeats.test.ts b/packages/e2e/tests/unit/report-repeats.test.ts index 7bf3bc9af..e87df9957 100644 --- a/packages/e2e/tests/unit/report-repeats.test.ts +++ b/packages/e2e/tests/unit/report-repeats.test.ts @@ -21,14 +21,26 @@ describe('repeat tally', () => { ], (entry) => entry, ); - expect(groups.map((group) => [group.label, group.passed, group.runs.map((entry) => entry.repeat)])).toEqual([ - ['pays', 1, [0, 1, 2]], - ['stable', 3, [0, 1, 2]], + expect(groups.map((group) => [group.label, group.passed, group.unstable, group.runs.map((entry) => entry.repeat)])).toEqual([ + ['pays', 1, true, [0, 1, 2]], + ['stable', 3, false, [0, 1, 2]], ]); expect(repeatSummary(groups)).toBe('1 of 2 tests passed all 3 runs'); expect(repeatLine(groups[0]!)).toBe('1/3 passed · repeat 1 ASSERTION_FAILED · repeat 2 flaky (STEP_TIMEOUT)'); }); + it('counts a run the run interrupted or never started as a miss, not a flake', () => { + const groups = repeatGroups( + [run('cut', 0, 'passed'), run('cut', 1, 'interrupted', 'INTERRUPTED'), run('cut', 2, 'skipped'), run('flake', 0, 'passed'), run('flake', 1, 'failed', 'ASSERTION_FAILED')], + (entry) => entry, + ); + expect(groups.map((group) => [group.label, group.passed, group.unstable])).toEqual([ + ['cut', 1, false], + ['flake', 1, true], + ]); + expect(repeatLine(groups[0]!)).toBe('1/3 passed · repeat 1 interrupted · repeat 2 skipped'); + }); + it('is empty for a run that repeated nothing', () => { expect(repeatGroups([run('a', 0, 'passed'), run('b', 0, 'failed')], (entry) => entry)).toEqual([]); }); diff --git a/packages/e2e/tests/unit/retry.test.ts b/packages/e2e/tests/unit/retry.test.ts index 38d207584..4132a1b44 100644 --- a/packages/e2e/tests/unit/retry.test.ts +++ b/packages/e2e/tests/unit/retry.test.ts @@ -76,6 +76,15 @@ describe('runWithRetries', () => { expect(attempts).toEqual([0]); }); + it('keeps the failed verdict when an interrupt cuts its retry short', async () => { + const status = await runWithRetries(3, liveSignal(), async (index) => + index === 0 ? { status: 'failed', error: error('test') } : { status: 'interrupted' }, + ); + expect(status).toBe('failed'); + const timedOut = await runWithRetries(3, liveSignal(), async (index) => ({ status: index === 0 ? 'timed-out' : 'interrupted' })); + expect(timedOut).toBe('timed-out'); + }); + it('runs nothing when the interrupt signal is already aborted', async () => { const controller = new AbortController(); controller.abort(); diff --git a/packages/e2e/tests/unit/schema-fixtures.test.ts b/packages/e2e/tests/unit/schema-fixtures.test.ts index abf4e0898..c3b6b00f4 100644 --- a/packages/e2e/tests/unit/schema-fixtures.test.ts +++ b/packages/e2e/tests/unit/schema-fixtures.test.ts @@ -72,6 +72,14 @@ describe.each(schemas)('%s schema', (name) => { } }); + it('counts interrupted results in the summary apart from failed ones', () => { + const report = readJson('fixtures', 'report-v1.valid.json') as { run: { summary: Record } }; + expect(validate(report)).toBe(true); + delete report.run.summary['interrupted']; + expect(validate(report)).toBe(false); + expect(validate.errors).toEqual(expect.arrayContaining([expect.objectContaining({ keyword: 'required', params: { missingProperty: 'interrupted' } })])); + }); + it('requires judgment evidence after a model call, while allowing capture failures before one', () => { const report = readJson('fixtures', 'report-v1.valid.json') as { run: { results: { attempts: { steps: { api: string; status: string; metrics: { modelCalls: number }; observationRevision?: string; explanation?: string }[] }[] }[] }; diff --git a/packages/e2e/tests/unit/telemetry-events.test.ts b/packages/e2e/tests/unit/telemetry-events.test.ts index ebc1d3d2f..478fd374e 100644 --- a/packages/e2e/tests/unit/telemetry-events.test.ts +++ b/packages/e2e/tests/unit/telemetry-events.test.ts @@ -98,8 +98,9 @@ describe('telemetry events', () => { tests_executed: 2, tests_passed: 1, tests_failed: 1, + tests_interrupted: 0, tests_flaky: 0, - tests_skipped: 1, + tests_skipped: 0, attempts_total: 2, tests_retried: 0, targets: 2, diff --git a/packages/github/src/merge.ts b/packages/github/src/merge.ts index 4e557b7a6..f7d13dd68 100644 --- a/packages/github/src/merge.ts +++ b/packages/github/src/merge.ts @@ -15,8 +15,8 @@ type ReportResult = ReportRun['results'][number]; type ReportSerialGroup = ReportRun['serialGroups'][number]; type ReportSummary = ReportRun['summary']; -/** Statuses that count as a failure of the run, the ones `--last-failed` runs again. */ -const FAILED_STATUSES = new Set(['failed', 'timed-out', 'interrupted']); +/** Statuses that count as a failure of the run. `--last-failed` also runs an interrupted test again, but that one reached no verdict. */ +const FAILED_STATUSES = new Set(['failed', 'timed-out']); /** * A test per target and agent, whatever its `--repeat-each` run: the identity @@ -30,13 +30,15 @@ function testKey(result: ReportResult): string { /** * The rerun's result with the run before's attempts in front of its own: * the same test's result there, and the further repeats the rerun has no - * result of its own for. A pass after a failure there is flaky. + * result of its own for. A pass after a failure there is flaky; an + * interrupted rerun of a test that failed there keeps that failure. */ function foldRerun(current: ReportResult, history: readonly ReportResult[]): ReportResult { if (history.length === 0) return current; const before = history.toSorted((a, b) => a.repeat - b.repeat); const attempts = [...before.flatMap((result) => result.attempts), ...current.attempts]; - const status = current.status === 'passed' && before.some((result) => FAILED_STATUSES.has(result.status)) ? 'flaky' : current.status; + const failure = before.findLast((result) => FAILED_STATUSES.has(result.status)); + const status = failure === undefined ? current.status : current.status === 'passed' ? 'flaky' : current.status === 'interrupted' ? failure.status : current.status; return { ...current, status, attempts }; } @@ -51,14 +53,16 @@ function foldGroup(current: ReportSerialGroup, before: ReportSerialGroup | undef return { ...current, attempts }; } -/** The counts the way the runner tallies them, over the folded results. */ +/** The counts the way the runner tallies them, over the folded results: the status counts are over the selected ones. */ function summarize(results: readonly ReportResult[], base: ReportSummary): ReportSummary { - const summary: ReportSummary = { ...base, discovered: results.length, selected: 0, executed: 0, passed: 0, failed: 0, flaky: 0, skipped: 0 }; + const summary: ReportSummary = { ...base, discovered: results.length, selected: 0, executed: 0, passed: 0, failed: 0, interrupted: 0, flaky: 0, skipped: 0 }; for (const result of results) { - if (result.selected) summary.selected += 1; if (result.attempts.length > 0 || (result.serialGroupId !== undefined && result.status !== 'skipped')) summary.executed += 1; + if (!result.selected) continue; + summary.selected += 1; if (result.status === 'passed') summary.passed += 1; else if (result.status === 'flaky') summary.flaky += 1; + else if (result.status === 'interrupted') summary.interrupted += 1; else if (FAILED_STATUSES.has(result.status)) summary.failed += 1; else summary.skipped += 1; } diff --git a/packages/github/tests/unit/merge.test.ts b/packages/github/tests/unit/merge.test.ts index e26dc038f..147c40cb7 100644 --- a/packages/github/tests/unit/merge.test.ts +++ b/packages/github/tests/unit/merge.test.ts @@ -90,7 +90,7 @@ describe('foldLastRun', () => { // An explicit skip stays what the run before said it was; a test the run before never had is the rerun's. expect(entries.get('never wanted#0')).toMatchObject({ status: 'skipped', skip: { cause: 'explicit' } }); expect(entries.get('new since#0')).toMatchObject({ status: 'passed', selected: true }); - expect(folded.run.summary).toMatchObject({ discovered: 5, selected: 5, executed: 4, passed: 2, flaky: 1, failed: 1, skipped: 1 }); + expect(folded.run.summary).toMatchObject({ discovered: 5, selected: 5, executed: 4, passed: 2, flaky: 1, failed: 1, interrupted: 0, skipped: 1 }); // The rerun decides the outcome; the start is the run before's, so the duration spans both. expect(folded.run.status).toBe('failed'); expect(folded.run.startedAt).toBe(firstPass.run.startedAt); @@ -169,6 +169,39 @@ describe('foldLastRun', () => { expect(renderMarkdownReport(folded)).toContain('### 🔴 e2e: 1 failed, 1 flaky'); }); + it('counts an interrupted test apart from failures: one that passes on the rerun is not flaky, and a carried one keeps the page from reading green', () => { + const interruptedAttempt = attempt({ status: 'interrupted', error: { category: 'interrupted', code: 'INTERRUPTED', message: 'run interrupted in phase test' } }); + const before = report({ + status: 'interrupted', + results: [ + result({ title: 'A', status: 'interrupted', attempts: [interruptedAttempt] }), + result({ title: 'B', status: 'interrupted', attempts: [interruptedAttempt] }), + ], + }); + const onlyA = report({ + results: [ + result({ title: 'A', status: 'passed', attempts: [attempt()] }), + result({ title: 'B', status: 'skipped', selected: false, skip: { cause: 'filtered', reason: 'title does not match --grep' } }), + ], + }); + const folded = foldLastRun(onlyA, before); + expect(byTitle(folded).get('A#0')).toMatchObject({ status: 'passed' }); + expect(byTitle(folded).get('B#0')).toMatchObject({ status: 'interrupted', selected: true }); + expect(folded.run.summary).toMatchObject({ selected: 2, passed: 1, failed: 0, interrupted: 1, flaky: 0, skipped: 0 }); + expect(folded.run.status).toBe('passed'); + expect(renderMarkdownReport(folded)).toContain('### ⏹️ e2e: 1 interrupted, 1 passed'); + }); + + it('keeps the first pass\'s failure when the rerun of that test is interrupted', () => { + const interruptedAttempt = attempt({ status: 'interrupted', error: { category: 'interrupted', code: 'INTERRUPTED', message: 'run interrupted in phase test' } }); + const before = report({ status: 'failed', results: [result({ title: 'A', status: 'failed', attempts: [failedAttempt] })] }); + const cut = report({ status: 'interrupted', results: [result({ title: 'A', status: 'interrupted', attempts: [interruptedAttempt] })] }); + const folded = foldLastRun(cut, before); + expect(byTitle(folded).get('A#0')).toMatchObject({ status: 'failed' }); + expect(folded.run.summary).toMatchObject({ failed: 1, interrupted: 0 }); + expect(renderMarkdownReport(folded)).toContain('### 🔴 e2e: 1 failed'); + }); + it('leaves the rerun alone when the run before is another project\'s', () => { const other = report({ projectId: 'dev.example.other', status: 'failed', results: firstPass.run.results }); expect(foldLastRun(rerun, other)).toBe(rerun); diff --git a/skills/e2e/references/running.md b/skills/e2e/references/running.md index e7c788d2b..fe8841fef 100644 --- a/skills/e2e/references/running.md +++ b/skills/e2e/references/running.md @@ -102,11 +102,14 @@ hosted service's video by URL. and `run.results[]`, one per test and target: `titlePath`, `file`, `source`, `tags` (`[]` when none), `agent`, `repeat` (0 unless `--repeat-each`), `selected`, `status`, `attempts[]` of `steps[]`, - `artifacts[]`, `error`. + `artifacts[]`, `error`. `run.summary` counts the selected results: + `passed`, `failed` (timed out included), `interrupted` (stopped by the + run: a signal, `--max-failures`, a run-level error; not a failure), `flaky`, and `skipped` add up + to `selected`; `discovered - selected` were left out. - `junit`: `junit.xml` for CI summaries; `--reporter list,junit` keeps the terminal output. - `markdown` (`--reporter list,markdown`): `summary.md` plus one page per - failed or flaky test under `failures/`. The summary holds counts and + failed or flaky test under `failures/` (an interrupted test gets none). The summary holds counts and spend, a block per failed test (error, facts, failing step, whether every attempt failed alike, last model turns, screen location and closest nodes, the line to look at, evidence paths), the flaky tests folded @@ -142,8 +145,9 @@ interrupted workers keeps the failures' or the error's code. Exit 2 is deterministic, never retry it; only exit 3 is worth a job-level retry. The first SIGINT or SIGTERM interrupts and writes the report if any test -had started; the second forces engine teardown; the third kills the app -process groups and exits 130 at once. +had started; the second forces engine teardown and still writes the +report, plus `junit.xml` and `summary.md` when those reporters are on; the +third kills the app process groups and exits 130 at once. ## Continuous integration From 79931d081a1a687bcd003a70a13da21b905f9011 Mon Sep 17 00:00:00 2001 From: Dery Ferdika <114888155+DeryFerd@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:02:34 +0700 Subject: [PATCH 08/50] fix(cache): treat short prefixed record ids as minted segments (#777) A route segment of letters, a dash, and a digit tail of two or more (`PROJ-016`, `INV-2041`) now abstracts to `:id` like the other minted shapes. These human-readable record ids stayed literals, so a step recorded on one record missed the start-path precondition on the next record and ran live every time. A single trailing digit (`page-2`) stays a route word. Fixes #769 --- .changeset/prefixed-record-ids.md | 5 +++++ packages/e2e/src/cache/route.ts | 9 ++++++--- packages/e2e/tests/unit/trace-route.test.ts | 6 +++++- 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 .changeset/prefixed-record-ids.md diff --git a/.changeset/prefixed-record-ids.md b/.changeset/prefixed-record-ids.md new file mode 100644 index 000000000..0334ec745 --- /dev/null +++ b/.changeset/prefixed-record-ids.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +The replay cache abstracts short prefixed record ids as minted segments. A route segment of letters, a dash, and a digit tail of two or more (`PROJ-016`, `INV-2041`) now reduces to `:id`, so a step recorded on one record replays on the next instead of missing with `wrong-context` and running live every time. A single trailing digit (`page-2`) stays a route word. diff --git a/packages/e2e/src/cache/route.ts b/packages/e2e/src/cache/route.ts index 4805e5e3a..ce8e5b259 100644 --- a/packages/e2e/src/cache/route.ts +++ b/packages/e2e/src/cache/route.ts @@ -38,15 +38,18 @@ const PARAM = ':id'; /** * A segment shaped like a value the app mints per record: a uuid, a hex or * digit run, a long token carrying letters and at least two digits (a route - * word such as `companies-v2` has one), or text with whitespace, which no - * router puts in a path. A capitalized or dashed word is none of these; two - * of them differing is left to the screen. + * word such as `companies-v2` has one), a prefixed record id — letters, a + * dash, then a digit tail of two or more (`PROJ-016`, `INV-2041`; a single + * digit as in `page-2` is a route word) — or text with whitespace, which no + * router puts in a path. Any other capitalized or dashed word is none of + * these; two of them differing is left to the screen. */ const MINTED_SEGMENT: readonly RegExp[] = [ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i, /^[0-9a-f]{8,}$/i, /^\d+$/, /^(?=(?:[^\d]*\d){2})(?=.*[A-Za-z])[A-Za-z0-9_-]{12,}$/, + /^[A-Za-z]+-\d{2,}$/, /\s/, ]; diff --git a/packages/e2e/tests/unit/trace-route.test.ts b/packages/e2e/tests/unit/trace-route.test.ts index 2175b5a4a..e104daa53 100644 --- a/packages/e2e/tests/unit/trace-route.test.ts +++ b/packages/e2e/tests/unit/trace-route.test.ts @@ -7,7 +7,7 @@ const key = (location: string) => routeKey(routeOf(location)); const verdict = (recorded: string, live: string) => compareRoutes(routeOf(recorded), routeOf(live)); describe('routeOf', () => { - it('abstracts the ids apps mint: uuids, hex and digit runs, random tokens', () => { + it('abstracts the ids apps mint: uuids, hex and digit runs, random tokens, prefixed record ids', () => { expect(key('/runs/3f2504e0-4f89-11d3-9a0c-0305e82c3301')).toBe('/runs/:id'); expect(key('/projects/a77c12665e90/tests')).toBe('/projects/:id/tests'); expect(key('/orders/42')).toBe('/orders/:id'); @@ -15,6 +15,8 @@ describe('routeOf', () => { expect(key('/blog/2024/09/18/launch-day')).toBe('/blog/:id/:id/:id/launch-day'); expect(key('/dashboard/e2e-45961c4c/projects/0c1d2e3f4a5b')).toBe('/dashboard/:id/projects/:id'); expect(key('/keys/omk_7Qx9Lm2Pz4Rt8Wv1')).toBe('/keys/:id'); + expect(key('/projects/PROJ-016')).toBe('/projects/:id'); + expect(key('/invoices/INV-2041')).toBe('/invoices/:id'); }); it('keeps the words a router owns: versions, locales, dates in words, capitalized and dashed slugs', () => { @@ -23,6 +25,7 @@ describe('routeOf', () => { expect(key('/en-US/settings/general')).toBe('/en-US/settings/general'); expect(key('/Products/Summer-Sneaker')).toBe('/Products/Summer-Sneaker'); expect(key('/users/john_doe')).toBe('/users/john_doe'); + expect(key('/news/page-2')).toBe('/news/page-2'); expect(key('/docs/index.html')).toBe('/docs/index.html'); expect(key('/tags/c++')).toBe('/tags/c++'); expect(key('/shop/%E9%9E%8B')).toBe('/shop/\u978b'); @@ -68,6 +71,7 @@ describe('compareRoutes', () => { expect(verdict('/orders/1', '/orders/2')).toBe('same'); expect(verdict('/companies?search=a', '/companies?search=b#x')).toBe('same'); expect(verdict('Settings', 'Settings')).toBe('same'); + expect(verdict('/projects/PROJ-016', '/projects/PROJ-017')).toBe('same'); }); it('leaves exactly one unexplained segment to the screen, and nothing else', () => { From 37497dc7500fb7f78736dbc7fccd87759df8b172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Kapu=C5=9Bciak?= <39658211+kacperkapusciak@users.noreply.github.com> Date: Sat, 3 Oct 2026 11:02:56 +0200 Subject: [PATCH 09/50] docs: split Writing tests into Goals, Assertions, and Locators pages (#780) --- docs/assertions.mdx | 94 ++++++++++++++++++++ docs/docs.json | 8 +- docs/{writing-tests.mdx => goals.mdx} | 118 ++++---------------------- docs/index.mdx | 8 +- docs/locators.mdx | 64 ++++++++++++++ docs/migrate/cypress.mdx | 4 +- docs/migrate/maestro.mdx | 4 +- docs/migrate/playwright.mdx | 4 +- docs/migrate/selenium.mdx | 4 +- docs/mobile.mdx | 4 +- docs/quickstart.mdx | 4 +- 11 files changed, 199 insertions(+), 117 deletions(-) create mode 100644 docs/assertions.mdx rename docs/{writing-tests.mdx => goals.mdx} (58%) create mode 100644 docs/locators.mdx diff --git a/docs/assertions.mdx b/docs/assertions.mdx new file mode 100644 index 000000000..2ab2d4dcc --- /dev/null +++ b/docs/assertions.mdx @@ -0,0 +1,94 @@ +--- +title: Assertions +description: Check the screen with agent.assert, wait for a condition, and extract data. +--- + +An assertion checks what is on screen. The agent has three judgment calls: +`assert` checks the screen now, `waitFor` waits until a condition holds, and +`extract` reads structured data. When you know the exact value, use an +`expect` [locator assertion](#locator-assertions) instead, with no model call. + +```ts +await agent.assert('the dashboard shows a trial badge'); +``` + +## `agent.assert` + +`assert` checks a statement against the current screen. A false statement +fails with `ASSERTION_FAILED`. If the screen does not provide enough evidence, +it fails with `ASSERTION_INCONCLUSIVE`. + +The judging model sees the statement and the current screen. It does not see +earlier steps or the acting agent's summaries. + +## `agent.waitFor` + +`assert` checks the screen now. Use `waitFor` when the condition may become true +later. It polls the screen and only calls the model when the observation changes. + +```ts +await agent.waitFor('the export finishes and a download link appears', { + interval: 500, + timeout: 120_000, +}); +``` + +## `agent.extract` + +`extract` reads structured data from the screen. Provide a +[Standard Schema](https://standardschema.dev) validator, such as Zod, to define +the result. + +```ts +import { expect } from 'e2e'; +import { z } from 'zod'; + +const data = await agent.extract('every todo title and how many remain', { + schema: z.object({ titles: z.array(z.string()), remaining: z.number().int() }), +}); +expect(data.titles).toContain('Buy milk'); +``` + + +For model-generated descriptions, check the relevant meaning rather than the +entire sentence. Use a [locator assertion](/locators) when the exact text matters. + + +## Judging pixels + +By default, judgments read a redacted text snapshot of the screen. Add +`vision: true` for visual checks, such as charts or layout. Use +`vision: 'only'` to judge the screenshot alone. + +```ts +await agent.assert('the chart trends upward', { vision: true }); +await agent.assert('the search form is not covered by a banner', { vision: 'only' }); +``` + +Images add input tokens. Screenshots are withheld for the rest of an attempt +after a secret is filled. The [agent reference](/reference/agent#vision) +lists the modes and restrictions. + +## Locator assertions + +`expect` on a locator checks an exact value and calls no model. It retries +until the check passes or the assertion timeout runs out. + +```ts +await expect(screen.getByRole('status')).toHaveText('2 remaining'); +await expect(screen.getByRole('heading', 'Dashboard')).toBeVisible(); +``` + +[Locators](/locators) covers queries and actions. The +[expect reference](/reference/expect#locator-matchers) lists every matcher. + +## Next + + + + Exact actions and checks with `screen` and `expect`. + + + Verdicts, exit codes, and what the model sees. + + diff --git a/docs/docs.json b/docs/docs.json index a41744320..ebccf6d81 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -134,7 +134,9 @@ "pages": [ "index", "quickstart", - "writing-tests" + "goals", + "assertions", + "locators" ] }, { @@ -340,6 +342,10 @@ { "source": "/reference/app-web", "destination": "/reference/web" + }, + { + "source": "/writing-tests", + "destination": "/goals" } ] } diff --git a/docs/writing-tests.mdx b/docs/goals.mdx similarity index 58% rename from docs/writing-tests.mdx rename to docs/goals.mdx index 9d2d2e62a..813a40ccb 100644 --- a/docs/writing-tests.mdx +++ b/docs/goals.mdx @@ -1,10 +1,11 @@ --- -title: Writing tests -description: Goals, checks, extraction, and the deterministic APIs. +title: Goals +description: Write test steps as goals with agent.act, and structure the tests around them. --- -Use agent calls for goals and questions about the screen. Use `screen` and -`expect` when you know the exact action or value to check. A test can freely mix both. +Use agent calls for goals and questions about the screen. Use +[locators](/locators) when you know the exact action or value to check. A test +can freely mix both. ```ts title="tests/todos.e2e.ts" import { test, expect } from 'e2e'; @@ -68,60 +69,9 @@ A step can pass, fail, or be blocked by something such as missing credentials. [How agent steps work](/agent-steps#how-a-step-ends) explains the verdicts and exit codes. -## Checks: `assert`, `waitFor`, `extract` - -`assert` checks a statement against the current screen. A false statement -fails with `ASSERTION_FAILED`. If the screen does not provide enough evidence, -it fails with `ASSERTION_INCONCLUSIVE`. - -The judging model sees the statement and the current screen. It does not see -earlier steps or the acting agent's summaries. - -```ts -await agent.assert('the dashboard shows a trial badge'); -``` - -`assert` checks the screen now. Use `waitFor` when the condition may become true -later. It polls the screen and only calls the model when the observation changes. - -```ts -await agent.waitFor('the export finishes and a download link appears', { - interval: 500, - timeout: 120_000, -}); -``` - -`extract` reads structured data from the screen. Provide a -[Standard Schema](https://standardschema.dev) validator, such as Zod, to define -the result. - -```ts -import { z } from 'zod'; - -const data = await agent.extract('every todo title and how many remain', { - schema: z.object({ titles: z.array(z.string()), remaining: z.number().int() }), -}); -expect(data.titles).toContain('Buy milk'); -``` - - -For model-generated descriptions, check the relevant meaning rather than the -entire sentence. Use a locator assertion when the exact text matters. - - -### Judging pixels - -By default, judgments read a redacted text snapshot of the screen. Add -`vision: true` for visual checks, such as charts or layout. Use -`vision: 'only'` to judge the screenshot alone. - -```ts -await agent.assert('the chart trends upward', { vision: true }); -await agent.assert('the search form is not covered by a banner', { vision: 'only' }); -``` - -`act` requests screenshots itself when it needs them, so it has no `vision` -option. On supported engines, it can tap and type at points in the image: +`act` requests screenshots itself when it needs them, so unlike the +[judgments](/assertions#judging-pixels) it has no `vision` option. On +supported engines, it can tap and type at points in the image: ```ts await agent.act('enter the code 3141 on the drawn keypad and press OK'); @@ -131,44 +81,8 @@ Images add input tokens. Screenshots are withheld for the rest of an attempt after a secret is filled. The [agent reference](/reference/agent#vision) lists the modes and restrictions. -## Deterministic APIs: `screen` and `expect` - -Use locators to interact with known controls and check exact values. -An `expect` after `act` checks the exact result. It is also what lets the -[replay cache](/cache) record the step and keep it. - -```ts -// Semantic queries: role and name first, label second, test ID last. -await screen.getByRole('button', 'Sign in').tap(); -await screen.getByLabel('Email').fill('ada@example.test'); -await screen.getByTestId('todo').first().tap(); - -// Scope into a row, gesture like a user: -const row = screen.getByRole('listitem').filter({ hasText: 'Design review' }); -await row.getByRole('button', 'Archive').tap(); -await screen.swipe({ direction: 'up', momentum: 'fast' }); - -// Coordinates, for what the tree does not list: -await screen.getByRole('image', 'Seat map').tap({ position: { x: 120, y: 48 } }); -await screen.swipe({ from: { x: 300, y: 400 }, to: { x: 40, y: 400 } }); - -// Polling assertions: -await expect(screen.getByRole('status')).toHaveText('2 remaining'); -await expect(screen.getByRole('heading', 'Dashboard')).toBeVisible(); -``` - -Actions and single-node reads or assertions fail when a query matches more -than one node. Narrow it with `.filter()`, `.first()`, or `.nth()`. -Locator assertions retry until they pass or time out. Direct reads such as -`textContent()` do not retry a mismatched value. - -Use the `browser` fixture for browser features such as routes, cookies, dialogs, -frames, and downloads; see [Working with the browser](/browser). The -[screen reference](/reference/screen) lists every query and action. - -Share setup and cleanup through `test.extend`. A fixture can create a test -workspace before the test and delete it afterward, including when the test -fails. See [Your own fixtures](/reference/test#your-own-fixtures). +To check what a goal left on screen, follow it with an +[assertion](/assertions) or a [locator check](/locators). ## Tests without a screen @@ -220,6 +134,10 @@ fixtures or hooks. When a flow must span several tests, mark the group `{ serial: true }`. Members share app state, run in order on one worker, and retry as a whole. +Share setup and cleanup through `test.extend`. A fixture can create a test +workspace before the test and delete it afterward, including when the test +fails. See [Your own fixtures](/reference/test#your-own-fixtures). + ## Run one test @@ -247,10 +165,10 @@ The [CLI reference](/reference/cli#e2e-run) lists every flag. ## Next - - Sessions and secrets. + + Check the screen with `assert`, `waitFor`, and `extract`. - - Several agents in one suite, and running a flow as each of them. + + Exact actions and checks with `screen` and `expect`. diff --git a/docs/index.mdx b/docs/index.mdx index bc532bbe3..0a810e814 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -9,9 +9,9 @@ e2e is a next-generation testing framework for web and mobile apps, built to keep up with the speed we ship software today. You choose how agentic each test is: fully deterministic with -[locators](/writing-tests#deterministic-apis-screen-and-expect) and +[locators](/locators) and [assertions](/reference/expect#locator-matchers), fully driven by an agent -from [goals](/writing-tests#goals-agent-act) in natural language, or a mix of +from [goals](/goals) in natural language, or a mix of both in the same test. One prompt for your coding agent, or set up by hand. - - Write goals, check outcomes, and extract data. + + Write test steps as goals in natural language. Let your coding agent write, run, and debug tests. diff --git a/docs/locators.mdx b/docs/locators.mdx new file mode 100644 index 000000000..1cdabe914 --- /dev/null +++ b/docs/locators.mdx @@ -0,0 +1,64 @@ +--- +title: Locators +description: Deterministic actions and checks with screen and expect. +--- + +Use locators to interact with known controls and check exact values. Locator +steps call no model. + +```ts title="tests/todos.e2e.ts" +import { test, expect } from 'e2e'; + +test('adds a todo', async ({ app, screen }) => { + await app.open('/todos'); + await screen.getByLabel('New todo').fill('Buy milk'); + await screen.getByRole('button', 'Add').tap(); + await expect(screen.getByTestId('todo')).toHaveCount(1); +}); +``` + +An `expect` after an [`agent.act` goal](/goals) checks the exact result. It is +also what lets the [replay cache](/cache) record the step and keep it. + +## Queries, actions, and assertions + +```ts +// Semantic queries: role and name first, label second, test ID last. +await screen.getByRole('button', 'Sign in').tap(); +await screen.getByLabel('Email').fill('ada@example.test'); +await screen.getByTestId('todo').first().tap(); + +// Scope into a row, gesture like a user: +const row = screen.getByRole('listitem').filter({ hasText: 'Design review' }); +await row.getByRole('button', 'Archive').tap(); +await screen.swipe({ direction: 'up', momentum: 'fast' }); + +// Coordinates, for what the tree does not list: +await screen.getByRole('image', 'Seat map').tap({ position: { x: 120, y: 48 } }); +await screen.swipe({ from: { x: 300, y: 400 }, to: { x: 40, y: 400 } }); + +// Polling assertions: +await expect(screen.getByRole('status')).toHaveText('2 remaining'); +await expect(screen.getByRole('heading', 'Dashboard')).toBeVisible(); +``` + +Actions and single-node reads or assertions fail when a query matches more +than one node. Narrow it with `.filter()`, `.first()`, or `.nth()`. +Locator assertions retry until they pass or time out. Direct reads such as +`textContent()` do not retry a mismatched value. + +Use the `browser` fixture for browser features such as routes, cookies, dialogs, +frames, and downloads; see [Working with the browser](/browser). The +[screen reference](/reference/screen) lists every query and action, and the +[expect reference](/reference/expect#locator-matchers) lists every matcher. + +## Next + + + + Sessions and secrets. + + + Several agents in one suite, and running a flow as each of them. + + diff --git a/docs/migrate/cypress.mdx b/docs/migrate/cypress.mdx index 8c3d0f700..686a21cc4 100644 --- a/docs/migrate/cypress.mdx +++ b/docs/migrate/cypress.mdx @@ -115,8 +115,8 @@ Start with a spec that often breaks and whose fixed `cy.wait` calls can become assertions on the condition the test needs. - - Goals, checks, extraction, and the deterministic APIs. + + Write test steps as goals instead of scripted selectors. Routes, dialogs, downloads, iframes, and page evaluation. diff --git a/docs/migrate/maestro.mdx b/docs/migrate/maestro.mdx index 159ccf120..c293363c7 100644 --- a/docs/migrate/maestro.mdx +++ b/docs/migrate/maestro.mdx @@ -223,7 +223,7 @@ often need updating. Configure your app and devices. - - Goals, checks, extraction, and the deterministic APIs. + + Write test steps as goals instead of scripted selectors. diff --git a/docs/migrate/playwright.mdx b/docs/migrate/playwright.mdx index 1797a3b19..cef738f82 100644 --- a/docs/migrate/playwright.mdx +++ b/docs/migrate/playwright.mdx @@ -420,8 +420,8 @@ Start with a flow whose interaction sequence changes often, such as a wizard or checkout; stable Playwright tests can stay where they are. - - Goals, checks, extraction, and where a deterministic call still belongs. + + Write test steps as goals instead of scripted selectors. Every option and every `browser` method. diff --git a/docs/migrate/selenium.mdx b/docs/migrate/selenium.mdx index 0c5a9ae3a..183a40bc2 100644 --- a/docs/migrate/selenium.mdx +++ b/docs/migrate/selenium.mdx @@ -101,8 +101,8 @@ Start with one page object and the flows that use it; its setup code can become a fixture. - - Goals, checks, extraction, and the deterministic APIs. + + Write test steps as goals instead of scripted selectors. Every query, refinement, action, and read. diff --git a/docs/mobile.mdx b/docs/mobile.mdx index a867013a8..8a7886cec 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -595,7 +595,7 @@ xcrun simctl shutdown && xcrun simctl boot Every option, the `device` fixture, and the tool pack. - - Goals, checks, and the deterministic APIs, on any platform. + + Exact taps, swipes, and checks with `screen` and `expect`, on any platform. diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 0958fd97a..a96a76b63 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -245,8 +245,8 @@ to watch a web test, `--no-cache` to skip replaying cached actions. ## Next - - Write goals, check outcomes, and extract data. + + Write test steps as goals in natural language. Reuse an app login and keep passwords out of model input. From eb7830d0b9423359c1bb4059746894fa083d27de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Kapu=C5=9Bciak?= <39658211+kacperkapusciak@users.noreply.github.com> Date: Sat, 3 Oct 2026 11:33:54 +0200 Subject: [PATCH 10/50] docs(models): add an AI SDK provider picker to the Models page (#781) * docs(models): add an AI SDK provider picker to the Models page Readers did not see that e2e takes a model from any AI SDK provider. The Models page now has a provider dropdown with 51 providers from ai-sdk.dev, and three setup steps that follow the selection: install, environment, and config. The steps are four instances of one snippet component. Mintlify evaluates each snippet export alone, so the instances share the selection through a store on globalThis and React.useSyncExternalStore. Code renders with Mintlify's CodeBlock for syntax highlighting. * docs(models): an unlisted provider's model needs tool calls and images --- docs/models.mdx | 52 ++- docs/reference/config.mdx | 5 +- docs/snippets/model-provider.jsx | 755 +++++++++++++++++++++++++++++++ docs/style.css | 120 +++++ skills/e2e/references/agent.md | 12 +- 5 files changed, 933 insertions(+), 11 deletions(-) create mode 100644 docs/snippets/model-provider.jsx diff --git a/docs/models.mdx b/docs/models.mdx index 4d00c3a32..259d1901d 100644 --- a/docs/models.mdx +++ b/docs/models.mdx @@ -3,8 +3,12 @@ title: Models description: Configure a model with your own subscription, an API key, or a local server. --- +import { ModelProvider } from './snippets/model-provider.jsx'; + Agent steps use the model configured in `e2e.config.ts`. You can use an [existing subscription](/subscriptions), an API key, or a local server. +e2e uses the [AI SDK](https://ai-sdk.dev/providers). You can use +[all AI SDK providers that support tool calls](#ai-sdk-providers). `e2e init` offers these choices and writes the config for you. For the first setup, follow the [quickstart](/quickstart#use-a-subscription-or-api-key). @@ -77,7 +81,8 @@ const agent = { model: openai('gpt-6-luna') }; ``` Use `agent` as `agents.default`. Other AI SDK providers work the same way, -with their own package, model IDs, and authentication settings. +with their own package, model IDs, and authentication settings. See +[AI SDK providers](#ai-sdk-providers). @@ -103,11 +108,6 @@ Use `agent` as `agents.default`. Add `apiKey` if the server requires one. Model IDs depend on the provider. Gateways usually use `provider/model`; direct providers use their own model IDs. -DeepSeek's V4 models (`@ai-sdk/deepseek`) think by default, and in that mode -they refuse the forced tool choice agent steps ask for. The step is sent again -with `auto` and an instruction to reply with tool calls only, and later steps -on that model start that way, so thinking can stay on. - ### Vercel OIDC instead of a key Without `AI_GATEWAY_API_KEY`, `gateway()` authenticates with a Vercel OIDC @@ -145,6 +145,46 @@ An OIDC failure is the same `MODEL_PROVIDER_FAILED` as a bad key, and the AI SDK's message names only `AI_GATEWAY_API_KEY`. With no key set, check that the Vercel CLI is signed in and that `.vercel/project.json` exists. +## AI SDK providers + +Select a provider. The three steps below show the setup for that provider. + + + +The list shows the AI SDK providers that support tool calls. A provider that +is not in the list also works if it uses the AI SDK language model +specification v2 or later. Its model must support tool calls and images. + +### Install the provider package + +Install `ai` and the provider package. + + + +### Set the credentials + +The provider reads its credentials from the environment. e2e does not load +`.env` files. Export the variables in your shell or CI, or load the file in +the config with [`process.loadEnvFile()`](/reference/config#loading). + + + +If a key is missing or wrong, the first agent step fails with +`MODEL_PROVIDER_FAILED`. Some providers read the key when the config loads. +For these providers, a missing key stops all runs with `CONFIG_LOAD_FAILED`. + +### Add the model to the config + +Set the model in `agents.default`. The model ID is an example. Use a model +that supports tool calls and images. + + + +DeepSeek's V4 models (`@ai-sdk/deepseek`) think by default, and in that mode +they refuse the forced tool choice agent steps ask for. The step is sent again +with `auto` and an instruction to reply with tool calls only, and later steps +on that model start that way, so thinking can stay on. + ## Several models in one config Give agents different names to use different models: diff --git a/docs/reference/config.mdx b/docs/reference/config.mdx index 8321f0a38..1a5b5d833 100644 --- a/docs/reference/config.mdx +++ b/docs/reference/config.mdx @@ -754,8 +754,9 @@ export interface ModelInstance { } ``` -Construct the model in your config with an AI SDK provider. For subscription -sign-in, use the [`e2e/oauth/*` models](/subscriptions). +Construct the model in your config with an AI SDK provider. You can use all +providers that support tool calls, see [AI SDK providers](/models#ai-sdk-providers). +For subscription sign-in, use the [`e2e/oauth/*` models](/subscriptions). | Gateway | Constructor | Package | Reads | | --- | --- | --- | --- | diff --git a/docs/snippets/model-provider.jsx b/docs/snippets/model-provider.jsx new file mode 100644 index 000000000..f903eefe0 --- /dev/null +++ b/docs/snippets/model-provider.jsx @@ -0,0 +1,755 @@ +/** + * One part of the provider setup on the Models page: `show` is `select`, + * `install`, `env`, or `config`. All parts on the page share one selection. + * + * Mintlify evaluates each snippet export alone, so exports share no module + * scope. For this reason, one component takes `show`, and the selection is a + * store on `globalThis` read with `React.useSyncExternalStore`. The server + * snapshot is the default, and only event handlers write the store. + * + * The list comes from https://ai-sdk.dev/providers. It excludes providers + * without a language model, models without tool calls (Perplexity, + * llama.cpp, QuiverAI), agent CLIs with their own tool loop (Claude Code, + * Codex CLI, Gemini CLI, OpenCode, ACP, A2A), browser- and device-only + * runtimes, packages on the v1 model specification, and packages that do not + * load as ESM or do not typecheck (Apertis 3.0.0, Requesty 3.6.2). + * `readsAtLoad` marks providers that throw at construction when their + * variables are missing. + */ +export const ModelProvider = ({ show }) => { + const providers = [ + { + id: 'gateway', + name: 'Vercel AI Gateway', + group: 'AI SDK', + packages: [], + code: "import { gateway } from 'ai';", + model: "gateway('openai/gpt-6-luna-fast')", + env: ['AI_GATEWAY_API_KEY'], + note: 'You can use a Vercel OIDC token from a linked project instead of a key.', + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/ai-gateway', + }, + { + id: 'openai', + name: 'OpenAI', + group: 'AI SDK', + packages: ['@ai-sdk/openai'], + code: "import { openai } from '@ai-sdk/openai';", + model: "openai('gpt-6-luna')", + env: ['OPENAI_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/openai', + }, + { + id: 'anthropic', + name: 'Anthropic', + group: 'AI SDK', + packages: ['@ai-sdk/anthropic'], + code: "import { anthropic } from '@ai-sdk/anthropic';", + model: "anthropic('claude-sonnet-5-5')", + env: ['ANTHROPIC_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/anthropic', + }, + { + id: 'google', + name: 'Google Generative AI', + group: 'AI SDK', + packages: ['@ai-sdk/google'], + code: "import { google } from '@ai-sdk/google';", + model: "google('gemini-3.8-flash')", + env: ['GOOGLE_GENERATIVE_AI_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/google', + }, + { + id: 'google-vertex', + name: 'Google Vertex AI', + group: 'AI SDK', + packages: ['@ai-sdk/google-vertex'], + code: "import { googleVertex } from '@ai-sdk/google-vertex';", + model: "googleVertex('gemini-3.8-flash')", + env: ['GOOGLE_VERTEX_PROJECT', 'GOOGLE_VERTEX_LOCATION', 'GOOGLE_APPLICATION_CREDENTIALS'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/google-vertex', + readsAtLoad: true, + }, + { + id: 'azure', + name: 'Azure OpenAI', + group: 'AI SDK', + packages: ['@ai-sdk/azure'], + code: "import { azure } from '@ai-sdk/azure';", + model: "azure('your-deployment-name')", + env: ['AZURE_RESOURCE_NAME', 'AZURE_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/azure', + }, + { + id: 'amazon-bedrock', + name: 'Amazon Bedrock', + group: 'AI SDK', + packages: ['@ai-sdk/amazon-bedrock'], + code: "import { amazonBedrock } from '@ai-sdk/amazon-bedrock';", + model: "amazonBedrock('eu.anthropic.claude-sonnet-4-6')", + env: ['AWS_REGION', 'AWS_BEARER_TOKEN_BEDROCK'], + note: 'You can use AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY instead of the bearer token. The eu. model prefix needs an EU region.', + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/amazon-bedrock', + }, + { + id: 'anthropic-aws', + name: 'Claude Platform on AWS', + group: 'AI SDK', + packages: ['@ai-sdk/anthropic-aws'], + code: "import { anthropicAws } from '@ai-sdk/anthropic-aws';", + model: "anthropicAws('claude-sonnet-5-5')", + env: ['AWS_REGION', 'ANTHROPIC_AWS_WORKSPACE_ID', 'ANTHROPIC_AWS_API_KEY'], + note: 'You can use AWS credentials instead of the API key.', + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/anthropic-aws', + readsAtLoad: true, + }, + { + id: 'xai', + name: 'xAI Grok', + group: 'AI SDK', + packages: ['@ai-sdk/xai'], + code: "import { xai } from '@ai-sdk/xai';", + model: "xai('grok-4.7')", + env: ['XAI_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/xai', + }, + { + id: 'mistral', + name: 'Mistral AI', + group: 'AI SDK', + packages: ['@ai-sdk/mistral'], + code: "import { mistral } from '@ai-sdk/mistral';", + model: "mistral('mistral-medium-3.5')", + env: ['MISTRAL_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/mistral', + }, + { + id: 'deepseek', + name: 'DeepSeek', + group: 'AI SDK', + packages: ['@ai-sdk/deepseek'], + code: "import { deepSeek } from '@ai-sdk/deepseek';", + model: "deepSeek('deepseek-v4-flash')", + env: ['DEEPSEEK_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/deepseek', + }, + { + id: 'moonshotai', + name: 'Moonshot AI', + group: 'AI SDK', + packages: ['@ai-sdk/moonshotai'], + code: "import { moonshotai } from '@ai-sdk/moonshotai';", + model: "moonshotai('kimi-k3')", + env: ['MOONSHOT_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/moonshotai', + }, + { + id: 'alibaba', + name: 'Alibaba', + group: 'AI SDK', + packages: ['@ai-sdk/alibaba'], + code: "import { alibaba } from '@ai-sdk/alibaba';", + model: "alibaba('qwen3-max')", + env: ['ALIBABA_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/alibaba', + }, + { + id: 'zai', + name: 'Z.AI', + group: 'AI SDK', + packages: ['@ai-sdk/zai'], + code: "import { zai } from '@ai-sdk/zai';", + model: "zai('glm-5.3')", + env: ['ZAI_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/zai', + }, + { + id: 'minimax', + name: 'MiniMax', + group: 'AI SDK', + packages: ['@ai-sdk/minimax'], + code: "import { minimax } from '@ai-sdk/minimax';", + model: "minimax('minimax-m3')", + env: ['MINIMAX_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/minimax', + }, + { + id: 'groq', + name: 'Groq', + group: 'AI SDK', + packages: ['@ai-sdk/groq'], + code: "import { groq } from '@ai-sdk/groq';", + model: "groq('meta-llama/llama-4-scout-17b-16e-instruct')", + env: ['GROQ_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/groq', + }, + { + id: 'cerebras', + name: 'Cerebras', + group: 'AI SDK', + packages: ['@ai-sdk/cerebras'], + code: "import { cerebras } from '@ai-sdk/cerebras';", + model: "cerebras('gemma-4-31b')", + env: ['CEREBRAS_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/cerebras', + }, + { + id: 'fireworks', + name: 'Fireworks', + group: 'AI SDK', + packages: ['@ai-sdk/fireworks'], + code: "import { fireworks } from '@ai-sdk/fireworks';", + model: "fireworks('accounts/fireworks/models/kimi-k2p6')", + env: ['FIREWORKS_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/fireworks', + }, + { + id: 'togetherai', + name: 'Together.ai', + group: 'AI SDK', + packages: ['@ai-sdk/togetherai'], + code: "import { togetherai } from '@ai-sdk/togetherai';", + model: "togetherai('mistralai/Mixtral-8x22B-Instruct-v0.1')", + env: ['TOGETHER_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/togetherai', + }, + { + id: 'deepinfra', + name: 'DeepInfra', + group: 'AI SDK', + packages: ['@ai-sdk/deepinfra'], + code: "import { deepInfra } from '@ai-sdk/deepinfra';", + model: "deepInfra('meta-llama/Llama-3.3-70B-Instruct')", + env: ['DEEPINFRA_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/deepinfra', + }, + { + id: 'cohere', + name: 'Cohere', + group: 'AI SDK', + packages: ['@ai-sdk/cohere'], + code: "import { cohere } from '@ai-sdk/cohere';", + model: "cohere('command-a-03-2025')", + env: ['COHERE_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/cohere', + }, + { + id: 'huggingface', + name: 'Hugging Face', + group: 'AI SDK', + packages: ['@ai-sdk/huggingface'], + code: "import { huggingFace } from '@ai-sdk/huggingface';", + model: "huggingFace('Qwen/Qwen2.5-VL-7B-Instruct')", + env: ['HUGGINGFACE_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/huggingface', + }, + { + id: 'baseten', + name: 'Baseten', + group: 'AI SDK', + packages: ['@ai-sdk/baseten'], + code: "import { baseten } from '@ai-sdk/baseten';", + model: "baseten('moonshotai/Kimi-K2-Instruct-0905')", + env: ['BASETEN_API_KEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/baseten', + }, + { + id: 'gmicloud', + name: 'GMI Cloud', + group: 'AI SDK', + packages: ['@ai-sdk/gmicloud'], + code: "import { gmicloud } from '@ai-sdk/gmicloud';", + model: "gmicloud('deepseek-ai/DeepSeek-V4-Flash-0731')", + env: ['GMI_CLOUD_APIKEY'], + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/gmicloud', + }, + { + id: 'open-responses', + name: 'Open Responses endpoint', + group: 'AI SDK', + packages: ['@ai-sdk/open-responses'], + code: `import { createOpenResponses } from '@ai-sdk/open-responses'; + +const local = createOpenResponses({ + name: 'local', + url: 'http://localhost:1234/v1/responses', +});`, + model: "local('your-model-id')", + env: [], + note: 'Use a server that has the Responses API. Add apiKey if the server needs a key.', + docs: 'https://ai-sdk.dev/providers/ai-sdk-providers/open-responses', + }, + { + id: 'openrouter', + name: 'OpenRouter', + group: 'Community', + packages: ['@openrouter/ai-sdk-provider'], + code: "import { openrouter } from '@openrouter/ai-sdk-provider';", + model: "openrouter('openai/gpt-6-luna-fast')", + env: ['OPENROUTER_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/openrouter', + }, + { + id: 'ollama', + name: 'Ollama', + group: 'Community', + packages: ['ollama-ai-provider-v2'], + code: "import { ollama } from 'ollama-ai-provider-v2';", + model: "ollama('qwen3:4b')", + env: [], + note: 'Start Ollama on its default port.', + docs: 'https://ai-sdk.dev/providers/community-providers/ollama', + }, + { + id: 'helicone', + name: 'Helicone', + group: 'Community', + packages: ['@helicone/ai-sdk-provider'], + code: `import { createHelicone } from '@helicone/ai-sdk-provider'; + +const helicone = createHelicone({ apiKey: process.env.HELICONE_API_KEY });`, + model: "helicone('claude-4.5-haiku')", + env: ['HELICONE_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/helicone', + }, + { + id: 'neon', + name: 'Neon AI Gateway', + group: 'Community', + packages: ['@neon/ai-sdk-provider'], + code: "import { neon } from '@neon/ai-sdk-provider';", + model: "neon('gpt-5-mini')", + env: ['NEON_AI_GATEWAY_BASE_URL', 'NEON_AI_GATEWAY_TOKEN'], + docs: 'https://ai-sdk.dev/providers/community-providers/neon-ai-gateway', + }, + { + id: 'aihubmix', + name: 'AIHubMix', + group: 'Community', + packages: ['@aihubmix/ai-sdk-provider'], + code: "import { aihubmix } from '@aihubmix/ai-sdk-provider';", + model: "aihubmix('gemini-2.5-flash')", + env: ['AIHUBMIX_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/aihubmix', + readsAtLoad: true, + }, + { + id: 'aimlapi', + name: 'AI/ML API', + group: 'Community', + packages: ['@ai-ml.api/aimlapi-vercel-ai'], + code: "import { aimlapi } from '@ai-ml.api/aimlapi-vercel-ai';", + model: "aimlapi('gpt-4o')", + env: ['AIMLAPI_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/aimlapi', + }, + { + id: 'cencori', + name: 'Cencori', + group: 'Community', + packages: ['cencori'], + code: "import { cencori } from 'cencori/vercel';", + model: "cencori('gpt-4o')", + env: ['CENCORI_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/cencori', + readsAtLoad: true, + }, + { + id: 'crusoe', + name: 'Crusoe', + group: 'Community', + packages: ['crusoe-ai-provider'], + code: "import { crusoe } from 'crusoe-ai-provider';", + model: "crusoe('zai/GLM-5.2')", + env: ['CRUSOE_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/crusoe', + }, + { + id: 'friendliai', + name: 'FriendliAI', + group: 'Community', + packages: ['@friendliai/ai-provider'], + code: "import { friendli } from '@friendliai/ai-provider';", + model: "friendli('meta-llama-3.3-70b-instruct')", + env: ['FRIENDLI_TOKEN'], + docs: 'https://ai-sdk.dev/providers/community-providers/friendliai', + }, + { + id: 'runpod', + name: 'Runpod', + group: 'Community', + packages: ['@runpod/ai-sdk-provider'], + code: "import { runpod } from '@runpod/ai-sdk-provider';", + model: "runpod('qwen/qwen3-32b-awq')", + env: ['RUNPOD_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/runpod', + }, + { + id: 'sambanova', + name: 'SambaNova', + group: 'Community', + packages: ['sambanova-ai-provider'], + code: "import { sambanova } from 'sambanova-ai-provider';", + model: "sambanova('Meta-Llama-3.1-70B-Instruct')", + env: ['SAMBANOVA_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/sambanova', + }, + { + id: 'telnyx', + name: 'Telnyx', + group: 'Community', + packages: ['@telnyx/ai-sdk-provider'], + code: "import { telnyx } from '@telnyx/ai-sdk-provider';", + model: "telnyx('Qwen/Qwen3-235B-A22B')", + env: ['TELNYX_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/telnyx', + readsAtLoad: true, + }, + { + id: 'zhipu', + name: 'Zhipu AI (BigModel)', + group: 'Community', + packages: ['zhipu-ai-provider'], + code: "import { zhipu } from 'zhipu-ai-provider';", + model: "zhipu('glm-4-plus')", + env: ['ZHIPU_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/zhipu', + }, + { + id: 'sap-ai', + name: 'SAP AI Core', + group: 'Community', + packages: ['@jerome-benoit/sap-ai-provider'], + code: "import { sapai } from '@jerome-benoit/sap-ai-provider';", + model: "sapai('gpt-4.1')", + env: ['AICORE_SERVICE_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/sap-ai', + }, + { + id: 'azure-ai', + name: 'Azure AI Foundry', + group: 'Community', + packages: ['@quail-ai/azure-ai-provider'], + code: `import { createAzure } from '@quail-ai/azure-ai-provider'; + +const azure = createAzure({ + endpoint: process.env.AZURE_API_ENDPOINT, + apiKey: process.env.AZURE_API_KEY, +});`, + model: "azure('your-deployment-name')", + env: ['AZURE_API_ENDPOINT', 'AZURE_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/azure-ai', + readsAtLoad: true, + }, + { + id: 'interfaze', + name: 'Interfaze', + group: 'Community', + packages: ['@interfaze-ai/ai-sdk'], + code: "import { interfaze } from '@interfaze-ai/ai-sdk';", + model: "interfaze('interfaze-beta')", + env: ['INTERFAZE_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/interfaze', + }, + { + id: 'ollm', + name: 'OLLM', + group: 'Community', + packages: ['@ofoundation/ollm'], + code: `import { createOLLM } from '@ofoundation/ollm'; + +const ollm = createOLLM({ apiKey: process.env.OLLM_API_KEY });`, + model: "ollm.chatModel('near/GLM-4.6')", + env: ['OLLM_API_KEY'], + docs: 'https://ai-sdk.dev/providers/community-providers/ollm', + }, + { + id: 'qvac', + name: 'QVAC', + group: 'Community', + packages: ['@qvac/ai-sdk-provider'], + code: `import { createQvac } from '@qvac/ai-sdk-provider'; + +const qvac = createQvac({ + baseURL: 'http://127.0.0.1:11434/v1', + apiKey: 'qvac', +});`, + model: "qvac('qwen3.5-0.8b')", + env: [], + note: 'Start qvac serve. Set baseURL to its port.', + docs: 'https://ai-sdk.dev/providers/community-providers/qvac', + }, + { + id: 'openai-compatible', + name: 'Any OpenAI-compatible endpoint', + group: 'OpenAI-compatible', + packages: ['@ai-sdk/openai-compatible'], + code: `import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; + +const local = createOpenAICompatible({ + name: 'local', + baseURL: 'http://127.0.0.1:11434/v1', + // apiKey: process.env.LLM_API_KEY, +});`, + model: "local.chatModel('your-model-id')", + env: [], + note: 'Use vLLM, llama-server, LiteLLM, or a different server with /v1/chat/completions.', + docs: 'https://ai-sdk.dev/providers/openai-compatible-providers', + }, + { + id: 'lmstudio', + name: 'LM Studio', + group: 'OpenAI-compatible', + packages: ['@ai-sdk/openai-compatible'], + code: `import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; + +const lmstudio = createOpenAICompatible({ + name: 'lmstudio', + baseURL: 'http://localhost:1234/v1', +});`, + model: "lmstudio('llama-3.2-1b')", + env: [], + docs: 'https://ai-sdk.dev/providers/openai-compatible-providers/lmstudio', + }, + { + id: 'nim', + name: 'NVIDIA NIM', + group: 'OpenAI-compatible', + packages: ['@ai-sdk/openai-compatible'], + code: `import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; + +const nim = createOpenAICompatible({ + name: 'nim', + baseURL: 'https://integrate.api.nvidia.com/v1', + headers: { Authorization: \`Bearer \${process.env.NIM_API_KEY}\` }, +});`, + model: "nim.chatModel('meta/llama-3.3-70b-instruct')", + env: ['NIM_API_KEY'], + docs: 'https://ai-sdk.dev/providers/openai-compatible-providers/nim', + }, + { + id: 'heroku', + name: 'Heroku', + group: 'OpenAI-compatible', + packages: ['@ai-sdk/openai-compatible'], + code: `import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; + +const heroku = createOpenAICompatible({ + name: 'heroku', + baseURL: process.env.INFERENCE_URL + '/v1', + apiKey: process.env.INFERENCE_KEY, +});`, + model: "heroku('claude-3-5-haiku')", + env: ['INFERENCE_URL', 'INFERENCE_KEY'], + docs: 'https://ai-sdk.dev/providers/openai-compatible-providers/heroku', + }, + { + id: 'clarifai', + name: 'Clarifai', + group: 'OpenAI-compatible', + packages: ['@ai-sdk/openai-compatible'], + code: `import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; + +const clarifai = createOpenAICompatible({ + name: 'clarifai', + baseURL: 'https://api.clarifai.com/v2/ext/openai/v1', + apiKey: process.env.CLARIFAI_PAT, +});`, + model: "clarifai.chatModel('https://clarifai.com/deepseek-ai/deepseek-chat/models/DeepSeek-R1-0528-Qwen3-8B')", + env: ['CLARIFAI_PAT'], + docs: 'https://ai-sdk.dev/providers/openai-compatible-providers/clarifai', + }, + { + id: 'nearai', + name: 'NEAR AI Cloud', + group: 'OpenAI-compatible', + packages: ['@ai-sdk/openai-compatible'], + code: `import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; + +const nearai = createOpenAICompatible({ + name: 'nearai', + baseURL: 'https://cloud-api.near.ai/v1', + apiKey: process.env.NEARAI_API_KEY, +});`, + model: "nearai('zai-org/GLM-5.1-FP8')", + env: ['NEARAI_API_KEY'], + docs: 'https://ai-sdk.dev/providers/openai-compatible-providers/nearai', + }, + { + id: 'modelrush', + name: 'ModelRush', + group: 'OpenAI-compatible', + packages: ['@ai-sdk/openai-compatible'], + code: `import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; + +const modelrush = createOpenAICompatible({ + name: 'modelrush', + baseURL: 'https://api.modelrush.ai/v1', + apiKey: process.env.MODELRUSH_API_KEY, +});`, + model: "modelrush.chatModel('modelrush/deepseek-v4-flash-0731')", + env: ['MODELRUSH_API_KEY'], + docs: 'https://ai-sdk.dev/providers/openai-compatible-providers/modelrush', + }, + { + id: 'cheaper-inference', + name: 'Cheaper Inference', + group: 'OpenAI-compatible', + packages: ['@ai-sdk/openai-compatible'], + code: `import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; + +const cheaperInference = createOpenAICompatible({ + name: 'cheaper-inference', + baseURL: 'https://api.cheaperinference.com/v1', + apiKey: process.env.CHEAPER_INFERENCE_API_KEY, +});`, + model: "cheaperInference('gpt-5.4')", + env: ['CHEAPER_INFERENCE_API_KEY'], + docs: 'https://ai-sdk.dev/providers/openai-compatible-providers/cheaper-inference', + }, + ]; + const groups = [ + ['AI SDK', 'AI SDK providers'], + ['Community', 'Community providers'], + ['OpenAI-compatible', 'OpenAI-compatible endpoints'], + ]; + const managers = [ + ['npm', 'npm install -D'], + ['pnpm', 'pnpm add -D'], + ['bun', 'bun add -d'], + ]; + const initial = { providerId: providers[0].id, manager: 'npm' }; + + const store = (globalThis[Symbol.for('e2e.docs.modelProvider')] ??= (() => { + const listeners = new Set(); + const self = { + state: initial, + subscribe(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + update(patch) { + self.state = { ...self.state, ...patch }; + for (const listener of listeners) listener(); + }, + }; + return self; + })()); + const state = React.useSyncExternalStore( + store.subscribe, + () => store.state, + () => initial, + ); + + const provider = providers.find((p) => p.id === state.providerId); + const install = [ + managers.find(([name]) => name === state.manager)[1], + 'ai', + ...provider.packages, + ].join(' '); + const env = provider.env.map((name) => `export ${name}=...`).join('\n'); + const config = `import type { E2EConfig } from 'e2e'; +import { web } from '@e2e-dev/web'; +${provider.code} + +export default { + targets: [{ engine: web(), app: { url: 'http://localhost:3000' } }], + agents: { default: { model: ${provider.model} } }, +} satisfies E2EConfig;`; + + /** Mintlify's own code block: Shiki highlighting in the site theme, and its copy button. */ + const code = (text, language) => {text}; + + const part = (title, body, aside = null) => ( +
+
+ + {title} + · {provider.name} + + {aside} +
+ {body} +
+ ); + + if (show === 'select') { + return ( +
+
+ + + + Provider docs + +
+
+ ); + } + + if (show === 'install') { + return part( + 'Install', + code(install, 'bash'), +
+ {managers.map(([name]) => ( + + ))} +
, + ); + } + + if (show === 'env') { + return part( + 'Environment', + <> + {provider.env.length === 0 ? ( +

No variables.

+ ) : ( + code(env, 'bash') + )} + {provider.readsAtLoad || provider.note ? ( +

+ {[ + provider.readsAtLoad + ? 'This provider reads the variables when the config loads. If one is missing, all runs stop with CONFIG_LOAD_FAILED.' + : null, + provider.note, + ] + .filter(Boolean) + .join(' ')} +

+ ) : null} + , + ); + } + + if (show === 'config') { + return part('e2e.config.ts', code(config, 'ts')); + } + + return null; +}; diff --git a/docs/style.css b/docs/style.css index 3bc34e7d8..b555a7ed6 100644 --- a/docs/style.css +++ b/docs/style.css @@ -348,6 +348,126 @@ html.dark body .code-block-fade-overlay { outline-offset: 3px; } +/* The provider walkthrough on the Models page: separate boxes sharing one + selection (snippets/model-provider.jsx). */ +#content .provider-part { + margin-block: 1.25rem; + border: 1px solid var(--e2e-hairline); + color: var(--e2e-ink); +} + +#content .provider-part-header { + display: flex; + flex-wrap: wrap; + align-items: center; + justify-content: space-between; + gap: 0.75rem; + min-height: 44px; + padding: 0.375rem 1rem; + color: var(--e2e-muted); + font-size: 0.8125rem; +} + +#content .provider-part:not(.provider-part-select) .provider-part-header { + border-bottom: 1px solid var(--e2e-hairline); +} + +#content .provider-part-select .provider-part-header { + padding-block: 0.75rem; +} + +#content .provider-part-provider { + color: var(--e2e-ink); +} + +#content .provider-part-header select { + flex: 1; + min-width: 12rem; + height: 2.25rem; + padding: 0 2rem 0 0.75rem; + border: 1px solid var(--e2e-hairline); + border-radius: 0; + background: var(--e2e-surface) + url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='none' stroke='rgb(115 115 115)' stroke-width='1.5'%3E%3Cpath d='M4 6l4 4 4-4'/%3E%3C/svg%3E") + right 0.625rem center / 14px no-repeat; + color: var(--e2e-ink); + font: inherit; + font-size: 0.875rem; + appearance: none; + cursor: pointer; +} + +#content .provider-part-header select:hover { + border-color: var(--e2e-ink); +} + +#content .provider-part-header option, +#content .provider-part-header optgroup { + background: var(--e2e-surface); + color: var(--e2e-ink); +} + +#content .provider-part-header a { + color: var(--e2e-muted); + text-decoration: underline; + text-underline-offset: 3px; +} + +#content .provider-part-header a:hover { + color: var(--e2e-ink); +} + +#content .provider-part-tabs { + display: flex; +} + +#content .provider-part-tabs button { + padding: 0.125rem 0.5rem; + border: 1px solid var(--e2e-hairline); + color: var(--e2e-muted); + font-size: 0.75rem; + cursor: pointer; +} + +#content .provider-part-tabs button + button { + border-left: 0; +} + +#content .provider-part-tabs button[aria-pressed='true'] { + background: var(--e2e-ink); + border-color: var(--e2e-ink); + color: var(--e2e-surface); +} + +/* Mintlify's CodeBlock inside a part: the part draws the frame and the + header, so the block drops its own margins and border. */ +#content .provider-part .code-block { + margin: 0 !important; + padding: 0 !important; + border: 0 !important; +} + +#content .provider-part-text { + margin: 0; + padding: 0.75rem 1rem; + font-size: 0.875rem; +} + +#content .provider-part-note { + color: var(--e2e-muted); +} + +#content .provider-part .code-block + .provider-part-note { + border-top: 1px solid var(--e2e-hairline); +} + +#content .provider-part button:focus-visible, +#content .provider-part select:focus-visible, +#content .provider-part a:focus-visible { + outline: 2px solid var(--e2e-ink); + outline-offset: 2px; +} + #content .video-card { display: flex; align-items: stretch; diff --git a/skills/e2e/references/agent.md b/skills/e2e/references/agent.md index 3d190d459..2c1837fe3 100644 --- a/skills/e2e/references/agent.md +++ b/skills/e2e/references/agent.md @@ -6,8 +6,13 @@ model-call budget, no shared transcript. No agent step, no model calls. ## Configure a model -Put an AI SDK model under `agents.default`. Vercel AI Gateway reads -`AI_GATEWAY_API_KEY` or, without it, a Vercel OIDC token: +Put an AI SDK model under `agents.default`. You can use all AI SDK providers +that support tool calls and the language model specification v2 or later +(https://ai-sdk.dev/providers). Examples: `@ai-sdk/openai`, +`@ai-sdk/anthropic`, `@ai-sdk/google`, `@ai-sdk/amazon-bedrock`, +`@openrouter/ai-sdk-provider`, `ollama-ai-provider-v2`. For a server with +`/v1/chat/completions`, use `@ai-sdk/openai-compatible`. Vercel AI Gateway +reads `AI_GATEWAY_API_KEY` or, without it, a Vercel OIDC token: ```ts import type { E2EConfig } from 'e2e'; @@ -32,7 +37,8 @@ any provider. - `model` drives `agent.act`. Judgments use `judge` when set, else `model`. - A missing model for the built-in agent raises one run-level `MODEL_UNAVAILABLE` when the first test acquires `agent`, exit 2; auth - failures surface on the first model call as `MODEL_PROVIDER_FAILED`. + failures surface on the first model call as `MODEL_PROVIDER_FAILED`, or as + `CONFIG_LOAD_FAILED` for a provider that reads its key when the config loads. - `context` is what the app calls things, sent to every model call, judges included; `system` is how the acting agent works, read only by the act loop. Both live on the agents entry; `agentContext` on a test or group From bfa58c77c252150f3a786cc4f2111a8a6c8527d4 Mon Sep 17 00:00:00 2001 From: Ibrahim Elkamali Date: Sat, 3 Oct 2026 12:36:01 +0300 Subject: [PATCH 11/50] fix(runner): preserve failures when tests skip (#741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(runner): preserve failures when tests skip * refactor(runner): one skipped-after-failure helper, evidence from failing attempt - failureBeforeSkip returns failing attempt's error, evidence, artifacts; list reporter prints those instead of skipped attempt's - someSkippedAfterFailure indexes serial groups once, shared by exit code and run status - collapse skip branch in execute, place failOnSkippedFailure next to retries with JSDoc --------- Co-authored-by: Oskar Kwaśniewski --- .changeset/quiet-skips-keep-failures.md | 12 ++ docs/reference/config.mdx | 13 ++ docs/reference/expect.mdx | 8 +- docs/reference/test.mdx | 15 +- packages/e2e/src/config/resolve.ts | 10 ++ packages/e2e/src/report/build.ts | 8 +- packages/e2e/src/report/list.ts | 28 ++-- packages/e2e/src/run/execute.ts | 12 +- packages/e2e/src/run/records.ts | 43 ++++++ packages/e2e/src/run/runner.ts | 3 +- packages/e2e/src/types.ts | 2 + .../integration/skip-after-failure.test.ts | 141 ++++++++++++++++++ packages/e2e/tests/types/sdk-types.ts | 1 + packages/e2e/tests/unit/config.test.ts | 11 ++ packages/e2e/tests/unit/report-list.test.ts | 31 ++++ skills/e2e/references/writing-tests.md | 6 +- 16 files changed, 319 insertions(+), 25 deletions(-) create mode 100644 .changeset/quiet-skips-keep-failures.md create mode 100644 packages/e2e/tests/integration/skip-after-failure.test.ts diff --git a/.changeset/quiet-skips-keep-failures.md b/.changeset/quiet-skips-keep-failures.md new file mode 100644 index 000000000..eb2e4b78a --- /dev/null +++ b/.changeset/quiet-skips-keep-failures.md @@ -0,0 +1,12 @@ +--- +"e2e": minor +--- + +Keep soft assertion failures in the report when a test skips itself, and show +them in the CLI under `Skipped After Failure`. A failed attempt followed by a +skip on retry also appears there with its original error. + +Add `failOnSkippedFailure: true` to fail the run with exit code 1 for these +tests. The default is `false`, preserving the existing exit code. Tests retain +their skipped status and reason, teardown still runs, and a skip does not +trigger another retry. diff --git a/docs/reference/config.mdx b/docs/reference/config.mdx index 1a5b5d833..6a0202d9b 100644 --- a/docs/reference/config.mdx +++ b/docs/reference/config.mdx @@ -96,6 +96,8 @@ export interface E2EConfig { cleanupTimeout?: number; /** Retries per test, 0 through 10; default 1 in CI, else 0. */ retries?: number; + /** Fail the run when a test skips itself after a soft failure or an earlier failed attempt; default false. */ + failOnSkippedFailure?: boolean; /** Parallel workers, 1 through 1024; default 1 in CI, else half the cores. An engine may cap it lower. */ workers?: number; /** `{ store }` hands every artifact to a host store as it is produced. */ @@ -158,6 +160,17 @@ Every numeric value must be a safe integer. `projectId` falls back to `package.json`. `workers` never exceeds what a target's engine declares. A targeted agent action is bounded by the smaller of `actionTimeout` and 15 s. +## failOnSkippedFailure + +Set `failOnSkippedFailure: true` to fail the run with exit code 1 when a test +skips after a soft assertion failure or skips a retry after an earlier failed +attempt. The default is `false`. + +The test keeps its `skipped` status and reason. Its failures remain in the +report, and the CLI shows them under `Skipped After Failure` with either +setting. Clean skips, successful retries, and engine cleanup diagnostics do +not trigger this policy. A skip still stops the body and does not retry. + ## targets ```ts diff --git a/docs/reference/expect.mdx b/docs/reference/expect.mdx index a219c779f..adde592fe 100644 --- a/docs/reference/expect.mdx +++ b/docs/reference/expect.mdx @@ -462,11 +462,15 @@ ASSERTION_FAILED: 2 soft assertions failed A soft locator or `browser` matcher is still a recorded `expect.` step, marked failed, so the report shows where in the flow it landed. A soft value matcher is not a step, the same as a hard one; its failure appears in the -final error only. +attempt's error records. When the body throws on its own, or times out, that error stays the attempt's verdict and the soft failures are recorded beside it under -`secondaryErrors`. A body that skips itself with `test.skip(...)` drops them. +`secondaryErrors`. A body that skips itself with `test.skip(...)` keeps them +there too. The CLI shows `Skipped After Failure`; set +[`failOnSkippedFailure: true`](/reference/config) to make that run fail with +exit code 1. Without that option, the test stays skipped and does not affect +the exit code. A soft `toMatchSchema` returns the schema's output when the value matches and `undefined` after a kept failure, and its type says so. diff --git a/docs/reference/test.mdx b/docs/reference/test.mdx index a639604d0..90821715a 100644 --- a/docs/reference/test.mdx +++ b/docs/reference/test.mdx @@ -97,16 +97,23 @@ group titles and then the test's own, are a `COLLECTION_ERROR`. ## Members -A skipped test never affects the exit code. With `test.only`, other tests +A skipped test does not affect the exit code by default. With `test.only`, other tests are reported as `filtered`, and a CI run fails with `ONLY_IN_CI`. Call `test.skip(condition, reason)` inside a test for conditions you learn at runtime, such as a disabled feature flag. When the condition is true, the call throws: the body stops at that line, as Playwright's does, and the test is reported `skipped` with `reason`, with no retry. Nothing after the call runs. -Steps before it stay in the report as they ran, but they do not decide the -status: a failed `expect.soft` before the call is dropped. Hooks and fixtures still run their teardown. When the condition is -a precondition, call it first, before any step. +Steps before it stay in the report as they ran. Soft failures before the call +remain in the attempt's `secondaryErrors`, and the CLI lists the test under +`Skipped After Failure`. A failed attempt followed by a skip on retry appears +there too, with its original error. Hooks and fixtures still run their teardown. +When the condition is a precondition, call it first, before any step. + +Set [`failOnSkippedFailure: true`](/reference/config) to fail the run with exit +code 1 when a runtime skip follows a failure in that test. The test remains +`skipped` with its reason, and the skip does not trigger another retry. Clean +skips and retries that pass are unaffected. The default is `false`. A setup test cannot skip because other tests depend on its sessions; the call fails with `INVALID_ARGUMENT`. Outside a test body, the conditional diff --git a/packages/e2e/src/config/resolve.ts b/packages/e2e/src/config/resolve.ts index 48cc14172..bfa3de39e 100644 --- a/packages/e2e/src/config/resolve.ts +++ b/packages/e2e/src/config/resolve.ts @@ -65,6 +65,7 @@ export interface ResolvedConfig { readonly assertionTimeout: number; readonly cleanupTimeout: number; readonly retries: number; + readonly failOnSkippedFailure: boolean; readonly workers: number; /** Host store every produced artifact is handed to; undefined keeps files local only. */ readonly artifactStore: ArtifactStore | undefined; @@ -154,6 +155,7 @@ const TOP_LEVEL_KEYS = new Set([ 'assertionTimeout', 'cleanupTimeout', 'retries', + 'failOnSkippedFailure', 'workers', 'artifacts', 'output', @@ -266,6 +268,11 @@ export function resolveConfig( boundedInt(raw.workers, 'workers', 1, 1024) ?? (ci ? 1 : Math.max(1, Math.floor(os.availableParallelism() / 2))); + const failOnSkippedFailure = raw.failOnSkippedFailure === undefined ? false : raw.failOnSkippedFailure; + if (typeof failOnSkippedFailure !== 'boolean') { + throw new ConfigurationError('INVALID_CONFIG', 'failOnSkippedFailure must be a boolean'); + } + const artifactStore = resolveArtifactStore(raw); const { reporters, customReporters } = resolveReporters(raw, cli); @@ -291,6 +298,7 @@ export function resolveConfig( assertionTimeout, cleanupTimeout, retries, + failOnSkippedFailure, workers, artifactStore, output, @@ -900,9 +908,11 @@ function computeConfigDigest( // reporter object changes nothing about what a run records, so it never // enters the digest either; the built-in ids digest as they always have, // so adding a reporter to a config leaves its cache valid. + // `failOnSkippedFailure` decides only the exit code, never what runs. // `targets` digest by declaration below and never enter the clone: an // engine holds `secrets.get()` handles, which refuse to serialize. const { + failOnSkippedFailure: _failOnSkippedFailure, artifacts: _artifacts, output: _output, trace: _trace, diff --git a/packages/e2e/src/report/build.ts b/packages/e2e/src/report/build.ts index ea7f2705e..26fdaf7a0 100644 --- a/packages/e2e/src/report/build.ts +++ b/packages/e2e/src/report/build.ts @@ -14,6 +14,7 @@ import { packageVersion } from '../internal/package-version.ts'; import { projectRelativePath } from '../internal/source.ts'; import type { VcsInfo } from '../internal/vcs.ts'; import type { SkipInfo } from '../collect/select.ts'; +import { someSkippedAfterFailure } from '../run/records.ts'; import type { ArtifactRecord, AttemptRecord, @@ -555,7 +556,8 @@ function computeSummary(results: readonly ResultRecord[]): ReportSummary { * nothing contradicts that. One genuine failure keeps the run failed, and so * does any run-level error (a launch that never came up, a cleanup or report * write that failed): it is its own fact about the run, not a blocked step, - * and it must stay visible to a host reading the status. Derivation requires + * and it must stay visible to a host reading the status. So does a skip after + * a failure under `failOnSkippedFailure`. Derivation requires * positive evidence, never absence of it. */ function deriveRunStatus( @@ -563,9 +565,11 @@ function deriveRunStatus( results: readonly ResultRecord[], serialGroups: readonly SerialGroupRecord[], runErrors: readonly RunError[], + failOnSkippedFailure: boolean, ): BuildReportOptions['status'] | 'blocked' { if (status !== 'failed' && status !== 'error') return status; if (runErrors.length > 0) return status; + if (failOnSkippedFailure && someSkippedAfterFailure(results, serialGroups)) return status; // Serial members carry no attempts of their own; their failing error lives // on the group's last attempt (or its failing member). const groupCode = new Map(); @@ -700,7 +704,7 @@ export function buildReport(options: BuildReportOptions): Report1Document { name: 'e2e', version: packageVersion(import.meta.url, '../../package.json', '0.0.0'), }, - status: deriveRunStatus(options.status, options.results, options.serialGroups, options.runErrors), + status: deriveRunStatus(options.status, options.results, options.serialGroups, options.runErrors, config?.failOnSkippedFailure === true), exitCode: options.exitCode, startedAt: options.startedAt, finishedAt: timestamp(), diff --git a/packages/e2e/src/report/list.ts b/packages/e2e/src/report/list.ts index 49e8c2b08..159b7a8f4 100644 --- a/packages/e2e/src/report/list.ts +++ b/packages/e2e/src/report/list.ts @@ -11,7 +11,7 @@ import picocolors from 'picocolors'; import type { SerializedError } from '../internal/errors.ts'; import { packageVersion } from '../internal/package-version.ts'; import type { RunEvent, RunEventFact, RunEventOf, RunEventResult, SetupStep } from '../run/events.ts'; -import type { ArtifactRecord, AttemptRecord, FailureEvidence, ResultStatus, SerialGroupRecord } from '../run/records.ts'; +import { failureBeforeSkip, type ArtifactRecord, type AttemptRecord, type FailureEvidence, type ResultStatus, type SerialGroupRecord } from '../run/records.ts'; import type { Reporter, ReporterSummary } from '../types.ts'; import { codeFrame, userFrame } from './code-frame.ts'; import { @@ -179,7 +179,7 @@ function serialMemberDetails(group: SerialGroupRecord, testId: string): ResultDe }; } -/** One failed pair, held for the `Failed Tests` section. */ +/** One failed pair, held for the `Failed Tests` section, or a skipped one for `Skipped After Failure`. */ interface Failure { readonly group: FileGroup; readonly title: string; @@ -226,6 +226,7 @@ export class ListReporter implements Reporter { /** Whether a live window paints; without one, finished steps stream permanently. */ private readonly live: boolean; private readonly failures: Failure[] = []; + private readonly skippedFailures: Failure[] = []; /** Every finished test's run, for the `Repeats` summary of a `--repeat-each` run. */ private readonly runs: { key: string; label: string; run: RepeatRun }[] = []; /** @@ -650,6 +651,8 @@ export class ListReporter implements Reporter { const steps = this.pairs.get(key)?.steps ?? []; this.pairs.delete(key); const group = this.group(result.test.file, result.target.name); + // Read before `detailsOf` releases the result's serial group. + const skippedAfter = failureBeforeSkip(result, (id) => this.pendingSerial.get(id)?.group); const { durationMs, usage, models, cache, error, videos, failure, screenPath } = this.detailsOf(result); addUsage(this.runUsage, usage); addModelTally(this.runModels, models); @@ -684,6 +687,9 @@ export class ListReporter implements Reporter { if (statusBucket(result.status) === 'failed') { this.failures.push({ group, title, status: result.status, error, videos, failure, screenPath }); } + if (skippedAfter !== undefined) { + this.skippedFailures.push({ group, title, status: result.status, error: skippedAfter.error, videos, ...failureOf(skippedAfter) }); + } if (group.planned !== undefined && group.lines.length >= group.planned) this.printGroup(group); this.window.redraw(); } @@ -820,6 +826,9 @@ export class ListReporter implements Reporter { const cache = cacheText(pc, this.runCache); if (cache !== undefined) rows.push(padTitle(pc, 'Cache') + cache); if (final) rows.push(...this.repeatRows()); + if (this.skippedFailures.length > 0) { + rows.push(padTitle(pc, 'Warnings') + pc.yellow(`${this.skippedFailures.length} skipped after failure`)); + } if (this.errors.length > 0) { const count = this.errors.length; rows.push(padTitle(pc, 'Errors') + pc.bold(pc.red(`${count} error${count === 1 ? '' : 's'}`))); @@ -912,15 +921,15 @@ export class ListReporter implements Reporter { } /** vitest's `Failed Tests` section: a banner, then each failure with its code frame. */ - private printFailures(): void { + private printFailures(heading: string, failures: readonly Failure[]): void { const { pc } = this; - if (this.failures.length === 0) return; + if (failures.length === 0) return; this.print(''); - this.print(this.errorBanner(`Failed Tests ${this.failures.length}`)); + this.print(this.errorBanner(`${heading} ${failures.length}`)); this.print(''); - this.failures.forEach(({ group, title, status, error, videos, failure, screenPath }, index) => { + failures.forEach(({ group, title, status, error, videos, failure, screenPath }, index) => { this.print( - `${pc.bold(pc.bgRed(' FAIL '))} ${this.badge(group.target)} ${bounded(group.file)}${this.separator}${title}`, + `${pc.bold(pc.bgRed(status === 'skipped' ? ' SKIP ' : ' FAIL '))} ${this.badge(group.target)} ${bounded(group.file)}${this.separator}${title}`, ); if (error === undefined) { this.print(pc.red(`${pc.bold(status)}: no error was recorded`)); @@ -932,7 +941,7 @@ export class ListReporter implements Reporter { } this.printEvidence(failure, screenPath); this.printVideos(videos); - const marker = `[${index + 1}/${this.failures.length}]`; + const marker = `[${index + 1}/${failures.length}]`; const { before, after } = rule(marker, 'right'); this.print(''); this.print(pc.red(pc.dim(`${before}${marker}${after}`))); @@ -1029,7 +1038,8 @@ export class ListReporter implements Reporter { if (!group.printed && group.lines.length > 0) this.printGroup(group); } this.printExplore(); - this.printFailures(); + this.printFailures('Failed Tests', this.failures); + this.printFailures('Skipped After Failure', this.skippedFailures); this.printErrors(); this.print(''); for (const row of this.summaryRows(true)) this.print(row); diff --git a/packages/e2e/src/run/execute.ts b/packages/e2e/src/run/execute.ts index 096a7d12b..3e93befc8 100644 --- a/packages/e2e/src/run/execute.ts +++ b/packages/e2e/src/run/execute.ts @@ -1039,14 +1039,14 @@ export class TargetExecutor implements SerialHost { await (registered.fn as SetupFn)(fixtures); } catch (cause) { // The body's own failure stays the verdict; the step it abandoned - // and the soft failures it kept are noted beside it. + // and the soft failures it kept are noted beside it. A skip keeps + // only the soft failures: the step it abandoned is not one. const notAwaited = await abandonNotAwaited(); const softFailure = soft.close(); - if (!isRuntimeSkip(cause)) { - for (const secondary of [notAwaited, softFailure]) { - if (secondary !== undefined) { - secondaryErrors.push(serializeError(secondary, { phase: 'body', projectRoot: this.config.projectRoot, redact })); - } + const kept = isRuntimeSkip(cause) ? [softFailure] : [notAwaited, softFailure]; + for (const secondary of kept) { + if (secondary !== undefined) { + secondaryErrors.push(serializeError(secondary, { phase: 'body', projectRoot: this.config.projectRoot, redact })); } } throw cause; diff --git a/packages/e2e/src/run/records.ts b/packages/e2e/src/run/records.ts index b6d728b8a..e3ca40906 100644 --- a/packages/e2e/src/run/records.ts +++ b/packages/e2e/src/run/records.ts @@ -160,6 +160,49 @@ export interface ResultRecord { */ export type WireResultRecord = Omit; +/** The failure a runtime skip followed, with the evidence its attempt captured. */ +export interface FailureBeforeSkip { + readonly error: SerializedError; + readonly failure: FailureEvidence | undefined; + /** The failing attempt's artifacts; for a serial member, the group attempt's. */ + readonly artifacts: readonly ArtifactRecord[]; +} + +/** + * The failure a test's `test.skip(...)` followed: the last attempt that + * failed before a skipped retry, or a soft failure the skipping attempt kept. + * Engine cleanup diagnostics after the skip are not one. A serial member's + * attempts live on its group, which `groupOf` looks up only for such a member. + */ +export function failureBeforeSkip( + result: Pick, + groupOf: (id: string) => SerialGroupRecord | undefined, +): FailureBeforeSkip | undefined { + if (result.status !== 'skipped' || result.skip?.cause !== 'explicit') return undefined; + const runs = + result.serialGroupId === undefined + ? result.attempts + : (groupOf(result.serialGroupId)?.attempts ?? []).flatMap((attempt) => + attempt.members + .filter((member) => member.testId === result.test.id) + .map((member) => ({ ...member, artifacts: attempt.artifacts })), + ); + for (const run of runs.toReversed()) { + const error = run.error ?? run.secondaryErrors.find((secondary) => secondary.phase !== 'cleanup'); + if (error !== undefined) return { error, failure: run.failure, artifacts: run.artifacts }; + } + return undefined; +} + +/** Whether any skipped result followed a failure: what `failOnSkippedFailure` fails a run for. */ +export function someSkippedAfterFailure( + results: readonly ResultRecord[], + serialGroups: readonly SerialGroupRecord[], +): boolean { + const groups = new Map(serialGroups.map((group) => [group.id, group])); + return results.some((result) => failureBeforeSkip(result, (id) => groups.get(id)) !== undefined); +} + /** Strips the live target from a result for transport. */ export function encodeResult(record: ResultRecord): WireResultRecord { const { target: _target, ...rest } = record; diff --git a/packages/e2e/src/run/runner.ts b/packages/e2e/src/run/runner.ts index f000bfc74..8474ebd1f 100644 --- a/packages/e2e/src/run/runner.ts +++ b/packages/e2e/src/run/runner.ts @@ -36,7 +36,7 @@ import { writeJsonReport } from '../report/write.ts'; import { createRunEventEmitter, toEventResult, type RunEventSink, type RunExitCode, type RunStatus, type RunEventFact, type SetupStep } from './events.ts'; import { allocateAppPorts } from './app-ports.ts'; import { inProcessSpawner } from './in-process.ts'; -import type { ResultRecord, RunError, SerialGroupRecord } from './records.ts'; +import { someSkippedAfterFailure, type ResultRecord, type RunError, type SerialGroupRecord } from './records.ts'; import { runUnits } from './scheduler.ts'; import { buildWorkPlans, plannedSlots, type TargetWorkPlan } from './units.ts'; import { SessionStore } from './sessions.ts'; @@ -402,6 +402,7 @@ export async function run(options: RunOptions = {}): Promise { combineExitCodes( [ ...resultExitCodes(results), + ...(loaded.config?.failOnSkippedFailure === true && someSkippedAfterFailure(results, serialGroups) ? [1] : []), ...runErrors.map((runError) => exitCodeForCategory(runError.error.category)), ...(interruptController.signal.aborted ? [130] : []), ].filter((code) => code !== 130 || !(runAborted || stoppedEarly)), diff --git a/packages/e2e/src/types.ts b/packages/e2e/src/types.ts index a34ea77eb..86c2e04e5 100644 --- a/packages/e2e/src/types.ts +++ b/packages/e2e/src/types.ts @@ -1414,6 +1414,8 @@ export interface E2EConfig { cleanupTimeout?: number; /** Retries per test, 0 through 10; default 1 in CI, else 0. */ retries?: number; + /** Fail the run when a test skips itself after a soft failure or an earlier failed attempt; default false. */ + failOnSkippedFailure?: boolean; /** Parallel workers, 1 through 1024; default 1 in CI, else half the cores. An engine may cap it lower. */ workers?: number; /** `{ store }` hands every artifact to a host store as it is produced. */ diff --git a/packages/e2e/tests/integration/skip-after-failure.test.ts b/packages/e2e/tests/integration/skip-after-failure.test.ts new file mode 100644 index 000000000..b6950b5a1 --- /dev/null +++ b/packages/e2e/tests/integration/skip-after-failure.test.ts @@ -0,0 +1,141 @@ +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; +import type { Report1Document } from '../../src/report/build.ts'; +import { assertValidReport } from '../helpers/report-schema.ts'; +import { createProject, type FixtureProject } from '../helpers/run-project.ts'; + +const CLI = fileURLToPath(new URL('../../dist/cli/bin.js', import.meta.url)); +const projects: FixtureProject[] = []; + +afterEach(() => { + for (const project of projects.splice(0)) project.cleanup(); +}); + +function runSkipped(source: string, policy = '', retries = 0, engineMembers = '') { + const project = createProject({ + 'e2e.config.ts': `import { defineEngine } from 'e2e/engine'; + export default { + targets: [{ name: 'headless', platform: 'test', engine: defineEngine({ name: 'noop', version: '1', spiVersion: 1, ${engineMembers} }) }], + workers: 1, retries: ${retries}, ${policy} + };`, + 'tests/skip.e2e.ts': `import { test, expect } from 'e2e'; + import { existsSync, writeFileSync } from 'node:fs'; + ${source}`, + }); + projects.push(project); + const cli = spawnSync(process.execPath, [CLI, 'run'], { + cwd: project.dir, + env: { ...process.env, CI: '', E2E_TELEMETRY_DISABLED: '1', NO_COLOR: '1' }, + encoding: 'utf8', + timeout: 20_000, + }); + expect(cli.error).toBeUndefined(); + const output = cli.stdout + cli.stderr; + expect(output).not.toContain('Run Errors'); + const report = JSON.parse(readFileSync(path.join(project.dir, '.e2e/report.json'), 'utf8')) as Report1Document; + assertValidReport(report); + return { project, output, report, exitCode: cli.status }; +} + +const SOFT_SKIP = ` + test.afterEach(() => writeFileSync('teardown.txt', 'ran')); + test('soft then skip', () => { + expect.soft(1, 'the count').toBe(2); + expect.soft('old', 'the version').toBe('new'); + test.skip('feature disabled'); + throw new Error('unreachable'); + });`; + +const RETRY_SKIP = ` + test('fails then skips its retry', { retries: 2 }, () => { + if (existsSync('first-attempt.txt')) test.skip('feature disabled on retry'); + writeFileSync('first-attempt.txt', 'ran'); + expect(1, 'original failure').toBe(2); + });`; + +describe('a runtime skip after a failure', () => { + it('keeps soft failures and prints them without changing the default exit code', () => { + const { report, output, project, exitCode } = runSkipped(SOFT_SKIP); + expect(exitCode).toBe(0); + const result = report.run.results[0]!; + expect(result.status).toBe('skipped'); + expect(result.skip?.reason).toBe('feature disabled'); + expect(result.attempts[0]?.secondaryErrors).toMatchObject([ + { code: 'ASSERTION_FAILED', phase: 'body', message: '2 soft assertions failed\n1. the count: expected 1 to be 2\n2. the version: expected "old" to be "new"' }, + ]); + expect(output).toContain('Skipped After Failure 1'); + expect(output).toContain('the count: expected 1 to be 2'); + expect(output).toContain('the version: expected "old" to be "new"'); + expect(readFileSync(path.join(project.dir, 'teardown.txt'), 'utf8')).toBe('ran'); + }); + + it('can fail CI on a soft failure followed by skip without retrying the skip', () => { + const { report, exitCode, output } = runSkipped(SOFT_SKIP, 'failOnSkippedFailure: true,', 2); + expect(exitCode).toBe(1); + expect(report.run.status).toBe('failed'); + expect(report.run.results[0]?.attempts).toHaveLength(1); + expect(report.run.results[0]?.status).toBe('skipped'); + expect(output).toContain('Skipped After Failure 1'); + }); + + it.each(['', 'failOnSkippedFailure: true,'])('shows the original failure when a retry skips (%s)', (policy) => { + const { report, output, exitCode } = runSkipped(RETRY_SKIP, policy); + expect(exitCode).toBe(policy === '' ? 0 : 1); + expect(report.run.results[0]?.attempts.map((attempt) => attempt.status)).toEqual(['failed', 'skipped']); + expect(output).toContain('Skipped After Failure 1'); + expect(output).toContain('original failure: expected 1 to be 2'); + }); + + it('keeps clean skips and passing soft assertions green with the policy enabled', () => { + const { report, output, exitCode } = runSkipped(` + test('clean skip', () => { expect.soft(1).toBe(1); test.skip('not applicable'); }); + test('false condition', () => { test.skip(false); expect(1).toBe(1); }); + test('soft failure then recovery', { retries: 1 }, () => { + if (existsSync('recovery.txt')) return; + writeFileSync('recovery.txt', 'ran'); + expect.soft(1).toBe(2); + });`, 'failOnSkippedFailure: true,'); + expect(exitCode).toBe(0); + expect(report.run.results.map((result) => result.status)).toEqual(['skipped', 'passed', 'flaky']); + expect(output).not.toContain('Skipped After Failure'); + }); + + it('finds a serial member failure without blaming another skipped member', () => { + const { report, output, exitCode } = runSkipped(` + test.describe('serial', { serial: true }, () => { + test('clean skip', () => { test.skip('not applicable'); }); + test('soft skip', () => { expect.soft(1).toBe(2); test.skip('feature disabled'); }); + test('next member', () => { expect(1).toBe(1); }); + });`, 'failOnSkippedFailure: true,'); + expect(exitCode).toBe(1); + expect(report.run.results.map((result) => result.status)).toEqual(['skipped', 'skipped', 'passed']); + expect(output).toContain('Skipped After Failure 1'); + expect(output).toContain('1 soft assertion failed'); + }); + + it('does not turn post-skip cleanup diagnostics into a test failure', () => { + const { report, output, exitCode } = runSkipped( + `test('clean skip', () => { test.skip('not applicable'); });`, + 'failOnSkippedFailure: true,', + 0, + `async endAttempt() { throw new Error('cleanup failed'); },`, + ); + expect(report.run.results[0]?.attempts[0]?.secondaryErrors).toMatchObject([{ phase: 'cleanup' }]); + expect(exitCode).toBe(0); + expect(output).not.toContain('Skipped After Failure'); + }); + + it('shows the original failure even when the skipped retry also fails cleanup', () => { + const { output, exitCode } = runSkipped( + RETRY_SKIP, + 'failOnSkippedFailure: true,', + 0, + `async endAttempt() { throw new Error('cleanup failed'); },`, + ); + expect(exitCode).toBe(1); + expect(output).toContain('original failure: expected 1 to be 2'); + }); +}); diff --git a/packages/e2e/tests/types/sdk-types.ts b/packages/e2e/tests/types/sdk-types.ts index e17f42408..c9bbe082e 100644 --- a/packages/e2e/tests/types/sdk-types.ts +++ b/packages/e2e/tests/types/sdk-types.ts @@ -95,6 +95,7 @@ engineSnapshot.treeUnavailable satisfies true | undefined; // @ts-expect-error targets is required ({ cache: 'read-write' }) satisfies E2EConfig; ({ targets, cache: 'read-write' }) satisfies E2EConfig; +({ targets, failOnSkippedFailure: true }) satisfies E2EConfig; ({ targets, cache: { mode: 'read-only', store: remoteStore, dir: 'shared-cache' } }) satisfies E2EConfig; ({ targets: [{ platform: 'ios' }] }) satisfies E2EConfig; // @ts-expect-error specVersion is gone; the runner version is the format version diff --git a/packages/e2e/tests/unit/config.test.ts b/packages/e2e/tests/unit/config.test.ts index d381fd2a7..227d6d974 100644 --- a/packages/e2e/tests/unit/config.test.ts +++ b/packages/e2e/tests/unit/config.test.ts @@ -7,6 +7,7 @@ import { ConfigurationError, defineEngine } from '../../src/engine/index.ts'; import { secrets } from '../../src/secrets.ts'; import type { CommandConfig, E2EConfig, Target, TargetApp } from '../../src/types.ts'; import { snapshot } from '../helpers/snapshot.ts'; +import { invalid } from '../helpers/invalid.ts'; const ROOT = '/tmp/e2e-config-project'; const BASE_ENV = {} as NodeJS.ProcessEnv; @@ -51,6 +52,16 @@ describe('CI mode', () => { }); describe('resolveConfig', () => { + it('validates the opt-in skip failure policy without invalidating recordings', () => { + expect(resolve({}).failOnSkippedFailure).toBe(false); + expect(resolve({ failOnSkippedFailure: true }).failOnSkippedFailure).toBe(true); + expect(resolve({ failOnSkippedFailure: false }).failOnSkippedFailure).toBe(false); + expect(resolve({ failOnSkippedFailure: true }).configDigest).toBe(resolve({}).configDigest); + for (const value of ['true', 1, null, {}]) { + expect(() => resolve(invalid({ failOnSkippedFailure: value }))).toThrow(/failOnSkippedFailure must be a boolean/); + } + }); + it('applies specification defaults', () => { const config = resolve({}); expect(config.timeout).toBe(120_000); diff --git a/packages/e2e/tests/unit/report-list.test.ts b/packages/e2e/tests/unit/report-list.test.ts index a9a1da396..699646615 100644 --- a/packages/e2e/tests/unit/report-list.test.ts +++ b/packages/e2e/tests/unit/report-list.test.ts @@ -569,6 +569,37 @@ describe('ListReporter', () => { expect(lines.some((line) => line.includes('trace.zip'))).toBe(false); }); + it('lists a retry that skipped with the failed attempt’s error and evidence', () => { + const { lines, output } = capture(); + const reporter = plainReporter(output); + reporter.handle(runStarted()); + reporter.handle(plan([{ file: 'tests/a.e2e.ts', tests: 1 }])); + const failed = failedAttempt('original failure'); + failed.failure = { url: 'https://app.test/checkout', candidates: ['button "Pay later"'], screen: 'attempt-1:artifact:0' }; + failed.artifacts = [ + { + id: 'attempt-1:artifact:0', + kind: 'log', + mediaType: 'text/plain', + path: 'chromium/a/attempt-0/screen.txt', + redaction: 'complete', + producer: { kind: 'attempt' }, + }, + ]; + const skipped = attempt({ id: 'attempt-2', index: 1, status: 'skipped' }); + reporter.handle(finished({ + ...result({ status: 'skipped', id: 'a', file: 'tests/a.e2e.ts', title: ['first'], attempts: [failed, skipped] }), + skip: { cause: 'explicit', reason: 'feature disabled' }, + } as RunEventResult)); + reporter.handle(runFinished({ status: 'passed', exitCode: 0, reportPath: '/project/.e2e/report.json' })); + expect(lines.join('\n')).toContain(' Skipped After Failure 1 '); + expect(lines).toContain(' SKIP |chromium| tests/a.e2e.ts > first'); + expect(lines).toContain('ASSERTION_FAILED: original failure'); + expect(lines).toContain(' ❯ at https://app.test/checkout'); + expect(lines).toContain(' ❯ on screen button "Pay later"'); + expect(lines).toContain(' ❯ screen .e2e/artifacts/chromium/a/attempt-0/screen.txt'); + }); + it('names the status when a failure recorded no error', () => { const { lines, output } = capture(); const reporter = plainReporter(output); diff --git a/skills/e2e/references/writing-tests.md b/skills/e2e/references/writing-tests.md index d56dd9849..2bf38e198 100644 --- a/skills/e2e/references/writing-tests.md +++ b/skills/e2e/references/writing-tests.md @@ -230,7 +230,11 @@ message?)` is synchronous. matcher passes (`assertionTimeout` and 100 ms by default, stopping with the attempt); a throwing read keeps polling, and it is not a report step. `expect.soft(x)` keeps a failure instead of throwing; the attempt fails -after the body with every soft failure listed. +after the body with every soft failure listed. If the body calls `test.skip`, +the test stays skipped and the failures remain in `secondaryErrors`. The CLI +shows `Skipped After Failure`, also when a retry skips after an earlier +failure. Set `failOnSkippedFailure: true` in the config to fail the run with +exit code 1 in either case. The default is `false`; clean skips stay green. `expect.any(Class)`, `expect.anything()`, `expect.objectContaining(obj)`, `expect.arrayContaining(arr)`, `expect.stringContaining(s)`, and `expect.stringMatching(s | RegExp)` stand in for values inside `toEqual`, From a3da00dcb147118ee64cdfbc2949225416e43b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Sat, 3 Oct 2026 11:40:46 +0200 Subject: [PATCH 12/50] fix(mobile): never pin an install's file path as the app app.open() launches (#711) * fix(mobile): never pin an install's file path as the app app.open() launches agent-device 0.21.18 can return an Android install with no package (its binary manifest parser misses aapt2 output, and the aapt fallback needs ANDROID_HOME), echoing the APK path as app. The engine pinned that path, so app.open() failed later with 'Android runtime hints require an installed package name'. installApp now opens by the reported id or the app passed in, and fails with ENGINE_FAILURE when there is neither. app.open() keeps the engine's own UNSUPPORTED_CAPABILITY reason instead of a generic rewrite. * test(mobile): pin the reported package, not the echoed path; clarify installApp failure docs --- .changeset/android-apppath-open.md | 6 +++ docs/mobile.mdx | 11 +++++ docs/reference/mobile.mdx | 5 ++- packages/e2e/src/run/fixtures.ts | 12 ++--- .../e2e/tests/unit/fixture-runtime.test.ts | 15 ++++++- packages/mobile/README.md | 3 +- packages/mobile/src/surface.ts | 19 ++++++-- packages/mobile/tests/unit/engine.test.ts | 45 +++++++++++++++++-- 8 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 .changeset/android-apppath-open.md diff --git a/.changeset/android-apppath-open.md b/.changeset/android-apppath-open.md new file mode 100644 index 000000000..999577a96 --- /dev/null +++ b/.changeset/android-apppath-open.md @@ -0,0 +1,6 @@ +--- +'@e2e-dev/mobile': patch +'e2e': patch +--- + +`device.installApp()` no longer pins the build's file path as the app when agent-device reports no bundle id or package for it, which on Android made `app.open()` fail with "Android runtime hints require an installed package name". It fails at the install with `ENGINE_FAILURE` naming `app.bundleId` or installApp's `app` option; the mobile docs' Troubleshooting covers the Android cause (agent-device 0.21.18 reads an aapt2-built APK's package only through the SDK's `aapt`, which it does not find in the macOS default SDK location, or when the install added the package). `app.open()` on a device now keeps the engine's own reason it cannot launch, such as a build not installed yet with `device.installApp()`, instead of a generic "pin one with app.bundleId or app.appPath". diff --git a/docs/mobile.mdx b/docs/mobile.mdx index 8a7886cec..528a54600 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -566,6 +566,17 @@ daemon (`npx agent-device daemon stop`) and reboot the simulator Do what it says. +**`device.installApp()` on Android reports no bundle id or package.** The +build installed, but agent-device could not say which package it is, so +`app.open()` would have nothing to launch. agent-device 0.21.18 cannot parse +the manifest of an APK built with aapt2 (every current Android build), so it +falls back to the Android SDK's `aapt`, which it looks for under +`ANDROID_SDK_ROOT`, `ANDROID_HOME`, or `~/Android/Sdk`, not the macOS default +`~/Library/Android/sdk`. Without `aapt` it names only a package the install +added, and a build already on the emulator adds none. Set the target's +`app.bundleId`, or export `ANDROID_HOME` and stop the agent-device daemon +(`daemon stop` below) so the next run starts it with the variable. + **A timeout on the device.** A simulator boot, runner connection, or app launch that times out on the device is `ENGINE_FAILURE` with agent-device's message and hint, never one of e2e's own timeout codes (`LAUNCH_TIMEOUT`, diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 60efb5fbf..96c3589e9 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -428,7 +428,10 @@ session opened. Android reads the device's foreground activity. `installApp` installs a build from inside a test or a fixture: the run's build once per device (no path installs `app.appPath`), or another build for an upgrade path. A plain install keeps the app's data; `reinstall: true` removes the app named by `app` (default: the pinned app) first. It resolves to the id to `openApp` -the build by. +the build by: the bundle id or package agent-device read from the build, else +the `app` passed in. With no reported id and no `app`, the install is +`ENGINE_FAILURE` instead of a file path posing as an id; see +[Troubleshooting](/mobile#troubleshooting). ```ts const { app } = await device.installApp('./build/MyApp.app', { reinstall: true }); diff --git a/packages/e2e/src/run/fixtures.ts b/packages/e2e/src/run/fixtures.ts index 125f3a74e..4929dea5f 100644 --- a/packages/e2e/src/run/fixtures.ts +++ b/packages/e2e/src/run/fixtures.ts @@ -445,7 +445,9 @@ function createApp(environment: AttemptEnvironment, engine: LocatorEngine, expos * `app.open()` on a target with no URL and no `navigate`, a device: it * launches the pinned app fresh, the way Maestro's `launchApp` does, and is * what a test calls before it looks at the screen. A path has nowhere to - * go there; a link goes through `device.openLink`. + * go there; a link goes through `device.openLink`. An engine without a + * relaunch hook is refused here; one with a hook reports its own reason it + * cannot launch, such as a build not installed yet. */ const launchPinnedApp = (openPath: string | undefined): Promise => steps.run('app', 'app.open', '', async () => { @@ -455,15 +457,13 @@ function createApp(environment: AttemptEnvironment, engine: LocatorEngine, expos `app.open("${openPath}") needs an app URL and target "${target.name}" has none: on a device app.open() launches the pinned app, and a link goes through device.openLink`, ); } - try { - await engine.session.app.restart(engine.operation(config.timeout)); - } catch (cause) { - if ((cause as { code?: unknown }).code !== 'UNSUPPORTED_CAPABILITY') throw cause; + if (target.engine?.session?.restart === undefined) { throw new ConfigurationError( 'UNSUPPORTED_CAPABILITY', - `app.open() has no app to launch on target "${target.name}": pin one with the target's app.bundleId or app.appPath, or bring one up with device.openApp`, + `app.open() has nothing to open on target "${target.name}": it has no app URL, and engine ${target.engine?.name ?? 'none'} does not relaunch a pinned app`, ); } + await engine.session.app.restart(engine.operation(config.timeout)); }); return { diff --git a/packages/e2e/tests/unit/fixture-runtime.test.ts b/packages/e2e/tests/unit/fixture-runtime.test.ts index 8aa031bd4..3673509c9 100644 --- a/packages/e2e/tests/unit/fixture-runtime.test.ts +++ b/packages/e2e/tests/unit/fixture-runtime.test.ts @@ -637,14 +637,25 @@ describe('app steering hooks', () => { expect(calls).toEqual([]); }); - it('app.open() on a device without a pinned app is UNSUPPORTED_CAPABILITY and names the app keys', async () => { + it('app.open() on an engine with no relaunch hook is UNSUPPORTED_CAPABILITY and says why', async () => { const { engine } = device(false); const { fixtures } = runtime(engine); await expect(fixtures.app.open()).rejects.toMatchObject({ code: 'UNSUPPORTED_CAPABILITY', - message: expect.stringContaining('appPath'), + message: expect.stringContaining('does not relaunch a pinned app'), }); }); + + it("app.open() keeps the engine's own reason it cannot launch", async () => { + const reason = 'launching the app needs the build installed first with device.installApp()'; + const engine = defineEngine({ + name: 'fake', version: '1', spiVersion: 1, + observe: async () => snapshot([]), + session: { restart: async () => { throw new EngineError('UNSUPPORTED_CAPABILITY', reason, { retryable: false }); } }, + }); + const { fixtures } = runtime(engine); + await expect(fixtures.app.open()).rejects.toMatchObject({ code: 'UNSUPPORTED_CAPABILITY', message: reason }); + }); }); describe('coordinate input', () => { diff --git a/packages/mobile/README.md b/packages/mobile/README.md index 4b962a10d..b94a45ccd 100644 --- a/packages/mobile/README.md +++ b/packages/mobile/README.md @@ -146,7 +146,8 @@ test, for upgrade and fresh-install paths the target's `app.appPath` cannot express. A plain install replaces the binary and keeps its data; `reinstall: true` removes the app named by `app` (default: the pinned app) first. It resolves to -the bundle id or package to `openApp` the build by. `openApp` takes an app id, +the bundle id or package to `openApp` the build by, else the `app` passed in; +with no reported id and no `app`, it fails with `ENGINE_FAILURE`. `openApp` takes an app id, never a link; `openLink` opens one, under the navigation rule. ## Agent tools diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 7a5247c1c..e9009eba6 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -100,9 +100,12 @@ export interface InstalledApp { /** Why nothing is pinned with `appPath` alone: the install is the suite's. */ const UNINSTALLED_BUILD = "the build the target's `app.appPath` names installed first with `device.installApp()`"; -/** The install fields this engine reads off agent-device's response. */ +/** + * The install fields this engine reads off agent-device's response. Its + * `app` is left out: it echoes the `app` passed in, else the identity, else + * the build's path, so it cannot tell an id from a file. + */ interface RawInstallResult { - readonly app: string; readonly appId?: string; readonly bundleId?: string; readonly package?: string; @@ -855,7 +858,15 @@ export class AgentDeviceSurface { signal, )) as RawInstallResult; const identity = result.bundleId ?? result.package ?? result.appId; - const installed: InstalledApp = { app: identity ?? result.app, ...(identity === undefined ? {} : { bundleId: identity }) }; + const opensBy = identity ?? app; + if (opensBy === undefined) { + throw new EngineError( + 'ENGINE_FAILURE', + `installed ${resolved}, but agent-device reported no bundle id or package for it, so nothing can open it: name the app with the target's \`app.bundleId\` or installApp's \`app\` option`, + { retryable: false }, + ); + } + const installed: InstalledApp = { app: opensBy, ...(identity === undefined ? {} : { bundleId: identity }) }; // The engine's own build, installed: without `app`, this is what `app.open()` launches from here on. if (engineBuild) this.installedApp = installed.app; return installed; @@ -1360,7 +1371,7 @@ export class AgentDeviceSurface { async restart(operation: OperationContext): Promise { const app = this.pinnedApp; - if (app === undefined) throw unsupported(`app.restart needs the target's \`app.bundleId\`, or ${UNINSTALLED_BUILD}`); + if (app === undefined) throw unsupported(`launching the app needs the target's \`app.bundleId\`, or ${UNINSTALLED_BUILD}`); await this.openApp(app, { relaunch: true }, operation.signal); } diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 67f32d232..1ce681502 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -526,6 +526,45 @@ describe('lifecycle', () => { expect(h.surface.pinnedApp).toBe('com.example.app'); }); + it('pins the package an Android install reports, and refuses the build path as an app id', async () => { + const signal = new AbortController().signal; + const h = harness({ appPath: './build/app.apk', platform: 'android' }, false); + h.fake.respond('apps.install', () => ({ + app: '/project/build/app.apk', + appPath: '/project/build/app.apk', + platform: 'android', + appId: 'dev.example.app', + package: 'dev.example.app', + identifiers: {}, + })); + await boot(h); + await h.engine.startAttempt!({ attemptId: 'a1', artifactsDir, signal, resolveSecret: noSecrets }); + expect(await h.surface.installApp(undefined, {}, signal)).toEqual({ app: 'dev.example.app', bundleId: 'dev.example.app' }); + await h.engine.session!.restart!(operation()); + expect(h.fake.lastArgs('apps.open')).toEqual({ app: 'dev.example.app', platform: 'android', relaunch: true }); + + // agent-device 0.21.18 echoes the path as `app` when it cannot read the + // package; that path is not something `open` can launch. + const unnamed = harness({ appPath: './build/app.apk', platform: 'android' }, false); + unnamed.fake.respond('apps.install', () => ({ app: '/project/build/app.apk', appPath: '/project/build/app.apk', platform: 'android', identifiers: {} })); + await boot(unnamed); + await unnamed.engine.startAttempt!({ attemptId: 'a1', artifactsDir, signal, resolveSecret: noSecrets }); + await expect(unnamed.surface.installApp(undefined, {}, signal)).rejects.toMatchObject({ + code: 'ENGINE_FAILURE', + message: expect.stringContaining('reported no bundle id or package'), + }); + expect(unnamed.surface.pinnedApp).toBeUndefined(); + await expect(unnamed.engine.session!.restart!(operation())).rejects.toMatchObject({ + code: 'UNSUPPORTED_CAPABILITY', + message: expect.stringContaining('device.installApp()'), + }); + expect(unnamed.fake.methods()).not.toContain('apps.open'); + + // An `app` passed in is what the build opens by, reported or not. + expect(await unnamed.surface.installApp(undefined, { app: 'dev.example.app' }, signal)).toEqual({ app: 'dev.example.app' }); + expect(unnamed.surface.pinnedApp).toBe('dev.example.app'); + }); + it('installs the engine build under the pinned app and device, and keeps opening the pinned app', async () => { const h = harness({ bundleId: 'com.example.app', appPath: '/builds/app.apk', device: 'Pixel 8', platform: 'android' }); h.fake.respond('apps.install', () => ({ app: 'com.example.app', appPath: '/builds/app.apk', platform: 'android', identifiers: {} })); @@ -1330,12 +1369,12 @@ describe('device fixture', () => { const h = harness(); await openAttempt(h); h.fake.respond('apps.install', () => ({ app: './b/App.app', appPath: '/b/App.app', platform: 'ios', bundleId: 'com.example.app', identifiers: {} })); - h.fake.respond('apps.reinstall', () => ({ app: 'com.example.app', appPath: '/b/App.app', platform: 'ios', identifiers: {} })); + h.fake.respond('apps.reinstall', (args) => ({ app: (args as { app: string }).app, appPath: '/b/App.app', platform: 'ios', identifiers: {} })); const device = fixture(h); const before = h.fake.calls.length; expect(await device.installApp('./b/App.app')).toEqual({ app: 'com.example.app', bundleId: 'com.example.app' }); - expect(await device.installApp('/b/App.app', { reinstall: true })).toEqual({ app: 'com.example.app' }); - expect(await device.installApp('/b/App.app', { app: 'com.other', reinstall: true })).toEqual({ app: 'com.example.app' }); + expect(await device.installApp('/b/App.app', { reinstall: true })).toEqual({ app: 'Settings' }); + expect(await device.installApp('/b/App.app', { app: 'com.other', reinstall: true })).toEqual({ app: 'com.other' }); expect(h.fake.calls.slice(before).map((call) => [call.method, call.args])).toEqual([ ['apps.install', { platform: 'ios', appPath: '/project/b/App.app' }], ['apps.reinstall', { platform: 'ios', app: 'Settings', appPath: '/b/App.app' }], From 94ddbfe46faa5cf7b9159fd0a604aa4317056051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Sat, 3 Oct 2026 12:58:30 +0200 Subject: [PATCH 13/50] fix(config): refuse secrets or credentials sharing an override env var (#721) * fix(config): refuse secrets and credentials sharing an override env var * docs(environment): name both credential override vars in collision rule --- .changeset/secret-env-collisions.md | 5 +++ docs/authentication.mdx | 4 +- docs/reference/config.mdx | 7 +++- docs/reference/environment.mdx | 8 +++- docs/reference/errors.mdx | 2 +- packages/e2e/src/config/resolve.ts | 28 +++++++++++++ packages/e2e/tests/unit/config.test.ts | 57 ++++++++++++++++++++++++++ skills/e2e/references/setup.md | 2 +- skills/e2e/references/writing-tests.md | 4 +- 9 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 .changeset/secret-env-collisions.md diff --git a/.changeset/secret-env-collisions.md b/.changeset/secret-env-collisions.md new file mode 100644 index 000000000..3f8d8815b --- /dev/null +++ b/.changeset/secret-env-collisions.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +Two credentials or two secrets whose names map to the same override variable (`api-key` and `api_key` both read `E2E_SECRET_API_KEY`) now fail the config load with `INVALID_CONFIG` instead of both silently taking one value. diff --git a/docs/authentication.mdx b/docs/authentication.mdx index 13a6ddc94..9a1cabd9e 100644 --- a/docs/authentication.mdx +++ b/docs/authentication.mdx @@ -211,7 +211,9 @@ export default { ``` `E2E_SECRET_` overrides an entry per run, with the same uppercasing rule -as credentials, so a CI job can supply the value without touching the config: +as credentials, so a CI job can supply the value without touching the config. +Two entries that map to one variable, like `stripe-key` and `stripe_key`, fail +the config load with `INVALID_CONFIG`, as two such credentials do: ```bash npm diff --git a/docs/reference/config.mdx b/docs/reference/config.mdx index 6a0202d9b..909b2ad7a 100644 --- a/docs/reference/config.mdx +++ b/docs/reference/config.mdx @@ -812,7 +812,9 @@ Credentials and secrets are separate namespaces: `secrets.get(name)` returns a reports know that handle as `.password` (``), so `credentials.admin` and `secrets.admin` can both exist; a `secrets` entry named exactly `admin.password` beside an `admin` credential is -`INVALID_CONFIG`. +`INVALID_CONFIG`. So are two credentials whose names map to one +`E2E_USER__*` override, like `svc-a` and `svc_a` (see +[Environment variables](/reference/environment)). ```ts /** Named test identity. The password remains an opaque Secret. */ @@ -838,7 +840,8 @@ export type SecretConfig = string | SecretProvider; An entry is a static value, or a provider called on every authorized fill (a vault lookup, a freshly computed TOTP). It is recorded in the config digest by -name only, and an `E2E_SECRET_` override wins over both. Every entry +name only, and an `E2E_SECRET_` override wins over both. Two entries whose +names map to one variable, like `api-key` and `api_key`, are `INVALID_CONFIG`. Every entry must resolve to a string of at least 6 characters (code points) or a function; anything else is `INVALID_CONFIG`. The minimum exists for redaction: every occurrence of a value becomes ``, and a shorter value would take ordinary diff --git a/docs/reference/environment.mdx b/docs/reference/environment.mdx index 767e5d6d3..d78b2448d 100644 --- a/docs/reference/environment.mdx +++ b/docs/reference/environment.mdx @@ -32,7 +32,13 @@ See [Models](/models) and [Subscriptions](/subscriptions). | `E2E_SECRET_` | set | Replaces the value of `secrets.` | `` is the name uppercased with every character outside `A-Z0-9` -replaced by `_`, so `power-user` becomes `E2E_USER_POWER_USER_USERNAME`. A +replaced by `_`, so `power-user` becomes `E2E_USER_POWER_USER_USERNAME`. Two +credentials, or two secrets, whose names map to the same `` (`api-key` +and `api_key` both read `E2E_SECRET_API_KEY`; `svc-a` and `svc_a` both read +`E2E_USER_SVC_A_USERNAME` and `_PASSWORD`) are `INVALID_CONFIG` at config +load, whether or not a variable is set, so one rotation never replaces two +entries; the message names the entries and the variables. A credential and a +secret may share a name, since their prefixes differ. A variable for a name absent from the config is ignored. An empty value is `INVALID_CONFIG`, and so is a password or secret shorter than 6 characters (code points): redaction rewrites every occurrence of a value, and one that diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index 6650115e1..c801d76b6 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -84,7 +84,7 @@ These appear in terminal output and in the report as plain string codes. | `CONFIG_NOT_FOUND` | `--config ` names a file that does not exist, or no `e2e.config.ts` or `e2e.config.mts` exists from the working directory up to the repository root | configuration | 2 | | `CONFIG_AMBIGUOUS` | Both `e2e.config.ts` and `e2e.config.mts` in one directory | configuration | 2 | | `CONFIG_LOAD_FAILED` | The config module failed to import: a package that is declared but not installed, an undeclared package, a wrong subpath, a removed export, or any other throw while it evaluates. The message names the fix. A config error raised while it evaluates keeps its own code: an option a factory the config calls (`web()`, `mobile()`, `defineEngine()`) refuses, or a `secrets.get()` handle turned into a string, is `INVALID_CONFIG` | configuration | 2 | -| `INVALID_CONFIG` | A factory the config module calls (`web()`, `mobile()`, `defineEngine()`) refused its options, with the factory's own message; or an unknown key, out-of-range value, invalid enum, missing `targets`, a target key outside `{ name, platform, engine, app, trace, video }`, a target `app` key that is unknown or malformed, `app` on a target without an engine, a `web()` target without `app.url` or with a device field, a `mobile()` target without `app.bundleId` or `app.appPath`, with `app.url` (not supported yet), a link in `bundleId`, or an unknown permission name or state (the message names the nearest name), an unknown key in a `web()` or `mobile()` option, including `web({ connect })` and `web({ basicAuth })` (the message names the nearest one), a `trace` or `video` that is not a recording mode, the removed artifact kinds list (`artifacts: [...]` or `artifacts.kinds`) and `artifacts.trace` block (the message names the `trace` mode they meant), the removed `artifacts.video` block or `video` artifact kind (the message names the `video` option), the removed `specVersion` key, a `secrets.get()` in an engine option that names no `secrets` entry (a credential's password is not one), a `secrets.get()` reference turned into a string while the config evaluates (a template literal, `String()`) or given where a string is needed (an `app.command` `args` entry or `env` value; only an engine option that declares secrets takes one), an `app.command` `args` entry or `env` value that is not a string (an `undefined` `env` value is dropped instead), an unknown key in `app.command` (the message names the nearest one), an `engine` that is not a `defineEngine` handle, `json` reporter combined with `list`, duplicate or malformed target name, a secret or credential password shorter than 6 characters (code points), oversized `agents..context`, an `expect.poll` `timeout` or `interval` that is not a finite number of milliseconds, a `tests` value that is not a glob or a list of globs, a `tests` list of only `!` exclusions, the removed top-level `limits` key, an agents entry that is a bare `StepExecutor` (the message names `{ executor }`) or a model instance (the message names `{ model }`), an unknown agent key (the message names the nearest one), the removed agent keys `timeout` and `maxTurns` (the message names `judgmentTimeout` and `maxModelCalls`), `system` or `tools` beside `executor`, a `tools` entry `defineTool` did not build or that takes a built-in tool name (a grammar verb, `complete_step`, `locate`, `start_recording`, `stop_recording`, or `report_finding`), `createToolLoopExecutor({ maxTurns })`, `reuseExisting` on an app whose `app.url` asks for a free port, or a port-0 `app.url` with no `app.command` to serve it | configuration | 2 | +| `INVALID_CONFIG` | A factory the config module calls (`web()`, `mobile()`, `defineEngine()`) refused its options, with the factory's own message; or an unknown key, out-of-range value, invalid enum, missing `targets`, a target key outside `{ name, platform, engine, app, trace, video }`, a target `app` key that is unknown or malformed, `app` on a target without an engine, a `web()` target without `app.url` or with a device field, a `mobile()` target without `app.bundleId` or `app.appPath`, with `app.url` (not supported yet), a link in `bundleId`, or an unknown permission name or state (the message names the nearest name), an unknown key in a `web()` or `mobile()` option, including `web({ connect })` and `web({ basicAuth })` (the message names the nearest one), a `trace` or `video` that is not a recording mode, the removed artifact kinds list (`artifacts: [...]` or `artifacts.kinds`) and `artifacts.trace` block (the message names the `trace` mode they meant), the removed `artifacts.video` block or `video` artifact kind (the message names the `video` option), the removed `specVersion` key, a `secrets.get()` in an engine option that names no `secrets` entry (a credential's password is not one), a `secrets.get()` reference turned into a string while the config evaluates (a template literal, `String()`) or given where a string is needed (an `app.command` `args` entry or `env` value; only an engine option that declares secrets takes one), an `app.command` `args` entry or `env` value that is not a string (an `undefined` `env` value is dropped instead), an unknown key in `app.command` (the message names the nearest one), an `engine` that is not a `defineEngine` handle, `json` reporter combined with `list`, duplicate or malformed target name, a secret or credential password shorter than 6 characters (code points), two credentials or two secrets whose names map to one `E2E_USER_*` or `E2E_SECRET_*` override variable (`api-key` and `api_key`), oversized `agents..context`, an `expect.poll` `timeout` or `interval` that is not a finite number of milliseconds, a `tests` value that is not a glob or a list of globs, a `tests` list of only `!` exclusions, the removed top-level `limits` key, an agents entry that is a bare `StepExecutor` (the message names `{ executor }`) or a model instance (the message names `{ model }`), an unknown agent key (the message names the nearest one), the removed agent keys `timeout` and `maxTurns` (the message names `judgmentTimeout` and `maxModelCalls`), `system` or `tools` beside `executor`, a `tools` entry `defineTool` did not build or that takes a built-in tool name (a grammar verb, `complete_step`, `locate`, `start_recording`, `stop_recording`, or `report_finding`), `createToolLoopExecutor({ maxTurns })`, `reuseExisting` on an app whose `app.url` asks for a free port, or a port-0 `app.url` with no `app.command` to serve it | configuration | 2 | | `CONFIG_NOT_DETERMINISTIC` | A worker re-resolved the config module and got a different digest than the runner | configuration | 2 | | `APP_URL_REQUIRED` | A navigation or relative URL on a target that declares no `app.url`, or an `app.command` with no `app.url` or `app.readyUrl` to poll | configuration | 2 | | `INVALID_APP_URL` | Base URL with userinfo, a query, a fragment, a non-HTTP scheme, plain HTTP on a non-loopback host, port 0 on a host other than a literal loopback address (`127.0.0.1`, `[::1]`), a loopback address the run could not bind a free port on, or unparseable | configuration | 2 | diff --git a/packages/e2e/src/config/resolve.ts b/packages/e2e/src/config/resolve.ts index bfa3de39e..dbfe74b70 100644 --- a/packages/e2e/src/config/resolve.ts +++ b/packages/e2e/src/config/resolve.ts @@ -769,6 +769,8 @@ function resolveSecrets( const credentials = new Map(); const secrets = new Map(); const allSecrets = new Map(); + checkEnvNameCollisions('E2E_USER', 'credentials', Object.keys(raw.credentials ?? {})); + checkEnvNameCollisions('E2E_SECRET', 'secrets', Object.keys(raw.secrets ?? {})); for (const [name, credential] of Object.entries(raw.credentials ?? {})) { const prefix = envName('E2E_USER', name); const username = env[`${prefix}_USERNAME`] ?? credential.username; @@ -801,6 +803,32 @@ function resolveSecrets( return { credentials, secrets, allSecrets }; } +/** + * Two entries of one namespace whose names map to the same override variable + * (`api-key` and `api_key` both read `E2E_SECRET_API_KEY`) would both take one + * rotated value, so the config load refuses them, whether or not the variable + * is set. The message names the entries and the variable, never a value. + */ +function checkEnvNameCollisions(prefix: 'E2E_USER' | 'E2E_SECRET', namespace: string, names: readonly string[]): void { + const byEnvName = new Map(); + for (const name of names) { + const variable = envName(prefix, name); + byEnvName.set(variable, [...(byEnvName.get(variable) ?? []), name]); + } + for (const [variable, group] of byEnvName) { + if (group.length < 2) continue; + const quoted = group.map((name) => `"${name}"`); + const entries = `${quoted.slice(0, -1).join(', ')} and ${quoted.at(-1)!}`; + const variables = prefix === 'E2E_USER' + ? `the override variables ${variable}_USERNAME and ${variable}_PASSWORD` + : `the override variable ${variable}`; + throw new ConfigurationError( + 'INVALID_CONFIG', + `${namespace} ${entries} share ${variables}, so one value set there would replace all of them; rename all but one`, + ); + } +} + /** * Every secret an engine's options hold (a `secrets.get()` the config * evaluated before any run existed) names a configured secret, so a typo diff --git a/packages/e2e/tests/unit/config.test.ts b/packages/e2e/tests/unit/config.test.ts index 227d6d974..41cc17f14 100644 --- a/packages/e2e/tests/unit/config.test.ts +++ b/packages/e2e/tests/unit/config.test.ts @@ -634,6 +634,63 @@ describe('resolveConfig', () => { ).toThrow(/secret "admin.password" has the name of credential "admin"'s password handle; rename the secret/); }); + it('refuses two secrets that share an E2E_SECRET_* variable, naming the entries and never a value, whether or not it is set', () => { + const values = ['secret-value-a1', 'secret-value-b2', 'secret-value-c3']; + const colliding = { 'api-key': values[0]!, api_key: values[1]! }; + for (const env of [BASE_ENV, { ...BASE_ENV, E2E_SECRET_API_KEY: 'rotated-value-z9' } as NodeJS.ProcessEnv]) { + let error: unknown; + try { + resolve({ secrets: colliding }, env); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ code: 'INVALID_CONFIG' }); + const message = (error as Error).message; + expect(message).toBe( + 'secrets "api-key" and "api_key" share the override variable E2E_SECRET_API_KEY, so one value set there would replace all of them; rename all but one', + ); + for (const value of [...values, 'rotated-value-z9']) expect(message).not.toContain(value); + } + expect(() => resolve({ secrets: { 'api.key': values[0]!, 'API-KEY': values[1]!, apikey: values[2]! } })).toThrow( + /^secrets "api\.key" and "API-KEY" share the override variable E2E_SECRET_API_KEY/, + ); + expect(() => resolve({ secrets: { 'a-b': values[0]!, a_b: values[1]!, 'a.b': values[2]! } })).toThrow( + /^secrets "a-b", "a_b" and "a\.b" share/, + ); + expect(() => resolve({ secrets: { 'pk-live': () => 'from-provider', pk_live: values[0]! } })).toThrow(/E2E_SECRET_PK_LIVE/); + }); + + it('refuses two credentials that share E2E_USER_* variables, naming the entries and never a value', () => { + const credential = (password: string) => ({ username: 'user@example.test', password }); + let error: unknown; + try { + resolve({ credentials: { 'svc-a': credential('password-one-1'), svc_a: credential('password-two-2') } }); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ code: 'INVALID_CONFIG' }); + const message = (error as Error).message; + expect(message).toBe( + 'credentials "svc-a" and "svc_a" share the override variables E2E_USER_SVC_A_USERNAME and E2E_USER_SVC_A_PASSWORD, so one value set there would replace all of them; rename all but one', + ); + expect(message).not.toMatch(/password-one-1|password-two-2|user@example\.test/); + }); + + it('keeps E2E_USER_* and E2E_SECRET_* apart and accepts punctuation that maps to a distinct variable', () => { + const config = resolve( + { + credentials: { 'svc-a': { username: 'u', password: 'password-1' }, 'svc-b': { username: 'u', password: 'password-2' } }, + secrets: { 'svc-a': 'secret-value-1', 'stripe.key': 'secret-value-2', 'stripe-key-2': 'secret-value-3' }, + }, + { ...BASE_ENV, E2E_SECRET_STRIPE_KEY: 'rotated-stripe', E2E_USER_SVC_A_PASSWORD: 'rotated-password' } as NodeJS.ProcessEnv, + ); + expect(config.secrets.get('stripe.key')?.value).toBe('rotated-stripe'); + expect(config.secrets.get('stripe-key-2')?.value).toBe('secret-value-3'); + expect(config.secrets.get('svc-a')?.value).toBe('secret-value-1'); + expect(config.allSecrets.get('svc-a.password')?.value).toBe('rotated-password'); + expect(config.allSecrets.get('svc-b.password')?.value).toBe('password-2'); + }); + it('refuses a static secret or password under 6 code points, from config or the environment, naming the minimum', () => { expect(() => resolve({ secrets: { pin: '12345' } })).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG' })); expect(() => resolve({ secrets: { pin: '12345' } })).toThrow( diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 1eedcbdbe..c62b5b340 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -246,7 +246,7 @@ For an app started elsewhere, point `app.url` at it, literally or via | --- | --- | | `AI_GATEWAY_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, ... | Read by provider packages, not the runner. | | `E2E_USER__USERNAME`, `E2E_USER__PASSWORD` | Override `credentials.`; `` is the name uppercased, other characters `_`. | -| `E2E_SECRET_` | Overrides `secrets.`, same rule. | +| `E2E_SECRET_` | Overrides `secrets.`, same rule. Two entries of one namespace mapping to one variable are `INVALID_CONFIG`. | | `CI` | CI defaults; list in topic `running`. | | `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK` | Disable anonymous telemetry, as does `e2e telemetry disable`; `E2E_TELEMETRY_DEBUG=1` prints events instead of sending. | diff --git a/skills/e2e/references/writing-tests.md b/skills/e2e/references/writing-tests.md index 2bf38e198..c25b3d9be 100644 --- a/skills/e2e/references/writing-tests.md +++ b/skills/e2e/references/writing-tests.md @@ -327,7 +327,9 @@ credentials: { A function is read at fill time and redacted only from that fill on. - `E2E_USER__USERNAME` and `E2E_USER__PASSWORD` override either field per run, even over a function; `` is the credential name - uppercased, every character outside `[A-Z0-9]` as `_`. + uppercased, every character outside `[A-Z0-9]` as `_`. Two credentials (or + two secrets) mapping to one variable, `svc-a` and `svc_a`, are + `INVALID_CONFIG`. - `credentials.user('admin').password` is a `Secret` with no plaintext accessor, named `admin.password` to the agent and in reports; `secrets.get()` never returns it (separate namespaces). Only `fill()` and `agent.act` params accept it, stringifying it From f0f9c8daaefd8751045e540007a09ba830147d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Sat, 3 Oct 2026 13:16:39 +0200 Subject: [PATCH 14/50] fix(security): allow only http(s) and about:blank on explicit navigation (#722) * fix(security): allow only http(s) navigation, refuse wrapped schemes on device links view-source:file:// passed the file:/data:/javascript: denylist and loaded local files via app.open, browser.goto, and the navigate verb. resolveNavigationUrl now admits http: and https: only; mobile links also refuse view-source:, blob:, filesystem:. * fix(security): admit exact about:blank, sync rule text in AGENTS.md and SECURITY.md * fix(web): refuse about:blank cookie URL, qualify scheme rule text --- .changeset/nav-scheme-allowlist.md | 7 +++ AGENTS.md | 13 ++-- SECURITY.md | 7 ++- docs/browser.mdx | 4 +- docs/reference/app.mdx | 2 +- docs/reference/config.mdx | 5 +- docs/reference/engine.mdx | 4 +- docs/reference/errors.mdx | 4 +- docs/reference/mobile.mdx | 20 +++--- docs/reference/web.mdx | 2 +- docs/security.mdx | 12 ++-- packages/e2e/src/engine/index.ts | 4 +- packages/e2e/src/internal/urls.ts | 17 ++++-- .../e2e/tests/integration/mcp-server.test.ts | 4 +- .../tests/integration/web-platform.test.ts | 61 ++++++++++++++++++- packages/e2e/tests/unit/urls.test.ts | 44 +++++++++++++ packages/mobile/src/device.ts | 4 +- packages/mobile/src/links.ts | 17 +++--- packages/mobile/tests/unit/engine.test.ts | 4 +- packages/mobile/tests/unit/links.test.ts | 17 ++++++ packages/mobile/tests/unit/tools.test.ts | 2 +- packages/web/src/browser.ts | 9 ++- packages/web/src/observation.ts | 2 +- skills/e2e/references/debugging.md | 2 +- 24 files changed, 210 insertions(+), 57 deletions(-) create mode 100644 .changeset/nav-scheme-allowlist.md diff --git a/.changeset/nav-scheme-allowlist.md b/.changeset/nav-scheme-allowlist.md new file mode 100644 index 000000000..d65c0c4b3 --- /dev/null +++ b/.changeset/nav-scheme-allowlist.md @@ -0,0 +1,7 @@ +--- +"e2e": patch +"@e2e-dev/mobile": patch +"@e2e-dev/web": patch +--- + +Explicit navigation now uses an allowlist instead of a denylist: `app.open`, `browser.goto`, and the agent's `navigate` verb admit `http:`, `https:`, and the exact `about:blank`, and every other scheme (`chrome:`, `blob:`, `about:srcdoc`, ...) is `POLICY_DENIED`. A wrapped scheme such as `view-source:file:///...` no longer loads a local file; it is `POLICY_DENIED` like `file:` itself. `device.openLink` and `device.openApp` also refuse `view-source:`, `blob:`, and `filesystem:` links. `browser.setCookies` refuses an `about:blank` cookie URL with `POLICY_DENIED`. diff --git a/AGENTS.md b/AGENTS.md index 6340a3daf..a70a410d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -411,10 +411,15 @@ trees, on both platforms, without a device. denied destinations, forbidden fills, and tainted pixels. Model text is never evaluated as code, selectors, shell, or config. App content, ledger text, and pixels are quoted as untrusted evidence with no policy authority. - - Every navigation a test or the agent asks for (`app.open`, the `navigate` - verb, `device.openLink`) goes through one rule: `file:`, `data:`, and - `javascript:` destinations and malformed URLs are `POLICY_DENIED`. There - is no origin or host allowlist; PR #290 removed them on purpose, since a + - Every navigation a test or the agent asks for (`app.open`, + `browser.goto`, the `navigate` verb) goes through one rule, + `resolveNavigationUrl`: only `http:`, `https:`, and the exact + `about:blank` pass; every other scheme (`view-source:file:` included) and + a malformed URL is `POLICY_DENIED` (a relative URL on a target with no + `app.url` is `APP_URL_REQUIRED`). `device.openLink` takes custom app + schemes, so it refuses a list instead (`packages/mobile/src/links.ts`): + `file:`, `data:`, `javascript:`, `view-source:`, `blob:`, `filesystem:`. + There is no origin or host allowlist; PR #290 removed them on purpose, since a click reaches any origin a typed URL could. - Sessions are per-run, target-bound, AES-256-GCM encrypted with a memory-only key, and deleted at cleanup; payloads never enter diagnostics. diff --git a/SECURITY.md b/SECURITY.md index 3126c7125..75bbae0c7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -55,8 +55,11 @@ with a memory-only key, and deleted at cleanup. ## Navigation and origins -A test or the agent may open any http(s) URL; `file:`, `data:`, and -`javascript:` are denied. There is no origin allowlist, on navigation or on +On the web a test or the agent may open any http(s) URL, and `about:blank`; +every other scheme (`file:`, `data:`, `javascript:`, `view-source:`, `blob:`) +is denied. A device link may use an app's custom scheme, so +`device.openLink` refuses a list instead: `file:`, `data:`, `javascript:`, +`view-source:`, `blob:`, and `filesystem:`. There is no origin allowlist, on navigation or on secret fills. A click, a redirect, or a popup reaches another origin just as a typed URL would, so a gate on typed navigation guarded nothing; and a secret is only ever typed into a field the test itself handed to the step, a diff --git a/docs/browser.mdx b/docs/browser.mdx index bf4e4a5b6..7397dbfd7 100644 --- a/docs/browser.mdx +++ b/docs/browser.mdx @@ -120,8 +120,8 @@ assertions for visible UI so tests depend less on the DOM structure. await browser.setCookies([{ url: 'http://localhost:3000', name: 'tz', value: 'UTC' }]); ``` -A cookie URL that does not parse, or uses `file:`, `data:`, or -`javascript:`, is `POLICY_DENIED`; a domain cookie is checked as the origin +A cookie URL that does not parse, or uses a scheme other than `http:` or +`https:` (`about:blank` included), is `POLICY_DENIED`; a domain cookie is checked as the origin it would be sent to. If any entry is refused, no cookies are set. To resize the page, see [Testing viewports](/viewports). diff --git a/docs/reference/app.mdx b/docs/reference/app.mdx index 6313a4b9c..16401f814 100644 --- a/docs/reference/app.mdx +++ b/docs/reference/app.mdx @@ -43,7 +43,7 @@ timeout. They do not accept per-call timeout options. | Method | Timeout | Throws | | --- | --- | --- | -| `open(path?)` | `config.timeout`, 120000 ms | `APP_URL_REQUIRED` for a path when the target declares no `app.url`, `POLICY_DENIED` for a `file:`, `data:`, or `javascript:` URL, `APP_UNREACHABLE`, `ACTION_FAILED`, `UNSUPPORTED_CAPABILITY` without `session.open` on a target with a URL or without `session.restart` on one without (a device launches its pinned app) | +| `open(path?)` | `config.timeout`, 120000 ms | `APP_URL_REQUIRED` for a path when the target declares no `app.url`, `POLICY_DENIED` for a URL whose scheme is not `http:` or `https:` (`about:blank` aside), `APP_UNREACHABLE`, `ACTION_FAILED`, `UNSUPPORTED_CAPABILITY` without `session.open` on a target with a URL or without `session.restart` on one without (a device launches its pinned app) | | `restart()` | `config.timeout`, 120000 ms | `ACTION_FAILED`, `ENGINE_FAILURE`, `UNSUPPORTED_CAPABILITY` without `session.restart` | | `clearState()` | `config.timeout`, 120000 ms | `ACTION_FAILED`, `ENGINE_FAILURE`, `UNSUPPORTED_CAPABILITY` without `session.reset` | | `back()` | `config.actionTimeout`, 30000 ms | `APP_NOT_OPEN`, `ACTION_FAILED`, `UNSUPPORTED_CAPABILITY` without `session.back` | diff --git a/docs/reference/config.mdx b/docs/reference/config.mdx index 909b2ad7a..b9834354f 100644 --- a/docs/reference/config.mdx +++ b/docs/reference/config.mdx @@ -280,8 +280,9 @@ it (no `app.command`) is `INVALID_CONFIG`. An `app.command` with neither ### The app's site -A test or the agent may open any http(s) URL; `file:`, `data:`, and -`javascript:` are `POLICY_DENIED`. The site of `url`, its registrable domain +A test or the agent may open any http(s) URL, and `about:blank`; every +other scheme (`file:`, `data:`, `javascript:`, `view-source:`, `blob:`) is +`POLICY_DENIED`. The site of `url`, its registrable domain (`app.example.com` and `auth.example.com` are one site), scopes two things: the browser engine's [`headers`](/reference/web#options) reach the site and no other host, and child frames off the site stay out of diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 994ab50e9..84dc9ed10 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -1185,8 +1185,8 @@ export interface EngineFixtureContext { /** * Resolves a navigation target against the base URL. Throws * `APP_URL_REQUIRED` when the target declares no URL and `POLICY_DENIED` - * for a `file:`, `data:`, or `javascript:` scheme, so a fixture never - * re-implements the rule the harness owns. + * for any scheme but `http:` and `https:` (`about:blank` aside), so a + * fixture never re-implements the rule the harness owns. */ resolveUrl(url: string): string; }; diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index c801d76b6..7eb5c4cca 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -56,7 +56,7 @@ try { | `TEST_SETUP_FAILED` | Test prerequisites outside the product failed to arrange (blocked: test_setup). Also a `test.extend()` fixture that broke its contract: returned without calling `use()`, called it twice, or redefined a fixture the target's engine contributes | configuration | 2 | | `AUTOMATION_UNSUPPORTED` | The step needs an interaction the available toolset cannot perform, e.g. drag-and-drop (blocked: automation) | test | 1 | | `MODEL_UNAVAILABLE` | No model configured: the built-in agent's `agents..model` holds no AI SDK model instance; the message names the agent that was selected. Checked once per run when the `agent` fixture is first acquired; recorded as one run-level error that stops the run. `e2e explore` checks before anything starts, a custom executor's entry included, since explore runs the built-in agent. A custom executor without a model is not checked: its `act` and `assert` steps run, and `waitFor` or `extract` fails with this code on its first call. Evicts no [recording](/cache). | configuration | 2 | -| `POLICY_DENIED` | A denied destination (`file:`, `data:`, `javascript:`, a malformed URL), a forbidden fill (an undeclared secret, a non-editable sink, a password into a field that is not a password field), or pixel evidence after a secret fill (`vision: 'only'` with pixels withheld). Also reading a value, text, or attribute from a secure field, the getters and the `toHaveValue`, `toHaveText`, `toContainText`, and `toHaveAttribute` matchers alike. An unknown tool or an undeclared field is refused to the model as the call's result and is not this code | configuration | 2 | +| `POLICY_DENIED` | A denied destination (a navigation scheme other than `http:` or `https:`, `about:blank` aside, a malformed URL), a forbidden fill (an undeclared secret, a non-editable sink, a password into a field that is not a password field), or pixel evidence after a secret fill (`vision: 'only'` with pixels withheld). Also reading a value, text, or attribute from a secure field, the getters and the `toHaveValue`, `toHaveText`, `toContainText`, and `toHaveAttribute` matchers alike. An unknown tool or an undeclared field is refused to the model as the call's result and is not this code | configuration | 2 | | `APP_UNREACHABLE` | The app URL could not be reached, or an app command exited early or exceeded `startupTimeout`. The startup error includes recent log output with `command.env` values redacted, and suggests adding a `log` if none is configured. Under the agent, also any engine fault with no more specific code: on a device, an iOS automation runner that is busy or wedged reports here with a message that names the runner, the session and device, and the recovery; see [Testing iOS and Android](/mobile#troubleshooting) | infrastructure | 3 | | `APP_ALREADY_RUNNING` | `readyUrl` already answered before the target's `app.command` was spawned and `reuseExisting` was not in effect; stop that process or set `reuseExisting: true` locally | infrastructure | 3 | | `MODEL_PROVIDER_FAILED` | Provider transport failure (network error, 5xx, rate limit, or a request with no response within 120 s) after the adapter's transport retries (up to 5 per model call, honoring `retry-after`, bounded by the remaining step timeout). A request that stalls is aborted at 120 s and sent again, and an `agent.act` step's `--debug` transcript notes it. The bound covers the whole request, so a model that needs longer than 120 s for one answer fails here too. Evicts no [recording](/cache) | infrastructure | 3 | @@ -104,7 +104,7 @@ These appear in terminal output and in the report as plain string codes. | `NO_LAST_RUN` | `--last-failed` found no `report.json` where the run writes it, or a file there that is not a `report-1` document. Run once without the flag first. | configuration | 2 | | `ONLY_IN_CI` | A focused test reached a CI run | configuration | 2 | | `COLLECTION_ERROR` | Registration outside collection, an invalid title, a non-function body, an async `describe` body, nested serial groups, an option forbidden inside a serial group, a setup test inside `describe`, an invalid session name, a malformed test option such as `tags` that is not a list of distinct names, a test or `describe` option key the call does not take (the message names the nearest one), a test file that fails to import (in a run with no positionals, or a file the positionals selected), a session that a selected test consumes or `e2e explore --session` names but no collected setup test produces (the message lists the declared sessions), a consumed session on a target whose engine has no `state` capability to save it, or a positional path outside the project root | configuration | 2 | -| `POLICY_DENIED` | A denied destination (`file:`, `data:`, `javascript:`, a malformed URL), a forbidden fill (an undeclared secret, a non-editable sink, a mismatched purpose, a `Secret` reaching a JSON sink), or tainted pixels (`app.screenshot()` after a secret fill, `vision: 'only'` with pixels withheld); see the [security model](/security). An unknown tool or an undeclared field goes back to the model as the call's result, never this code | configuration | 2 | +| `POLICY_DENIED` | A denied destination (a navigation scheme other than `http:` or `https:`, `about:blank` aside, a malformed URL), a forbidden fill (an undeclared secret, a non-editable sink, a mismatched purpose, a `Secret` reaching a JSON sink), or tainted pixels (`app.screenshot()` after a secret fill, `vision: 'only'` with pixels withheld); see the [security model](/security). An unknown tool or an undeclared field goes back to the model as the call's result, never this code | configuration | 2 | | `SESSION_MISMATCH` | A stored session does not match the current run, target, engine, platform, or app identity | configuration | 2 | | `SESSION_EXPIRED` | A stored session is past its expiry at restore time (configuration, exit 2), or a setup test saved state whose expiry is not after its creation (test, exit 1) | configuration or test | 2 or 1 | | `SESSION_INVALID` | A stored session failed authentication. Tampered or corrupt. | configuration | 2 | diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 96c3589e9..b5dd87e02 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -280,8 +280,8 @@ export interface Device { * `relaunch` restarts it fresh. `launchArguments` and `permissions` apply * to this launch alone; the target's configured ones apply to a `relaunch` * of the pinned app, and to no foreground-only open. A link is - * `INVALID_ARGUMENT` (`file:`, `data:`, and `javascript:` are - * `POLICY_DENIED`): `openLink` opens one. + * `INVALID_ARGUMENT` (`file:`, `data:`, `javascript:`, `view-source:`, + * `blob:`, and `filesystem:` are `POLICY_DENIED`): `openLink` opens one. */ openApp(app: string, options?: OpenAppOptions): Promise; /** @@ -291,8 +291,8 @@ export interface Device { * a web link the app does not claim goes to Safari, and the next * observation brings the app back. Android starts the link on that * package. Without an app, Android lets the OS route the link; iOS needs - * one (`INVALID_ARGUMENT` without). `file:`, `data:`, and `javascript:` - * links are `POLICY_DENIED`. + * one (`INVALID_ARGUMENT` without). `file:`, `data:`, `javascript:`, + * `view-source:`, `blob:`, and `filesystem:` links are `POLICY_DENIED`. */ openLink(url: string, options?: { app?: string }): Promise; /** @@ -449,9 +449,9 @@ name (`Settings`); `relaunch: true` restarts it fresh, and `launchArguments` and `permissions` apply to that launch alone (the target's own apply to a `relaunch` of the pinned app). It takes no link: agent-device opens any `scheme:rest` string as a URL, so a link is refused -before the device sees it. A `file:`, `data:`, or `javascript:` link is -`POLICY_DENIED`, the rule every navigation follows; any other link is -`INVALID_ARGUMENT` pointing at `openLink`. +before the device sees it. A `file:`, `data:`, `javascript:`, +`view-source:`, `blob:`, or `filesystem:` link is `POLICY_DENIED`; any other +link is `INVALID_ARGUMENT` pointing at `openLink`. `openLink` opens a deep link or a web link into `app` (default: the pinned app), for magic-link sign-in and deep-link routes, and the session observes @@ -461,8 +461,10 @@ observation brings the app back. Android starts the link on that package. Without an app, Android lets the OS route the link, to a verified app link owner or the browser, and the session follows the package that took it. iOS needs an app: an open bound to none leaves no app session to observe, so it -is `INVALID_ARGUMENT`. `file:`, `data:`, and `javascript:` links are -`POLICY_DENIED`, as on the web. The report shows the link cut before its +is `INVALID_ARGUMENT`. `file:`, `data:`, `javascript:`, `view-source:`, +`blob:`, and `filesystem:` links are `POLICY_DENIED`. A device link may use +any app's custom scheme, so the rule refuses these schemes rather than +admitting http(s) alone as the web does. The report shows the link cut before its query and fragment, where a magic link carries its token. The replay cache anchors on the location the link lands on: a step recorded after `openLink` replays when the link lands on the same screen again. diff --git a/docs/reference/web.mdx b/docs/reference/web.mdx index 2bdd8a42c..a1697d1ee 100644 --- a/docs/reference/web.mdx +++ b/docs/reference/web.mdx @@ -319,7 +319,7 @@ in a suite that also runs on other platforms. ### Navigation `goto` defaults `waitUntil` to `'load'` and throws `POLICY_DENIED` for a -`file:`, `data:`, or `javascript:` URL. `waitForURL` polls every 100 ms; a +URL whose scheme is not `http:` or `https:` (`about:blank` is admitted). `waitForURL` polls every 100 ms; a relative string resolves against the base URL and matches exactly after normalization, a `RegExp` tests the full URL, anything else (a predicate function, say) is `INVALID_ARGUMENT`, and a timeout is `ASSERTION_FAILED`. It diff --git a/docs/security.mdx b/docs/security.mdx index a3b07a879..a26d37bea 100644 --- a/docs/security.mdx +++ b/docs/security.mdx @@ -48,7 +48,7 @@ A tool call that breaks a rule fails with `POLICY_DENIED`: | Denied | Why | | --- | --- | -| Navigation to `file:`, `data:`, or `javascript:` | These URL schemes are forbidden. | +| Navigation to any scheme but `http:` and `https:`, other than `about:blank` | `file:`, `data:`, `javascript:`, and wrappers such as `view-source:file:` would load local or inline content. | | A malformed URL | Nothing is repaired. | | A `Secret` reaching a JSON sink | Handles carry no value, and no value may be serialized. | | Reading a value, text, or attribute from a secure field, with a getter or a `toHaveValue`, `toHaveText`, `toContainText`, or `toHaveAttribute` matcher | Assert the outcome, not the secret. A withheld value is redacted, not empty or absent. | @@ -77,10 +77,12 @@ list. Shared hosting domains such as `vercel.app` count as one site, so headers can reach other hosts under that domain. `basicAuth` is broader: it answers a `401` challenge from any origin. -The runner checks forbidden schemes on explicit navigation: `app.open`, the -agent's `navigate` verb, and `device.openLink`. `device.openApp` and the -`open_app` tool take an app id and refuse a link, since the device would -open one as a URL. The runner does not re-check redirects or attach to popup +The runner checks the scheme on explicit navigation. `app.open`, +`browser.goto`, and the agent's `navigate` verb admit `http:`, `https:`, +and the exact `about:blank`. A device link may use an app's custom scheme, so `device.openLink` +refuses a list instead: `file:`, `data:`, `javascript:`, `view-source:`, +`blob:`, and `filesystem:`. `device.openApp` and the `open_app` tool take an +app id and refuse a link, since the device would open one as a URL. The runner does not re-check redirects or attach to popup pages. Register `browser.onDialog` to handle browser dialogs. ## Secrets diff --git a/packages/e2e/src/engine/index.ts b/packages/e2e/src/engine/index.ts index 280525dc1..af2748353 100644 --- a/packages/e2e/src/engine/index.ts +++ b/packages/e2e/src/engine/index.ts @@ -203,8 +203,8 @@ export interface EngineFixtureContext { /** * Resolves a navigation target against the base URL. Throws * `APP_URL_REQUIRED` when the target declares no URL and `POLICY_DENIED` - * for a `file:`, `data:`, or `javascript:` scheme, so a fixture never - * re-implements the rule the harness owns. + * for any scheme but `http:` and `https:` (`about:blank` aside), so a + * fixture never re-implements the rule the harness owns. */ resolveUrl(url: string): string; }; diff --git a/packages/e2e/src/internal/urls.ts b/packages/e2e/src/internal/urls.ts index d2bd0f902..c2102ea3e 100644 --- a/packages/e2e/src/internal/urls.ts +++ b/packages/e2e/src/internal/urls.ts @@ -110,13 +110,18 @@ export function isImplicitTestHost(hostname: string): boolean { return isLoopbackHost(hostname) || hostname.endsWith('.test'); } -const FORBIDDEN_PROTOCOLS = new Set(['file:', 'data:', 'javascript:']); +const NAVIGABLE_PROTOCOLS = new Set(['http:', 'https:']); + +/** The one non-http(s) page navigation opens: empty, with no local or inline content. */ +const BLANK_PAGE = 'about:blank'; /** - * Resolves a navigation URL against the base and refuses the schemes no test - * may open. Returns the absolute URL string. Any http(s) origin is admitted: - * a click can reach one just as well, so a gate on typed navigation alone - * would guard nothing. + * Resolves a navigation URL against the base and refuses every scheme but + * http(s), except the exact `about:blank`. Returns the absolute URL string. + * Any http(s) origin is admitted: a click can reach one just as well, so a + * gate on typed navigation alone would guard nothing. The rule is an + * allowlist because a browser wraps and nests schemes (`view-source:file:`, + * `blob:`, `filesystem:`), and a list of forbidden ones misses the wrapper. */ export function resolveNavigationUrl(input: string, base: NormalizedBaseUrl | undefined): { url: string } { let url: URL; @@ -133,7 +138,7 @@ export function resolveNavigationUrl(input: string, base: NormalizedBaseUrl | un } throw new ConfigurationError('POLICY_DENIED', `malformed URL: ${input}`); } - if (FORBIDDEN_PROTOCOLS.has(url.protocol)) { + if (!NAVIGABLE_PROTOCOLS.has(url.protocol) && url.href !== BLANK_PAGE) { throw new ConfigurationError('POLICY_DENIED', `forbidden URL scheme: ${url.protocol}`); } return { url: url.href }; diff --git a/packages/e2e/tests/integration/mcp-server.test.ts b/packages/e2e/tests/integration/mcp-server.test.ts index f65f9bcf2..5f3ddf6e4 100644 --- a/packages/e2e/tests/integration/mcp-server.test.ts +++ b/packages/e2e/tests/integration/mcp-server.test.ts @@ -280,10 +280,10 @@ describe('e2e mcp', { timeout: 120_000 }, () => { expect(pointed.text).toContain('PIXEL_TAINTED'); expect(pointed.text).not.toContain('Tapped'); - for (const url of ['javascript:alert(1)', 'file:///etc/passwd']) { + for (const url of ['javascript:alert(1)', 'file:///etc/passwd', 'view-source:file:///etc/passwd', 'VIEW-SOURCE:http://example.test/', 'about:srcdoc']) { const denied = await call('navigate', { url }); expect(denied.isError, url).toBe(true); - expect(denied.text.split('\n')[0]).toBe(`navigate ${url} failed: POLICY_DENIED: forbidden URL scheme: ${url.slice(0, url.indexOf(':') + 1)}`); + expect(denied.text.split('\n')[0]).toBe(`navigate ${url} failed: POLICY_DENIED: forbidden URL scheme: ${url.slice(0, url.indexOf(':') + 1).toLowerCase()}`); expect(denied.text).not.toContain('Navigated'); } diff --git a/packages/e2e/tests/integration/web-platform.test.ts b/packages/e2e/tests/integration/web-platform.test.ts index 4ef7f6027..8822e966a 100644 --- a/packages/e2e/tests/integration/web-platform.test.ts +++ b/packages/e2e/tests/integration/web-platform.test.ts @@ -615,6 +615,38 @@ test('a route handler assertion that no step follows, then a teardown that navig }); `; +// A local file the wrapped schemes would load. Each test navigates once and +// fails with POLICY_DENIED; a navigation that went through fails differently. +const WRAPPED_SCHEMES = `import { test } from '@e2e-dev/web'; +const marker = new URL('../marker.txt', import.meta.url).href; +const leakCheck = async (browser) => { + const text = String(await browser.evaluate(() => document.body?.innerText ?? '')); + if (text.includes('marker-local-file')) throw new Error('LEAKED'); +}; +for (const url of ['view-source:' + marker, 'VIEW-SOURCE:' + marker, ' view-source:' + marker, 'blob:http://127.0.0.1/x', 'about:srcdoc']) { + test('app.open refuses ' + JSON.stringify(url), async ({ app, browser }) => { + await app.open(); + await app.open(url); + await leakCheck(browser); + }); +} +test('app.open and browser.goto admit about:blank', async ({ app, browser }) => { + await app.open(); + await app.open('about:blank'); + await browser.goto('about:blank'); + if ((await browser.url()) !== 'about:blank') throw new Error('not blank: ' + (await browser.url())); +}); +test('setCookies refuses about:blank', async ({ app, browser }) => { + await app.open(); + await browser.setCookies([{ name: 'flavor', value: 'oatmeal', url: 'about:blank' }]); +}); +test('browser.goto refuses view-source', async ({ app, browser }) => { + await app.open(); + await browser.goto('view-source:' + marker); + await leakCheck(browser); +}); +`; + describe('web platform integration', () => { let app: FixtureApp; let outcome: RunOutcome; @@ -623,7 +655,12 @@ describe('web platform integration', () => { beforeAll(async () => { app = await startFixtureApp(); ({ outcome, project } = await runProject( - { 'tests/kitchen.e2e.ts': KITCHEN_SINK, 'tests/late-handler.e2e.ts': LATE_HANDLER_THEN_TEARDOWN }, + { + 'tests/kitchen.e2e.ts': KITCHEN_SINK, + 'tests/late-handler.e2e.ts': LATE_HANDLER_THEN_TEARDOWN, + 'tests/schemes.e2e.ts': WRAPPED_SCHEMES, + 'marker.txt': 'marker-local-file\n', + }, { appUrl: app.url, config: { actionTimeout: 5_000, assertionTimeout: 4_000, timeout: 30_000 }, @@ -820,6 +857,28 @@ describe('web platform integration', () => { expect(result.attempts[0]!.error?.code).toBe('POLICY_DENIED'); }); + it('denies a wrapped or non-http(s) scheme on app.open and browser.goto before it loads', () => { + const titles = [ + 'app.open refuses "view-source:file://', + 'app.open refuses "VIEW-SOURCE:file://', + 'app.open refuses " view-source:file://', + 'app.open refuses "blob:http://127.0.0.1/x"', + 'app.open refuses "about:srcdoc"', + 'browser.goto refuses view-source', + ]; + for (const title of titles) { + const result = outcome.results.find((candidate) => candidate.test.title.startsWith(title)); + expect(result, title).toBeDefined(); + expect(result!.status, title).toBe('failed'); + expect(result!.attempts[0]!.error?.code, title).toBe('POLICY_DENIED'); + expect(result!.attempts[0]!.error?.message, title).toMatch(/^forbidden URL scheme: (view-source|blob|about):$/); + } + expect(resultByTitle(outcome, 'app.open and browser.goto admit about:blank').status).toBe('passed'); + const cookie = resultByTitle(outcome, 'setCookies refuses about:blank'); + expect(cookie.status).toBe('failed'); + expect(cookie.attempts[0]!.error).toMatchObject({ code: 'POLICY_DENIED', message: 'cookie URL must be http(s): about:blank' }); + }); + it('exits with configuration precedence and writes report.json', () => { // POLICY_DENIED failures classify as configuration errors, and exit-code // precedence is 130 > 4 > 3 > 2 > 1 > 0. diff --git a/packages/e2e/tests/unit/urls.test.ts b/packages/e2e/tests/unit/urls.test.ts index 2a57709ac..8a032ad1c 100644 --- a/packages/e2e/tests/unit/urls.test.ts +++ b/packages/e2e/tests/unit/urls.test.ts @@ -95,6 +95,50 @@ describe('resolveNavigationUrl', () => { expect(() => resolveNavigationUrl('data:text/html,x', base)).toThrow(/scheme/); expect(() => resolveNavigationUrl('javascript:alert(1)', base)).toThrow(/scheme/); }); + + it.each([ + ['view-source:file:///etc/passwd', 'view-source:'], + ['view-source:http://localhost:3000/', 'view-source:'], + ['VIEW-SOURCE:file:///etc/passwd', 'view-source:'], + [' view-source:file:///etc/passwd', 'view-source:'], + ['\tview-source:file:///etc/passwd\n', 'view-source:'], + ['view-\tsource:file:///etc/passwd', 'view-source:'], + ['FiLe:///etc/passwd', 'file:'], + [' file:///etc/passwd', 'file:'], + ['blob:http://localhost:3000/0b7c4c1e', 'blob:'], + ['filesystem:http://localhost:3000/temporary/x', 'filesystem:'], + ['chrome://version', 'chrome:'], + ['chrome-extension://abcdefghijklmnop/page.html', 'chrome-extension:'], + ['devtools://devtools/bundled/inspector.html', 'devtools:'], + ['about:blank#x', 'about:'], + ['about:blank?x', 'about:'], + ['about:srcdoc', 'about:'], + ['about:version', 'about:'], + ['ftp://example.test/x', 'ftp:'], + ['ws://localhost:3000/socket', 'ws:'], + ['myapp://orders/42', 'myapp:'], + ])('denies %j: only http(s) is navigable', (input, scheme) => { + expect(() => resolveNavigationUrl(input, base)).toThrowError( + expect.objectContaining({ code: 'POLICY_DENIED', message: `forbidden URL scheme: ${scheme}` }), + ); + expect(() => resolveNavigationUrl(input, undefined)).toThrowError(expect.objectContaining({ code: 'POLICY_DENIED' })); + }); + + it('reads a percent-encoded scheme as a path, not a scheme', () => { + expect(resolveNavigationUrl('view-source%3Afile:///etc/passwd', base).url).toBe('http://localhost:3000/app/view-source%3Afile:///etc/passwd'); + expect(resolveNavigationUrl('%66ile:///etc/passwd', base).url).toBe('http://localhost:3000/app/%66ile:///etc/passwd'); + }); + + it('admits exactly about:blank, which loads nothing', () => { + expect(resolveNavigationUrl('about:blank', base).url).toBe('about:blank'); + expect(resolveNavigationUrl(' ABOUT:blank ', undefined).url).toBe('about:blank'); + }); + + it('admits http(s) in any case and with surrounding whitespace', () => { + expect(resolveNavigationUrl('HTTPS://Other.test/x', base).url).toBe('https://other.test/x'); + expect(resolveNavigationUrl(' http://localhost:3000/a ', base).url).toBe('http://localhost:3000/a'); + expect(resolveNavigationUrl('//other.test/x', base).url).toBe('http://other.test/x'); + }); }); describe('siteOf and sameSite', () => { diff --git a/packages/mobile/src/device.ts b/packages/mobile/src/device.ts index 62688d4bc..2749f01f2 100644 --- a/packages/mobile/src/device.ts +++ b/packages/mobile/src/device.ts @@ -85,8 +85,8 @@ export interface Device { * a web link the app does not claim goes to Safari, and the next * observation brings the app back. Android starts the link on that * package. Without an app, Android lets the OS route the link; iOS needs - * one (`INVALID_ARGUMENT` without). `file:`, `data:`, and `javascript:` - * links are `POLICY_DENIED`. + * one (`INVALID_ARGUMENT` without). `file:`, `data:`, `javascript:`, + * `view-source:`, `blob:`, and `filesystem:` links are `POLICY_DENIED`. */ openLink(url: string, options?: { app?: string }): Promise; /** diff --git a/packages/mobile/src/links.ts b/packages/mobile/src/links.ts index 393da65d4..413bbd0f8 100644 --- a/packages/mobile/src/links.ts +++ b/packages/mobile/src/links.ts @@ -5,13 +5,15 @@ * * The runner's own rule (`resolveNavigationUrl`, reached by a fixture as * `context.app.resolveUrl`) first demands an app URL, which a device engine - * never declares, so the same small check lives here rather than behind a - * gate that would refuse every link. + * never declares, and admits http(s) alone, where a device link is any app's + * custom scheme. So the check here is a list of refused schemes: the ones + * that read local content or inline a document, and the browser wrappers + * (`view-source:file:...`, `blob:`, `filesystem:`) that carry one inside. */ import { ConfigurationError, TestError } from 'e2e/engine'; -const FORBIDDEN_PROTOCOLS = new Set(['file:', 'data:', 'javascript:']); +const FORBIDDEN_PROTOCOLS = new Set(['file:', 'data:', 'javascript:', 'view-source:', 'blob:', 'filesystem:']); /** * What agent-device opens as a URL instead of resolving as an app: a @@ -34,8 +36,9 @@ function denyForbiddenScheme(protocol: string): void { /** * Parses a link a test asked to open. A string that is not an absolute URL - * is `INVALID_ARGUMENT`; a `file:`, `data:`, or `javascript:` link is - * `POLICY_DENIED`, as it is for `app.open` on the web. + * is `INVALID_ARGUMENT`; a `file:`, `data:`, `javascript:`, `view-source:`, + * `blob:`, or `filesystem:` link is `POLICY_DENIED`, as it is for `app.open` + * on the web. */ export function linkTarget(input: string): URL { let url: URL; @@ -54,8 +57,8 @@ export function linkTarget(input: string): URL { /** * Refuses a link handed to `openApp`, which opens an app by bundle id, * package, or display name. agent-device opens any `scheme:rest` string as - * a URL, so the check runs before the device sees it: a `file:`, `data:`, - * or `javascript:` link is `POLICY_DENIED` as it is for `openLink`, any + * a URL, so the check runs before the device sees it: a link with a + * forbidden scheme is `POLICY_DENIED` as it is for `openLink`, any * other link is `INVALID_ARGUMENT` pointing at `openLink`. The message * never echoes the link, whose query may carry a magic-link token. */ diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 1ce681502..af68d699c 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -1553,7 +1553,7 @@ describe('device fixture', () => { await openAttempt(h); const device = fixture(h); const before = h.fake.calls.length; - for (const denied of ['file:///etc/passwd', 'data:text/html,hi', 'javascript:alert(1)']) { + for (const denied of ['file:///etc/passwd', 'data:text/html,hi', 'javascript:alert(1)', 'view-source:file:///etc/passwd']) { await expect(device.openLink(denied)).rejects.toMatchObject({ code: 'POLICY_DENIED' }); } for (const malformed of ['orders/42', '', 'https://']) { @@ -1567,7 +1567,7 @@ describe('device fixture', () => { await openAttempt(h); const device = fixture(h); const before = h.fake.calls.length; - for (const denied of ['file:///etc/passwd', 'data:text/html,hi', 'javascript:alert(1)']) { + for (const denied of ['file:///etc/passwd', 'data:text/html,hi', 'javascript:alert(1)', 'view-source:file:///etc/passwd']) { await expect(device.openApp(denied)).rejects.toMatchObject({ code: 'POLICY_DENIED' }); } for (const link of ['https://example.com/verify', 'myapp://orders/42']) { diff --git a/packages/mobile/tests/unit/links.test.ts b/packages/mobile/tests/unit/links.test.ts index 17af9a69b..7460e2d09 100644 --- a/packages/mobile/tests/unit/links.test.ts +++ b/packages/mobile/tests/unit/links.test.ts @@ -7,6 +7,23 @@ describe('links', () => { expect(linkTarget('HTTPS://Example.com/verify').href).toBe('https://example.com/verify'); }); + it('refuses schemes that read local content or wrap a URL that does, in any case or padding', () => { + for (const denied of [ + 'file:///etc/passwd', + 'data:text/html,hi', + 'javascript:alert(1)', + 'view-source:file:///etc/passwd', + 'VIEW-SOURCE:file:///etc/passwd', + ' view-source:https://example.com/', + 'view-\tsource:file:///etc/passwd', + 'blob:https://example.com/0b7c4c1e', + 'filesystem:https://example.com/temporary/x', + ]) { + expect(() => linkTarget(denied), denied).toThrowError(expect.objectContaining({ code: 'POLICY_DENIED' })); + expect(() => assertAppId(denied.replace(/\s/g, '')), denied).toThrowError(expect.objectContaining({ code: 'POLICY_DENIED' })); + } + }); + it('lets an app id through to openApp and refuses a link, forbidden schemes first', () => { for (const app of ['Settings', 'com.apple.Preferences', 'com.android.settings', 'Notes: Pro', ' Clock ']) { expect(() => assertAppId(app)).not.toThrow(); diff --git a/packages/mobile/tests/unit/tools.test.ts b/packages/mobile/tests/unit/tools.test.ts index 7848b0e2b..7bb8f2910 100644 --- a/packages/mobile/tests/unit/tools.test.ts +++ b/packages/mobile/tests/unit/tools.test.ts @@ -92,7 +92,7 @@ describe('agent tool pack', () => { messages: [], }); const before = fake.calls.length; - for (const denied of ['file:///etc/passwd', 'data:text/html,hi', 'javascript:alert(1)']) { + for (const denied of ['file:///etc/passwd', 'data:text/html,hi', 'javascript:alert(1)', 'view-source:file:///etc/passwd']) { await expect(open(denied)).rejects.toMatchObject({ code: 'POLICY_DENIED' }); } for (const link of ['https://example.com/verify', 'myapp://orders/42']) { diff --git a/packages/web/src/browser.ts b/packages/web/src/browser.ts index 4f9757bfc..351621fb1 100644 --- a/packages/web/src/browser.ts +++ b/packages/web/src/browser.ts @@ -15,6 +15,7 @@ import type { Download, Response, Route } from 'playwright'; import type { ActionOptions, Expectable, JsonValue, Locator, Screen, TextMatch } from 'e2e'; import { + ConfigurationError, Deadline, describePattern, EngineError, @@ -449,11 +450,15 @@ export function createBrowserFixture(surface: PlaywrightSurface, context: Engine setCookies(cookies) { const scheme = new URL(baseHref()).protocol === 'https:' ? 'https' : 'http'; // The harness's URL rule decides which cookie targets are admitted; a - // domain cookie is checked as the origin it would be sent to. + // domain cookie is checked as the origin it would be sent to. The rule + // admits `about:blank` for navigation, which holds no cookie. for (const cookie of cookies) { - context.app.resolveUrl( + const target = context.app.resolveUrl( cookie.url === undefined ? `${scheme}://${cookie.domain.replace(/^\./, '')}` : cookie.url, ); + if (!/^https?:/.test(target)) { + throw new ConfigurationError('POLICY_DENIED', `cookie URL must be http(s): ${target}`); + } } return surface.guard(context.operation(), 'setCookies', async () => { await surface.requireContext().addCookies( diff --git a/packages/web/src/observation.ts b/packages/web/src/observation.ts index eef5ae2b9..4e21e3c44 100644 --- a/packages/web/src/observation.ts +++ b/packages/web/src/observation.ts @@ -228,7 +228,7 @@ async function captureInto( * * A `data:` document is *not* admitted, even though its bytes are written by the * page that embeds it. It has an opaque origin rather than an inherited one, and - * navigation denies the scheme by name alongside `file:` and `javascript:`. + * navigation admits only http(s) and `about:blank`. */ function isOnSiteFrame(url: string, site: string | undefined): boolean { if (url === '' || url === 'about:blank' || url === 'about:srcdoc') return true; diff --git a/skills/e2e/references/debugging.md b/skills/e2e/references/debugging.md index abb9176b3..8b2ef72c4 100644 --- a/skills/e2e/references/debugging.md +++ b/skills/e2e/references/debugging.md @@ -65,7 +65,7 @@ jq '.run.results[] | select(.selected and .status != "passed") | .attempts[-1] | `AUTOMATION_UNSUPPORTED` | The step needs an interaction the engine's toolset lacks (drag on a device, say); blocked, exit 1 | Do that step with `screen` actions, or run the test on a target that supports it | | `STEP_TIMEOUT`, `STEP_BUDGET_EXHAUSTED` | Goal too big or ambiguous, or a slow provider | Split the goal, use on-screen wording, add vocabulary via `agents..context` or `agentContext`; raise `maxSteps` or `maxModelCalls` (budget), the config `timeout` (`act`), `judgmentTimeout` (judgments); `--debug` | | `CONTEXT_OVERFLOW` | Screen plus step history did not fit the model's context window (`act` after one shrink-and-retry; a judgment on the first overflow) | Lower `agents..maxObservationBytes`, split the step, or pick a larger-window model | -| `POLICY_DENIED` | A `file:`, `data:`, or `javascript:` URL; a password `Secret` into a non-password sink; reading a secure field, `toHaveValue`, `toHaveText`, `toContainText`, `toHaveAttribute` included; `app.screenshot()` after a secret fill | http(s) only; passwords into password inputs only; assert the outcome, not the value; screenshot before filling secrets | +| `POLICY_DENIED` | A URL whose scheme is not `http:` or `https:` (`file:`, `view-source:`, `data:`); a password `Secret` into a non-password sink; reading a secure field, `toHaveValue`, `toHaveText`, `toContainText`, `toHaveAttribute` included; `app.screenshot()` after a secret fill | http(s) or `about:blank` only; passwords into password inputs only; assert the outcome, not the value; screenshot before filling secrets | | `UNSUPPORTED_CAPABILITY` | A fixture the engine lacks (`browser` on a device), `schema` or `vision` on `act`, an action the surface lacks | Declare `requires: ['browser']`; drop the option or action | | `SESSION_UNAVAILABLE`, `SESSION_CONTRACT` | `session: 'x'` with no setup saving `x`; a setup that skipped a declared name. `SESSION_MISMATCH`, `SESSION_EXPIRED`, `SESSION_INVALID`: the stored session is another run's or app's, expired, or corrupt | Add or fix the `test.setup`; rerun it | | `ONLY_IN_CI` | `test.only` reached CI | Remove it | From 7f9ee108dbdcbfab43cff2cdd9435b043a7035d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?= Date: Sat, 3 Oct 2026 13:20:25 +0200 Subject: [PATCH 15/50] fix(web): read visibility as rendered, null box, checkable values, embedded names (#728) * fix(web): read visibility as rendered, null box, checkable values, embedded names * fix(web): narrow visible queries by the reader's hidden state * fix(web): drop direct text a content-visibility hidden element skips * docs(changeset): note the display contents divergence from Playwright --- .changeset/web-visibility-reads.md | 5 + docs/migrate/playwright.mdx | 3 +- docs/reference/screen.mdx | 23 ++- packages/e2e/src/types.ts | 2 +- packages/web/src/in-page/read-semantics.ts | 187 ++++++++++++++---- packages/web/src/locators.ts | 28 ++- packages/web/src/observation.ts | 4 +- packages/web/src/read-node.ts | 11 +- packages/web/src/selector-engines.ts | 2 + packages/web/src/shown-selector.ts | 31 +++ packages/web/tests/conformance/expected.txt | 9 - packages/web/tests/helpers/fixture-app.ts | 14 +- .../web/tests/integration/lifecycle.test.ts | 36 ++-- .../integration/read-node-checked.test.ts | 32 ++- .../tests/integration/read-node-names.test.ts | 76 +++++++ .../web/tests/integration/visibility.test.ts | 75 ++++++- packages/web/tests/unit/cdp-selectors.test.ts | 2 +- packages/web/tests/unit/locators.test.ts | 16 +- skills/e2e/references/writing-tests.md | 4 +- 19 files changed, 451 insertions(+), 109 deletions(-) create mode 100644 .changeset/web-visibility-reads.md create mode 100644 packages/web/src/shown-selector.ts diff --git a/.changeset/web-visibility-reads.md b/.changeset/web-visibility-reads.md new file mode 100644 index 000000000..e19ffbb02 --- /dev/null +++ b/.changeset/web-visibility-reads.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/web": patch +--- + +Web visibility and reads follow what renders, as Playwright reads them: an `aria-hidden` node that paints is visible, so `toBeHidden()` on a decorative spinner waits until it leaves; content under `content-visibility: hidden` is hidden; a child that sets `visibility: visible` under a hidden parent is visible and in the agent's observation; text under a `display: contents` element with `visibility: hidden` is hidden, where Playwright calls it visible, since it paints nothing. `boundingBox()` is `null` for a node with no layout box. `inputValue()` and `toHaveValue` read a checkbox's or radio's value (`on` by default) whatever its checked state. An accessible name reads an embedded control's value: `` is "Flash the screen 3 times". diff --git a/docs/migrate/playwright.mdx b/docs/migrate/playwright.mdx index cef738f82..0fab5b482 100644 --- a/docs/migrate/playwright.mdx +++ b/docs/migrate/playwright.mdx @@ -139,7 +139,8 @@ what to do today. | `fill(value)` | the same; also takes a `Secret` | same | | `pressSequentially(text, { delay })` | the same; keystrokes through the platform keyboard, a `Secret` is refused | same | | `press(key)` | the same; modifiers join with `+`, see the [key grammar](/reference/engine#keys) | same | -| `clear()`, `check()`, `uncheck()`, `focus()`, `hover()`, `selectOption()`, `setInputFiles()`, `dragTo()` | the same | same | +| `clear()`, `check()`, `uncheck()`, `focus()`, `hover()`, `setInputFiles()`, `dragTo()` | the same | same | +| `selectOption('pro')` | `selectOption('Pro')`: a string matches the option's label only, where Playwright matches its value or its label; `selectOption({ value: 'pro' })` for the value | by design | | `selectOption(['a', 'b'])` | not yet; one option per call. An array is `INVALID_ARGUMENT` | missing | | `setInputFiles({ name, buffer })` | not yet; paths only, resolved from the project root | missing | | `scrollIntoViewIfNeeded()` | `scrollIntoView()` | renamed | diff --git a/docs/reference/screen.mdx b/docs/reference/screen.mdx index 10f33adda..53e768724 100644 --- a/docs/reference/screen.mdx +++ b/docs/reference/screen.mdx @@ -107,7 +107,14 @@ await screen.getByTestId('memory-list', { visible: true }).getByRole('listitem') ``` `visible` uses the same visibility check as `toBeVisible()`, `isVisible()`, -and `waitFor()`. It works with scopes, `filter`, `first`, `last`, and `nth`: +and `waitFor()`: what renders, as Playwright reads it. `display: none`, +`visibility: hidden`, a box with no size, a closed `
` body, and +content under `content-visibility: hidden` are hidden; `aria-hidden` is not, +since a decorative spinner marked with it still paints, so +`toBeHidden()` on one waits until it leaves. A child that sets +`visibility: visible` under a hidden parent is visible. On the web, text +directly under a `display: contents` element with `visibility: hidden` is +hidden, where Playwright calls it visible. It works with scopes, `filter`, `first`, `last`, and `nth`: `getByText('Save', { visible: true }).first()` is the first visible match. Prefer it over `browser.locator('p:visible')`, which ties the test to CSS. @@ -407,6 +414,12 @@ a heading; it does not throw. `toHaveText`, `toContainText`, and `toHaveAttribute`. `isEnabled`, `isDisabled`, `isChecked`, and `boundingBox` are allowed there. +`boundingBox()` is `null` for a node that lays out no box (`display: none` +on it or an ancestor), and a box for one `visibility: hidden` keeps. +`inputValue()` and `toHaveValue` on a checkbox or radio read its value +attribute, `on` when it has none, whatever its checked state; `isChecked()` +reads the state. + `isHidden` is `!isVisible()` and `isDisabled` is `!isEnabled()`, with the same strictness: `isHidden` is `true` for zero matches, `isDisabled` is `LOCATOR_NOT_FOUND` there, and both are `LOCATOR_AMBIGUOUS` on several. @@ -529,7 +542,7 @@ export interface TextMatchOptions { /** * Role query options. A role query never matches a node hidden from the * accessibility tree, on every engine; `visible` (inherited) is the one knob - * that narrows the other query kinds the same way. + * that narrows the other query kinds, to the nodes `toBeVisible()` accepts. */ export interface RoleOptions extends TextMatchOptions { /** Accessible name filter. */ @@ -631,8 +644,10 @@ export type ScrollDirection = 'up' | 'down' | 'left' | 'right'; export type Momentum = 'none' | 'slow' | 'fast'; ``` -A role query never matches a node hidden from the accessibility tree. -`visible` applies the same rule to the other query kinds. See +A role query never matches a node hidden from the accessibility tree: one +that does not render, or one under `aria-hidden`. `visible` narrows the +other query kinds to the nodes that render, the check `toBeVisible()` makes, +so an `aria-hidden` node that still paints counts as visible there. See [Hidden duplicates](#hidden-duplicates). `img` is an alias, not a role: `getByRole('img', { name })` builds the same diff --git a/packages/e2e/src/types.ts b/packages/e2e/src/types.ts index 86c2e04e5..7bb38ae26 100644 --- a/packages/e2e/src/types.ts +++ b/packages/e2e/src/types.ts @@ -334,7 +334,7 @@ export interface TextMatchOptions { /** * Role query options. A role query never matches a node hidden from the * accessibility tree, on every engine; `visible` (inherited) is the one knob - * that narrows the other query kinds the same way. + * that narrows the other query kinds, to the nodes `toBeVisible()` accepts. */ export interface RoleOptions extends TextMatchOptions { /** Accessible name filter. */ diff --git a/packages/web/src/in-page/read-semantics.ts b/packages/web/src/in-page/read-semantics.ts index 779e1f6a8..7cb49686d 100644 --- a/packages/web/src/in-page/read-semantics.ts +++ b/packages/web/src/in-page/read-semantics.ts @@ -15,6 +15,7 @@ import type { RawNodeData, RawObservedNode, SemanticMode, SemanticResult } from * `mode.kind === 'node'` reads exactly one element for locator reads, in * the selector engine's task that found it (`read-selector.ts`, * `label-selector.ts`), so the element is always in its document. + * `mode.kind === 'hidden'` answers that one element's `states.hidden` alone. * `mode.kind === 'tree'` walks the subtree for one agent observation and * returns live element handles aligned with the flattened node list. The two * modes also project nodes differently; those differences are data (see @@ -61,8 +62,10 @@ export const readSemanticsFunction = ( /** * How the active mode projects one node, expressed as data so `describe` * stays branch-free. Tree mode is the model-bound projection: bounded text, - * a bounded attribute projection, hrefs reduced to origin+path, and the root - * document named by its title. Node mode is the full locator-read surface. + * a bounded attribute projection, hrefs reduced to origin+path, the root + * document named by its title, and no value on a checkbox or radio, whose + * value attribute is an app token its checked state already says more + * than. Node mode is the full locator-read surface. */ const projection: { attributes: readonly string[] | null; @@ -71,6 +74,7 @@ export const readSemanticsFunction = ( redactHref: boolean; directTextOnly: boolean; documentRoot: boolean; + checkableValue: boolean; } = options.mode.kind === 'tree' ? { @@ -80,6 +84,7 @@ export const readSemanticsFunction = ( redactHref: true, directTextOnly: true, documentRoot: true, + checkableValue: false, } : { attributes: null, @@ -88,6 +93,7 @@ export const readSemanticsFunction = ( redactHref: false, directTextOnly: false, documentRoot: false, + checkableValue: true, }; /** @@ -286,15 +292,16 @@ export const readSemanticsFunction = ( * `hiddenAllowed` when that traversal began at a hidden target, whose whole * subtree then counts (2A); `visited` holds every element already read, so a * descendant that references its ancestor, or two that reference each other, - * contribute once. `generated` is set for an accessible name, which reads - * CSS generated content (`::before`, `::after`) as accname does, and unset - * for label text, which Playwright's `getByLabel` reads without it. + * contribute once. `accessible` is set for an accessible name, which reads + * CSS generated content (`::before`, `::after`) and an embedded control's + * value as accname does, and unset for label text, which Playwright's + * `getByLabel` reads without either. */ interface NameWalk { readonly inReference: boolean; readonly hiddenAllowed: boolean; readonly visited: Set; - readonly generated: boolean; + readonly accessible: boolean; } /** Computed style of any element, SVG included, or undefined in a document with no view. */ @@ -302,11 +309,11 @@ export const readSemanticsFunction = ( el.ownerDocument.defaultView?.getComputedStyle(el); /** A walk from the top, with the elements that may not contribute again already visited. */ - const nameWalk = (visited: readonly Element[], generated: boolean): NameWalk => ({ + const nameWalk = (visited: readonly Element[], accessible: boolean): NameWalk => ({ inReference: false, hiddenAllowed: false, visited: new Set(visited), - generated, + accessible, }); /** @@ -458,7 +465,7 @@ export const readSemanticsFunction = ( inReference: true, hiddenAllowed: isReferenceHidden(target), visited: walk.visited, - generated: walk.generated, + accessible: walk.accessible, }), ); } @@ -466,14 +473,72 @@ export const readSemanticsFunction = ( return contributions.map((text) => text.replace(/\s+/g, ' ').trim()).filter((name) => name !== ''); }; + /** + * What an embedded control contributes to a name computed through another + * element (accname 2C), by the rules Playwright's role selector applies, so + * `` is the button + * "Flash the screen 3 times": a textbox or searchbox its current value (an + * editable element its text), a combobox or listbox its selected options' + * names (a native select with none selected its first option, a text field + * with none its value), and a range widget its `aria-valuetext`, else its + * `aria-valuenow`, else its `value` attribute. A menu contributes nothing, + * and neither does a secure field, whose value no name may carry. Null for + * any other element, and for one its own `aria-labelledby` names, which is + * named, not embedded. + */ + const embeddedControlNameOf = (el: Element, walk: NameWalk): string | null => { + const role = implicitRole(el); + if (role === null) return null; + if (el.id !== '' && (el.getAttribute('aria-labelledby') ?? '').trim().split(/\s+/).indexOf(el.id) !== -1) return null; + if (el.matches(options.secureFieldSelector)) return ''; + if (role === 'textbox' || role === 'searchbox') { + return el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement ? el.value : el.textContent ?? ''; + } + if (role === 'combobox' || role === 'listbox') { + let selected: Element[]; + if (el instanceof HTMLSelectElement) { + selected = Array.from(el.selectedOptions); + if (selected.length === 0 && el.options.length > 0) selected.push(el.options[0]!); + } else { + const listbox = role === 'combobox' ? ariaOwnedOf(el).find((owned) => implicitRole(owned) === 'listbox') : el; + selected = listbox === undefined + ? [] + : ariaOwnedOf(listbox).filter((owned) => owned.getAttribute('aria-selected') === 'true' && implicitRole(owned) === 'option'); + } + if (selected.length === 0 && el instanceof HTMLInputElement) return el.value; + return selected.map((option) => contentNameOf(option, styleOf(option), walk)).join(' '); + } + if (role === 'progressbar' || role === 'scrollbar' || role === 'slider' || role === 'spinbutton' || role === 'meter') { + if (el.hasAttribute('aria-valuetext')) return el.getAttribute('aria-valuetext') ?? ''; + if (el.hasAttribute('aria-valuenow')) return el.getAttribute('aria-valuenow') ?? ''; + return el.getAttribute('value') ?? ''; + } + return role === 'menu' ? '' : null; + }; + + /** + * Every element under `el` and every element its `aria-owns` names, with + * theirs: what a composite widget owns, as Playwright's role selector + * collects a combobox's listbox and a listbox's options. + */ + const ariaOwnedOf = (el: Element): Element[] => { + const owned = Array.from(el.querySelectorAll('*')); + for (const id of (el.getAttribute('aria-owns') ?? '').trim().split(/\s+/)) { + const target = id === '' ? null : referencedElementOf(el, id); + if (target !== null) owned.push(target, ...Array.from(target.querySelectorAll('*'))); + } + return owned; + }; + /** * What one element contributes to a name computed through another: as an * aria-labelledby target (accname 2B) or as a descendant read for name from - * content (2F). Its own references, else its `aria-label`, its `alt`, its - * children, then its `title`. So `` is - * the button "Search", an icon whose only child is + * content (2F). Its own references, else its value as an embedded control, + * its `aria-label`, its `alt`, its children, then its `title`. So + * `` is the button "Search", an icon + * whose only child is * `Close` is "Close", - * and a hidden `` a button references still names it. A form + * and a hidden `` a button references still names it. Any other form * control's content is its value, not label text, so it contributes nothing. * The title is the fallback only for empty content, untrimmed as Playwright * reads it: the spaces a block-level child adds count, so a flex container's @@ -486,6 +551,8 @@ export const readSemanticsFunction = ( if (!walk.hiddenAllowed && isNameHidden(el, style)) return ''; const referenced = referencedNamesOf(el, walk); if (referenced !== null) return referenced.join(' '); + const embedded = walk.accessible ? embeddedControlNameOf(el, walk) : null; + if (embedded !== null) return embedded; const ariaLabel = el.getAttribute('aria-label'); if (ariaLabel !== null && ariaLabel.trim() !== '') return ariaLabel.trim(); const alternative = altOf(el) ?? svgTitleOf(el); @@ -507,7 +574,7 @@ export const readSemanticsFunction = ( * `.next::after { content: " \2192" }` is the button "Go →". */ const childrenNameOf = (el: Element, walk: NameWalk): string => { - let out = walk.generated ? beforeContentOf(el) : ''; + let out = walk.accessible ? beforeContentOf(el) : ''; for (const child of contentChildrenOf(el)) { if (child.nodeType === 3) { out += child.nodeValue ?? ''; @@ -519,7 +586,7 @@ export const readSemanticsFunction = ( const block = child.tagName === 'BR' || (style?.display ?? 'inline') !== 'inline'; out += block ? ` ${token} ` : token; } - return walk.generated ? out + afterContentOf(el) : out; + return walk.accessible ? out + afterContentOf(el) : out; }; /** @@ -552,13 +619,15 @@ export const readSemanticsFunction = ( * its descendants' contributions, hidden and aria-hidden subtrees dropped (a * required-field marker, a decorative glyph), whitespace collapsed. The * element's own attributes are `accessibleName`'s business; this reads what - * is inside it. `generated` reads CSS generated content too, as an - * accessible name does and label text does not. + * is inside it. `accessible` reads CSS generated content and embedded + * controls' values too, as an accessible name does and label text does not. + * `named` is the element the text names when that is not `el` (a control + * read through its `