From 5b6a731ed4f87caaf4fe69232baa0f94ce1d8b19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?=
Date: Mon, 5 Oct 2026 10:40:02 +0200
Subject: [PATCH 01/46] test(e2e): integration suites on the fake engine, cut
to what only integration proves (#821)
* test(e2e): secret tests inspect model input and inflated trace entries
* test(mcp): project tool results and errors pass the ledger; poll instead of sleeping
* test(e2e): runProject without an app url runs on the fake engine
* test(e2e): secret tests fail their deliberate tap in 500 ms, not the 30 s action timeout
* test(e2e): trim agent-act, agent-act-stress, agent-policy duplicates; serial artifact checks move to artifact-store-serial
* test(run): runner lifecycle on the fake engine, interrupts on the worker path, cut duplicates
* test(web): trim web-platform to unique cases and fold the class and URL-pattern suites into its config-file run
* test(e2e): drop integration suites unit and testbed tests already cover
* test(e2e): drop a margin tap, wait for the full pipe instead of 2 s, no console noise
* test(run): parallel workers on the fake engine with event-driven waits, cover a worker-path timeout
* test(e2e): expect.poll scope and expect.soft run on the fake engine
* test(run): run events off the browser where no page is read, drop the sink duplicate
* test(e2e): trim agent, explore, ai-trace, model-usage duplicates; semantic fallback joins agent-fallback-handoff
* test(web): fewer reads against the node replaced every frame
* test(e2e): reporters on the fake engine, fold the boot precondition, trim skip-after-failure duplicates
* test(e2e): drop trace-cache-speed wall-clock gate; its replay check is in trace-cache-replay
* test(e2e): agent-act-verbs pages to row 24, records on the main run, drops duplicate verb and upload checks
* test(e2e): trim agent-trace-cache to integration-only proofs, 500 ms on failing expects
* test(e2e): fold the stale --strict-cache run into the changed-screen suite
* test(e2e): extended fixtures on the fake engine with shorter setup sleeps
* test(e2e): trace-modes proves --trace in one run
* test(mcp): fold mcp-custom into mcp-server on one server, drop duplicate recording and prose checks
* test(e2e): hold four negations instead of seventeen in scripted matchers
* test(config): loader-only cases move to config-load, cli-init runs the CLI once per loader shape, trim managed-process duplicates
* test(e2e): engine contract stalls one observe, drops cases unit tests own
* test(e2e): engine targets keep the hook cases engine-contract lacks, assert the capability code
* test(run): Ctrl-C releases what prepare acquired; exit 3, --shard, and CI defaults through the CLI
* test(e2e): scripted suites keep what unit tests lack, drop the scene nodes only cut cases used
* test(e2e): drop the agent-name and CJK-clip runs unit tests cover
* test(e2e): scene root id is internal to the scripted scene
* test(e2e): keep the no-model hand-off eviction check, a clean start without a session, and slack on mcp session polls
* test(e2e): keep fractional usage, executor model in explore, and --trace off; bound the backpressure wait; never leak APP_URL into fixture runs
---
.../tests/helpers/fixture-pages/gestures.ts | 6 +-
packages/e2e/tests/helpers/run-project.ts | 65 +-
packages/e2e/tests/helpers/scripted-engine.ts | 14 -
packages/e2e/tests/helpers/scripted-scene.ts | 23 +-
.../integration/agent-act-pixel-tools.test.ts | 20 +-
.../integration/agent-act-stress.test.ts | 116 +---
.../tests/integration/agent-act-verbs.test.ts | 134 ++--
.../e2e/tests/integration/agent-act.test.ts | 74 +--
.../tests/integration/agent-ai-trace.test.ts | 33 -
.../agent-fallback-handoff.test.ts | 97 ++-
.../tests/integration/agent-policy.test.ts | 89 +--
.../integration/agent-scroll-settle.test.ts | 1 -
.../agent-semantic-fallback.test.ts | 103 ---
.../integration/agent-trace-cache.test.ts | 412 ++----------
packages/e2e/tests/integration/agent.test.ts | 45 --
.../integration/artifact-store-serial.test.ts | 11 +-
.../e2e/tests/integration/cli-boot.test.ts | 5 +-
.../e2e/tests/integration/cli-init.test.ts | 66 +-
.../e2e/tests/integration/cli-run.test.ts | 147 ++++
.../e2e/tests/integration/config-load.test.ts | 46 +-
.../tests/integration/config-names.test.ts | 18 +-
.../tests/integration/engine-contract.test.ts | 132 +---
.../tests/integration/engine-target.test.ts | 209 +-----
.../integration/expect-poll-scope.test.ts | 75 ++-
.../e2e/tests/integration/expect-soft.test.ts | 42 +-
.../e2e/tests/integration/explore.test.ts | 69 +-
.../integration/extended-fixtures.test.ts | 42 +-
.../integration/interrupt-signals.test.ts | 8 +-
.../tests/integration/list-reporter.test.ts | 45 --
.../tests/integration/managed-process.test.ts | 47 +-
.../tests/integration/matcher-options.test.ts | 124 ----
.../e2e/tests/integration/mcp-custom.test.ts | 250 -------
.../e2e/tests/integration/mcp-server.test.ts | 195 ++++--
.../integration/mcp-session-host.test.ts | 65 +-
.../mcp-stdout-backpressure.test.ts | 5 +-
.../e2e/tests/integration/model-usage.test.ts | 36 +-
.../integration/parallel-workers.test.ts | 103 ++-
.../integration/press-sequentially.test.ts | 78 ---
.../tests/integration/report-markdown.test.ts | 45 --
.../e2e/tests/integration/reporters.test.ts | 38 +-
.../e2e/tests/integration/run-events.test.ts | 89 +--
.../integration/runner-lifecycle.test.ts | 541 +++------------
.../integration/scripted-actions.test.ts | 60 +-
.../integration/scripted-capabilities.test.ts | 72 --
.../integration/scripted-matchers.test.ts | 28 +-
.../integration/scripted-polling.test.ts | 101 +--
.../tests/integration/scripted-reads.test.ts | 90 +--
.../integration/selected-secrets.test.ts | 2 +-
.../integration/skip-after-failure.test.ts | 17 +-
.../integration/trace-cache-replay.test.ts | 86 +--
.../integration/trace-cache-speed.test.ts | 94 ---
.../e2e/tests/integration/trace-modes.test.ts | 38 +-
.../integration/transformed-secrets.test.ts | 33 +-
.../integration/truncated-secrets.test.ts | 5 +-
.../integration/web-class-assertions.test.ts | 106 ---
.../tests/integration/web-platform.test.ts | 625 ++++++------------
.../integration/web-replaced-nodes.test.ts | 6 +-
.../integration/web-url-patterns.test.ts | 107 ---
packages/e2e/tests/unit/config.test.ts | 2 +
59 files changed, 1269 insertions(+), 3966 deletions(-)
delete mode 100644 packages/e2e/tests/integration/agent-semantic-fallback.test.ts
create mode 100644 packages/e2e/tests/integration/cli-run.test.ts
delete mode 100644 packages/e2e/tests/integration/matcher-options.test.ts
delete mode 100644 packages/e2e/tests/integration/mcp-custom.test.ts
delete mode 100644 packages/e2e/tests/integration/press-sequentially.test.ts
delete mode 100644 packages/e2e/tests/integration/report-markdown.test.ts
delete mode 100644 packages/e2e/tests/integration/scripted-capabilities.test.ts
delete mode 100644 packages/e2e/tests/integration/trace-cache-speed.test.ts
delete mode 100644 packages/e2e/tests/integration/web-class-assertions.test.ts
delete mode 100644 packages/e2e/tests/integration/web-url-patterns.test.ts
diff --git a/packages/e2e/tests/helpers/fixture-pages/gestures.ts b/packages/e2e/tests/helpers/fixture-pages/gestures.ts
index e3f95a9ce..e3ba1e89d 100644
--- a/packages/e2e/tests/helpers/fixture-pages/gestures.ts
+++ b/packages/e2e/tests/helpers/fixture-pages/gestures.ts
@@ -41,8 +41,8 @@ export const GESTURE_PAGES: Record = {
-
Page the ledger down to Row 333 and stop there.
- Jump to Row 333
+
Page the ledger down to Row 24 and stop there.
+ Jump to Row 24
@@ -80,7 +80,7 @@ export const GESTURE_PAGES: Record = {
// A windowed list: 400 rows of 40 px exist as data, and only the rows
// inside the container's scrolled window are in the DOM, so the golden
// row is nowhere in the tree until the list is paged down to it.
- const ROWS = 400, ROW_PX = 40, GOLDEN = 333;
+ const ROWS = 400, ROW_PX = 40, GOLDEN = 24;
const ledger = document.getElementById('ledger');
document.getElementById('ledger-spacer').style.height = ROWS * ROW_PX + 'px';
const ledgerState = document.querySelector('output[aria-label="Ledger state"]');
diff --git a/packages/e2e/tests/helpers/run-project.ts b/packages/e2e/tests/helpers/run-project.ts
index 05165a16c..3f1393fcb 100644
--- a/packages/e2e/tests/helpers/run-project.ts
+++ b/packages/e2e/tests/helpers/run-project.ts
@@ -8,6 +8,7 @@ import { web } from '@e2e-dev/web';
import type { ListOptions, ListedPair, RunOptions, RunOutcome } from '../../src/run/runner.ts';
import type { E2EConfig } from '../../src/index.ts';
import { inflateEntry, readZip } from '../../src/internal/zip.ts';
+import { createFakeEngine, FAKE_APP } from './fake-engine.ts';
export type { RunOptions, RunOutcome };
@@ -32,11 +33,23 @@ export function webTarget(name: string, url: string): Target {
return { name, engine: web() as unknown as NonNullable, app: { url } };
}
-/** The web target integration suites run against unless their config names its own. */
-function defaultTargets(appUrl: string): NonNullable {
+/**
+ * The target a suite runs against unless its config names its own: the web
+ * target on the fixture app at `appUrl`, or, for a suite that opens no page
+ * and so passes none, a fake engine that starts no browser.
+ */
+function defaultTargets(appUrl: string | undefined): NonNullable {
+ if (appUrl === undefined) return [{ name: 'fake', platform: 'fake', engine: createFakeEngine({ state: true }).engine, app: FAKE_APP }];
return [webTarget('web', appUrl)];
}
+/** The run environment: the caller's, outside CI, with `APP_URL` the fixture app's URL or unset, never the caller's. */
+function fixtureEnv(appUrl: string | undefined): NodeJS.ProcessEnv {
+ const env: NodeJS.ProcessEnv = { ...process.env, CI: '' };
+ delete env['APP_URL'];
+ return appUrl === undefined ? env : { ...env, APP_URL: appUrl };
+}
+
export interface FixtureProject {
readonly dir: string;
cleanup(): void;
@@ -81,8 +94,9 @@ export function contentsUnder(dir: string): [string, string][] {
}
export interface RunProjectOptions {
- appUrl: string;
- /** Overrides of the fixture config; `targets` defaults to the web target on `appUrl`. */
+ /** The fixture app; without one, the default target is a fake engine and no browser starts. */
+ appUrl?: string;
+ /** Overrides of the fixture config; `targets` defaults to the web target on `appUrl`, or the fake engine without one. */
config?: Partial;
runOptions?: Partial;
}
@@ -109,11 +123,7 @@ export async function runExisting(
// Core knows no engine: a web target is served by the playwright engine
// the test explicitly passes, exactly as a project config would.
rawConfig: { targets: defaultTargets(options.appUrl), ...options.config },
- env: {
- ...process.env,
- APP_URL: options.appUrl,
- CI: '',
- },
+ env: fixtureEnv(options.appUrl),
quiet: true,
...options.runOptions,
});
@@ -128,13 +138,13 @@ export async function listProject(
const { pairs } = await list({
cwd: project.dir,
rawConfig: { targets: defaultTargets(options.appUrl), ...options.config },
- env: { ...process.env, APP_URL: options.appUrl, CI: '' },
+ env: fixtureEnv(options.appUrl),
...options.listOptions,
});
return { pairs, project };
}
-/** Default file-backed config used by worker-path integration tests. */
+/** Default file-backed config used by worker-path integration tests that open the fixture app. */
export function workerConfigSource(workers: number, extra = ''): string {
return `import type { E2EConfig } from 'e2e';
import { web } from '@e2e-dev/web';
@@ -146,6 +156,21 @@ export default {
`;
}
+/**
+ * A file-backed worker-path config on the fake engine, for suites that open
+ * no page: each worker imports the helper and builds its own instance.
+ */
+export function workerFakeConfigSource(workers: number, extra = ''): string {
+ return `import type { E2EConfig } from 'e2e';
+import { createFakeEngine, FAKE_APP } from '../../helpers/fake-engine.ts';
+
+export default {
+ targets: [{ name: 'fake', platform: 'fake', engine: createFakeEngine({ state: true }).engine, app: FAKE_APP }],
+ workers: ${workers},${extra}
+} satisfies E2EConfig;
+`;
+}
+
/**
* Runs a fixture project through the parallel worker path: the config is a
* real file, so the runner schedules units across worker processes.
@@ -155,16 +180,26 @@ export async function runProjectWithConfigFile(
options: RunProjectOptions & { configSource: string },
): Promise<{ outcome: RunOutcome; project: FixtureProject }> {
const project = createProject({ ...files, 'e2e.config.ts': options.configSource });
+ return { outcome: await runExistingWithConfigFile(project, options), project };
+}
+
+/**
+ * Runs an existing project through its own `e2e.config.ts` on the worker
+ * path. Repeated runs share on-disk state, as with `runExisting`.
+ */
+export async function runExistingWithConfigFile(
+ project: FixtureProject,
+ options: Omit,
+): Promise {
const previousAppUrl = process.env['APP_URL'];
- process.env['APP_URL'] = options.appUrl;
+ if (options.appUrl !== undefined) process.env['APP_URL'] = options.appUrl;
try {
- const outcome = await run({
+ return await run({
cwd: project.dir,
- env: { ...process.env, APP_URL: options.appUrl, CI: '' },
+ env: fixtureEnv(options.appUrl),
quiet: true,
...options.runOptions,
});
- return { outcome, project };
} finally {
if (previousAppUrl === undefined) delete process.env['APP_URL'];
else process.env['APP_URL'] = previousAppUrl;
diff --git a/packages/e2e/tests/helpers/scripted-engine.ts b/packages/e2e/tests/helpers/scripted-engine.ts
index c9946bd52..0e7c48069 100644
--- a/packages/e2e/tests/helpers/scripted-engine.ts
+++ b/packages/e2e/tests/helpers/scripted-engine.ts
@@ -207,19 +207,5 @@ export function scene(stage: Stage): ScriptedNode[] {
progress,
save,
growing,
- { id: 'below', role: 'button', name: 'Below the fold', appearsAfterSwipes: 2 },
- { id: 'stale', role: 'button', name: 'Stale', staleOnce: true },
- {
- id: 'flaky',
- role: 'button',
- name: 'Flaky',
- fail: { tap: { code: 'NOT_ACTIONABLE', message: 'covered by an overlay' } },
- },
- {
- id: 'commit',
- role: 'button',
- name: 'Commit',
- fail: { tap: { code: 'ACTION_MAY_HAVE_COMMITTED', message: 'the tap may have landed' } },
- },
];
}
diff --git a/packages/e2e/tests/helpers/scripted-scene.ts b/packages/e2e/tests/helpers/scripted-scene.ts
index 36ba7a858..ba295d66e 100644
--- a/packages/e2e/tests/helpers/scripted-scene.ts
+++ b/packages/e2e/tests/helpers/scripted-scene.ts
@@ -8,21 +8,15 @@
* owns one `Scene` per attempt and forwards its calls.
*/
-import type { LocatorAction, LocatorActionKind, LocatorExpression, SemanticNode } from '../../src/engine/index.ts';
+import type { LocatorAction, LocatorExpression, SemanticNode } from '../../src/engine/index.ts';
import { engineFailure, obj, parseKey, resolveExpression } from './engine-runtime.ts';
/** The stable id of the observation root; `perform(root, swipe)` is the viewport swipe. */
-export const SCENE_ROOT_ID = 'root';
+const SCENE_ROOT_ID = 'root';
type Mutable = { -readonly [K in keyof T]: T[K] };
type StateKey = keyof NonNullable;
-/** How one action kind on one node fails instead of performing. */
-export interface ScriptedFailure {
- readonly code: Parameters[0];
- readonly message?: string;
-}
-
/** One option a `selectOption` picks from. */
export interface ScriptedOption {
readonly label: string;
@@ -45,10 +39,6 @@ export interface ScriptedNode extends Mutable void;
- /** Fails the named action kinds instead of performing them. */
- fail?: Partial>;
- /** The next perform on the node throws a retryable NODE_STALE and gives the node a fresh id. */
- staleOnce?: boolean;
}
/** What a scene can schedule while the attempt runs. */
@@ -240,15 +230,6 @@ export function createScene(build: (stage: Stage) => ScriptedNode[], initialLoca
}
const node = findById(id);
if (node === undefined) throw engineFailure('NODE_STALE', `node ${id} is no longer in the tree`, true);
- if (node.staleOnce === true) {
- node.staleOnce = false;
- node.id = `${node.id}~relocated`;
- throw engineFailure('NODE_STALE', `node ${id} went stale`, true);
- }
- const scripted = node.fail?.[action.kind];
- if (scripted !== undefined) {
- throw engineFailure(scripted.code, scripted.message ?? `${action.kind} on ${node.id} failed with ${scripted.code}`);
- }
apply(node, action);
node.on?.(action, node);
},
diff --git a/packages/e2e/tests/integration/agent-act-pixel-tools.test.ts b/packages/e2e/tests/integration/agent-act-pixel-tools.test.ts
index 10dedb952..811433fd9 100644
--- a/packages/e2e/tests/integration/agent-act-pixel-tools.test.ts
+++ b/packages/e2e/tests/integration/agent-act-pixel-tools.test.ts
@@ -94,7 +94,7 @@ test('stops a streak of failing point verbs', async ({ app, agent }) => {
test('reads a repaint the tree cannot list off the screenshot', async ({ app, agent }) => {
await app.open('/canvas');
- await agent.act('tap the map twice, then the margin twice');
+ await agent.act('tap the map twice, then the margin once');
});
`;
@@ -171,14 +171,14 @@ function actModel(call: LoopCall) {
if (calls <= 5) return [{ toolName: 'select_at', input: { x: 380 + calls * 20, y: 180 + calls * 10, value: 'x' } }];
return conclude('failed');
}
- if (call.prompt.includes('tap the map twice, then the margin twice')) {
+ if (call.prompt.includes('tap the map twice, then the margin once')) {
// After the red pin, the same miss point at CSS (100, 100) twice: the status text repeats, so the
- // listing stands still while the map paints a new tap count. Then the blank margin at CSS (700, 500)
- // twice: nothing there repaints.
+ // listing stands still while the map paints a new tap count. Then the blank margin at CSS (700, 500):
+ // nothing there repaints.
if (calls === 0) return [{ toolName: 'screenshot', input: {} }];
if (calls === 1) return [{ toolName: 'tap_at', input: { x: 180, y: 36 } }];
if (calls <= 3) return [{ toolName: 'tap_at', input: { x: 60, y: 60 } }];
- if (calls <= 5) return [{ toolName: 'tap_at', input: { x: 420, y: 300 } }];
+ if (calls === 4) return [{ toolName: 'tap_at', input: { x: 420, y: 300 } }];
return conclude('passed');
}
return conclude('passed');
@@ -391,16 +391,16 @@ describe('agent.act pixel verbs', () => {
it('reports a moved screenshot under an unchanged listing, and blames the control only when the image stood still too', () => {
expect(resultByTitle(outcome, 'reads a repaint the tree cannot list off the screenshot').status).toBe('passed');
- const turns = turnsOf('tap the map twice, then the margin twice');
+ const turns = turnsOf('tap the map twice, then the margin once');
// The first miss changed the listed status text; the second repeated it, and only the drawn tap count moved.
expect(turns[3]!.lastToolResult).toMatch(/changed #\S+ status \\"Hit\\" text=\\"miss at 100,100\\"/);
expect(turns[4]!.lastToolResult).toContain('listed nodes unchanged; the screenshot changed');
expect(turns[4]!.lastToolResult).not.toContain('had no visible effect');
expect(turns[4]!.lastToolResult).toContain('"type":"file"');
- // The second margin tap changed neither the listing nor a pixel: the same PNG, so the tap is blamed as before.
- expect(turns[6]!.lastToolResult).toContain('had no visible effect');
- expect(turns[6]!.lastToolResult).not.toContain('the screenshot changed');
- expect(turns[6]!.lastToolResult).toContain('"type":"file"');
+ // The margin tap changed neither the listing nor a pixel: the same PNG, so the tap is blamed as before.
+ expect(turns[5]!.lastToolResult).toContain('had no visible effect');
+ expect(turns[5]!.lastToolResult).not.toContain('the screenshot changed');
+ expect(turns[5]!.lastToolResult).toContain('"type":"file"');
});
});
diff --git a/packages/e2e/tests/integration/agent-act-stress.test.ts b/packages/e2e/tests/integration/agent-act-stress.test.ts
index 60933272e..16bda1546 100644
--- a/packages/e2e/tests/integration/agent-act-stress.test.ts
+++ b/packages/e2e/tests/integration/agent-act-stress.test.ts
@@ -11,8 +11,6 @@ import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { installFakeLoopModel, loopCalls, nodeIdFor } from '../helpers/fake-loop-model.ts';
import { resultByTitle, runProject, runProjectWithConfigFile } from '../helpers/run-project.ts';
import type { StepExecutor, StepExecutorContext } from '../../src/agent/executor.ts';
-import { BLOCKABLE_CODES } from '../../src/agent/executor.ts';
-import { AGENT_CODE_TABLE } from '../../src/agent/error.ts';
const CREDS = { admin: { username: 'admin', password: 'admin-pass' } };
@@ -34,7 +32,7 @@ test('editor secret probe', async ({ app, agent }) => {
});
`;
-describe('secret fill policy under a hostile executor', () => {
+describe('secret fill policy under a hostile model', () => {
let app: FixtureApp;
beforeAll(async () => {
@@ -45,37 +43,6 @@ describe('secret fill policy under a hostile executor', () => {
await app?.close();
});
- const run = async (executor: StepExecutor) =>
- runProject(
- { 'tests/secret.e2e.ts': SECRET_SUITE },
- {
- appUrl: app.url,
- config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } }, credentials: CREDS },
- },
- );
-
- it('refuses to fill a secret into a non-secure field', async () => {
- const executor: StepExecutor = {
- name: 'sink-attacker',
- async runStep(context: StepExecutorContext) {
- const observation = await context.observe();
- const email = nodeIdFor(observation.text, /textbox "Email"/);
- await context.actions.typeSecret({ id: email }, 'admin.password');
- return { status: 'passed' as const, summary: 'should never get here' };
- },
- };
- const { outcome, project } = await run(executor);
- try {
- const result = resultByTitle(outcome, 'secret probe');
- expect(result.status).toBe('failed');
- expect(result.attempts.at(-1)!.error?.code).toBe('POLICY_DENIED');
- expect(result.attempts.at(-1)!.error?.message).toContain('purpose');
- } finally {
- project.cleanup();
- }
- }, 120_000);
-
-
it('fills a generic secret into a contenteditable host and refuses a password there', async () => {
const model = installFakeLoopModel((call) => {
const notes = () => nodeIdFor(call.prompt, /textbox "Notes"/);
@@ -220,7 +187,6 @@ test('defiant model', async ({ app, agent }) => {
await agent.act('never conclude', { maxModelCalls: 6, timeout: 30_000 });
});
`;
- const startedMs = Date.now();
const { outcome, project } = await runProject(
{ 'tests/defiant.e2e.ts': suite },
{ appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } } } },
@@ -235,13 +201,12 @@ test('defiant model', async ({ app, agent }) => {
'MODEL_OUTPUT_INVALID',
'STEP_BUDGET_EXHAUSTED',
]).toContain(result.attempts.at(-1)!.error?.code);
- expect(Date.now() - startedMs).toBeLessThan(30_000);
} finally {
project.cleanup();
}
}, 120_000);
- it('runs ten sequential act steps with fresh budgets each', async () => {
+ it('runs sequential act steps with fresh budgets each', async () => {
const executor: StepExecutor = {
name: 'counter-executor',
async runStep(context: StepExecutorContext) {
@@ -251,24 +216,24 @@ test('defiant model', async ({ app, agent }) => {
return { status: 'passed' as const, summary: 'tapped once' };
},
};
- const calls = Array.from({ length: 10 }, () => " await agent.act('tap once');").join('\n');
+ const calls = Array.from({ length: 3 }, () => " await agent.act('tap once');").join('\n');
const suite = `import { test, expect } from 'e2e';
-test('ten steps', async ({ app, agent, screen }) => {
+test('three steps', async ({ app, agent, screen }) => {
await app.open();
${calls}
- await expect(screen.getByRole('status')).toHaveText('10');
+ await expect(screen.getByRole('status')).toHaveText('3');
});
`;
const { outcome, project } = await runProject(
- { 'tests/ten.e2e.ts': suite },
+ { 'tests/three.e2e.ts': suite },
{ appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } },
);
try {
- const result = resultByTitle(outcome, 'ten steps');
+ const result = resultByTitle(outcome, 'three steps');
expect(result.status).toBe('passed');
const steps = result.attempts.at(-1)!.steps.filter((s) => s.api === 'agent.act');
- expect(steps).toHaveLength(10);
+ expect(steps).toHaveLength(3);
for (const step of steps) expect(step.metrics!.actionSteps).toBe(1);
} finally {
project.cleanup();
@@ -276,71 +241,6 @@ ${calls}
}, 120_000);
});
-describe('taxonomy and run derivation under mixed outcomes', () => {
- let app: FixtureApp;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- });
-
- afterAll(async () => {
- await app?.close();
- });
-
- it('every blockable code names a category; others name none', () => {
- for (const code of BLOCKABLE_CODES) {
- expect(AGENT_CODE_TABLE[code].blockedCategory).toBeDefined();
- }
- expect(AGENT_CODE_TABLE.CANCELLED.blockedCategory).toBeUndefined();
- expect(AGENT_CODE_TABLE.ACTION_FAILED.blockedCategory).toBeUndefined();
- expect(AGENT_CODE_TABLE.SEED_DATA_MISSING.blockedCategory).toBe('seed_data');
- expect(AGENT_CODE_TABLE.AUTH_CREDENTIAL_INVALID.blockedCategory).toBe('credentials');
- });
-
- it('blocks the run on a new code, but one real failure keeps it failed', async () => {
- const suite = `import { test } from 'e2e';
-
-test('blocked by seed data', async ({ app, agent }) => {
- await app.open();
- await agent.act('needs seed data');
-});
-
-test('genuinely broken', async ({ app, agent }) => {
- await app.open();
- await agent.act('the app is broken');
-});
-`;
- const executor: StepExecutor = {
- name: 'mixed-executor',
- async runStep(context: StepExecutorContext) {
- if (context.step.instruction.includes('seed')) {
- return {
- status: 'blocked' as const,
- summary: 'no fixtures exist in this environment',
- errorCode: 'SEED_DATA_MISSING' as const,
- };
- }
- return { status: 'failed' as const, summary: 'the button does nothing' };
- },
- };
- const { outcome, project } = await runProject(
- { 'tests/mixed.e2e.ts': suite },
- { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } },
- );
- try {
- const blocked = resultByTitle(outcome, 'blocked by seed data');
- expect(blocked.attempts.at(-1)!.error?.code).toBe('SEED_DATA_MISSING');
- expect(
- blocked.attempts.at(-1)!.steps.find((s) => s.api === 'agent.act')?.status,
- ).toBe('blocked');
- // One genuine product failure outvotes the blocked result.
- expect(outcome.report.run.status).not.toBe('blocked');
- } finally {
- project.cleanup();
- }
- }, 120_000);
-});
-
describe('config-file executor across worker processes', () => {
let app: FixtureApp;
diff --git a/packages/e2e/tests/integration/agent-act-verbs.test.ts b/packages/e2e/tests/integration/agent-act-verbs.test.ts
index a87661096..1214f2e19 100644
--- a/packages/e2e/tests/integration/agent-act-verbs.test.ts
+++ b/packages/e2e/tests/integration/agent-act-verbs.test.ts
@@ -12,7 +12,7 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { imagePointFor, installFakeLoopModel, loopCalls, nodeIdFor, type LoopCall, type LoopToolCall } from '../helpers/fake-loop-model.ts';
-import { createProject, resultByTitle, runExisting, runProject, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
+import { createProject, resultByTitle, runExisting, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
/** One agentic flow: what the test asks, how the scripted model answers, and the locator check that decides it. */
interface Flow {
@@ -20,7 +20,7 @@ interface Flow {
readonly instruction: string;
/** The test body's check after the step, as source; empty for a flow whose step is expected to fail. */
readonly check: string;
- /** Whether the replay pass records and replays this flow too. */
+ /** Whether the second, cached pass runs this flow and replays it. */
readonly replays?: true;
/** The tool calls for the turn with `calls` results so far; undefined concludes the step. */
readonly script: (calls: number, call: LoopCall) => readonly LoopToolCall[] | undefined;
@@ -105,14 +105,6 @@ const FLOWS: readonly Flow[] = [
verdict: 'failed',
script: (calls, call) => (calls === 0 ? [{ toolName: 'upload', input: { target: nodeIdFor(call.prompt, /"Attachment"/), files: ['.env'] } }] : undefined),
},
- {
- title: 'refuses a file outside the project',
- instruction: 'attach a file from outside the project',
- check: '',
- verdict: 'failed',
- script: (calls, call) =>
- calls === 0 ? [{ toolName: 'upload', input: { target: nodeIdFor(call.prompt, /"Attachment"/), files: ['../outside.txt'] } }] : undefined,
- },
{
title: 'scrolls a listed node into view',
instruction: 'scroll to the footnote',
@@ -126,7 +118,7 @@ const FLOWS: readonly Flow[] = [
check: `await expect(screen.getByLabel('Ledger state')).toHaveText('golden in view');`,
replays: true,
script: (calls, call) =>
- calls === 0 ? [{ toolName: 'scroll_to', input: { text: 'Row 333', target: nodeIdFor(call.prompt, /list "Ledger"/) } }] : undefined,
+ calls === 0 ? [{ toolName: 'scroll_to', input: { text: 'Row 24', target: nodeIdFor(call.prompt, /list "Ledger"/) } }] : undefined,
},
{
title: 'selects one word with a repeated press and sees the selection',
@@ -205,7 +197,10 @@ function actModel(call: LoopCall) {
return [{ toolName: 'complete_step', input: { status: flow?.verdict ?? 'passed', summary: call.lastToolResult.slice(0, 1_500) || 'done' } }];
}
-const VERB_TOOLS = ['hover', 'hover_at', 'double_tap', 'long_press', 'right_click', 'drag', 'check', 'upload', 'scroll_to', 'back'];
+/** A `grep` pattern that selects exactly the test with this title. */
+function exactTitle(title: string): RegExp {
+ return new RegExp(`^${title.replace(/[.*+?^$()|[\]\\{}]/g, String.raw`\$&`)}$`);
+}
/** The `agent.act` step of one test in a run. */
function actStepOf(outcome: RunOutcome, title: string) {
@@ -214,36 +209,31 @@ function actStepOf(outcome: RunOutcome, title: string) {
return step;
}
-describe('agent.act grammar verbs', () => {
- let app: FixtureApp;
- let outcome: RunOutcome;
- let project: FixtureProject;
+let app: FixtureApp;
+let project: FixtureProject;
+let outcome: RunOutcome;
- beforeAll(async () => {
- app = await startFixtureApp();
- const model = installFakeLoopModel(actModel);
- const run = await runProject(
- { 'tests/verbs.e2e.ts': suiteOf(FLOWS), ...PROJECT_FILES },
- { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } } } },
- );
- outcome = run.outcome;
- project = run.project;
- }, 240_000);
+/** The run options for one pass over the project, recording to and replaying from the trace cache. */
+function cachedRun(model: ReturnType) {
+ return { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } }, cache: 'read-write' as const } };
+}
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
+beforeAll(async () => {
+ app = await startFixtureApp();
+ project = createProject({ 'tests/verbs.e2e.ts': suiteOf(FLOWS), ...PROJECT_FILES });
+ outcome = await runExisting(project, cachedRun(installFakeLoopModel(actModel)));
+}, 240_000);
+afterAll(async () => {
+ project?.cleanup();
+ await app?.close();
+});
+
+describe('agent.act grammar verbs', () => {
const stepOf = (title: string) => actStepOf(outcome, title);
const engineEvents = (title: string) => stepOf(title).events.filter((event) => event.kind === 'engine');
const turnsOf = (instruction: string) => loopCalls.filter((call) => call.prompt.includes(instruction));
- it('offers every verb the browser engine declares', () => {
- const [first] = turnsOf('hover the account menu');
- for (const tool of VERB_TOOLS) expect(first!.toolNames).toContain(tool);
- });
-
it('hovers by id, reports the control the hover revealed, and records the hover', () => {
expect(resultByTitle(outcome, 'hovers the menu trigger and taps what it reveals').status).toBe('passed');
const actions = engineEvents('hovers the menu trigger and taps what it reveals');
@@ -307,23 +297,16 @@ describe('agent.act grammar verbs', () => {
expect(turnsOf('attach the fixture file')[1]!.lastToolResult).toMatch(/^Uploaded "fixtures\/attachment\.txt" to #\S+\./);
});
- it('refuses a hidden file and a path outside the project before any engine action, as POLICY_DENIED', () => {
- for (const [title, instruction, path] of [
- ['refuses a hidden file', 'attach the env file', '.env'],
- ['refuses a file outside the project', 'attach a file from outside', '../outside.txt'],
- ] as const) {
- expect(resultByTitle(outcome, title).status).toBe('failed');
- const step = stepOf(title);
- expect(step.events.filter((event) => event.kind === 'engine')).toHaveLength(0);
- expect(step.metrics!.actionSteps).toBe(0);
- expect(step.events.filter((event) => event.kind === 'policy')).toEqual([
- expect.objectContaining({ name: 'upload.path', decision: 'denied', code: 'POLICY_DENIED' }),
- ]);
- // The refusal reaches the model as the action's failure, led by the attempt and the code, naming the path it refused.
- expect(turnsOf(instruction)[1]!.lastToolResult).toMatch(
- new RegExp(`^upload ${JSON.stringify(path).replaceAll('.', '\\.')} to #\\S+ failed: POLICY_DENIED: ${JSON.stringify(path).replaceAll('.', '\\.')} is`),
- );
- }
+ it('refuses a hidden file before any engine action, as POLICY_DENIED', () => {
+ expect(resultByTitle(outcome, 'refuses a hidden file').status).toBe('failed');
+ const step = stepOf('refuses a hidden file');
+ expect(step.events.filter((event) => event.kind === 'engine')).toHaveLength(0);
+ expect(step.metrics!.actionSteps).toBe(0);
+ expect(step.events.filter((event) => event.kind === 'policy')).toEqual([
+ expect.objectContaining({ name: 'upload.path', decision: 'denied', code: 'POLICY_DENIED' }),
+ ]);
+ // The refusal reaches the model as the action's failure, led by the attempt and the code, naming the path it refused.
+ expect(turnsOf('attach the env file')[1]!.lastToolResult).toMatch(/^upload "\.env" to #\S+ failed: POLICY_DENIED: "\.env" is/);
});
it('scrolls a listed node into view as one action', () => {
@@ -339,11 +322,11 @@ describe('agent.act grammar verbs', () => {
const step = stepOf('pages a windowed list to a row it has not rendered');
const actions = step.events.filter((event) => event.kind === 'engine');
expect(actions.map((event) => event.name)).toEqual(['scrollUntil']);
- expect(actions[0]!.detail).toMatch(/^scroll down on list "Ledger" until "Row 333" shows \(\d+ screens\)$/);
- // Forty-odd pages of the list, one action of the budget.
+ expect(actions[0]!.detail).toMatch(/^scroll down on list "Ledger" until "Row 24" shows \(\d+ screens\)$/);
+ // Several pages of the list, one action of the budget.
expect(step.metrics!.actionSteps).toBe(1);
- expect(turnsOf('scroll the ledger')[1]!.lastToolResult).toMatch(/^Scrolled down until "Row 333" was in view\./);
- expect(turnsOf('scroll the ledger')[1]!.lastToolResult).toMatch(/Row 333 · Golden/);
+ expect(turnsOf('scroll the ledger')[1]!.lastToolResult).toMatch(/^Scrolled down until "Row 24" was in view\./);
+ expect(turnsOf('scroll the ledger')[1]!.lastToolResult).toMatch(/Row 24 · Golden/);
});
it('repeats a key in one call, one engine action per press, and shows the selection it made', () => {
@@ -394,45 +377,32 @@ describe('agent.act grammar verbs', () => {
describe('agent.act grammar verbs: record then zero-turn replay', () => {
const flows = FLOWS.filter((flow) => flow.replays === true);
- let app: FixtureApp;
- let project: FixtureProject;
- let firstRun: RunOutcome;
let secondRun: RunOutcome;
let secondRunModelCalls = 0;
beforeAll(async () => {
- app = await startFixtureApp();
- project = createProject({ 'tests/replay.e2e.ts': suiteOf(flows), ...PROJECT_FILES });
- const options = (model: ReturnType) => ({
- appUrl: app.url,
- config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } }, cache: 'read-write' as const },
- });
- firstRun = await runExisting(project, options(installFakeLoopModel(actModel)));
- secondRun = await runExisting(
- project,
- options(
- installFakeLoopModel((call) => {
- secondRunModelCalls += 1;
- return actModel(call);
- }),
- ),
+ const options = cachedRun(
+ installFakeLoopModel((call) => {
+ secondRunModelCalls += 1;
+ return actModel(call);
+ }),
);
- }, 300_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
+ secondRun = await runExisting(project, {
+ ...options,
+ runOptions: { grep: flows.map((flow) => exactTitle(flow.title)) },
+ });
+ }, 240_000);
it('records every verb on the first run', () => {
- expect(firstRun.exitCode).toBe(0);
for (const flow of flows) {
- expect(actStepOf(firstRun, flow.title).cache, flow.title).toMatchObject({ mode: 'missed', reason: 'no-entry' });
+ expect(resultByTitle(outcome, flow.title).status, flow.title).toBe('passed');
+ expect(actStepOf(outcome, flow.title).cache, flow.title).toMatchObject({ mode: 'missed', reason: 'no-entry' });
}
});
it('replays hover, drag, check, upload, and scroll into view without a model call, and runs a round trip that changed nothing live', () => {
expect(secondRun.exitCode).toBe(0);
+ expect(secondRun.results.filter((result) => result.status !== 'skipped').map((result) => result.test.title).toSorted()).toEqual(flows.map((flow) => flow.title).toSorted());
// A round trip leaves the screen and the route as it found them: nothing a
// replay could check, so it is never recorded and the model runs it again.
const roundTrip = actStepOf(secondRun, 'opens a page and comes back');
diff --git a/packages/e2e/tests/integration/agent-act.test.ts b/packages/e2e/tests/integration/agent-act.test.ts
index 15e07f39b..a4be76ea7 100644
--- a/packages/e2e/tests/integration/agent-act.test.ts
+++ b/packages/e2e/tests/integration/agent-act.test.ts
@@ -10,7 +10,6 @@ import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { installFakeLoopModel, loopCalls, nodeIdFor } from '../helpers/fake-loop-model.ts';
-import { assertValidReport } from '../helpers/report-schema.ts';
import { resultByTitle, runProject, type FixtureProject } from '../helpers/run-project.ts';
import type { RunOutcome } from '../helpers/run-project.ts';
import type { StepExecutor, StepExecutorContext } from '../../src/agent/executor.ts';
@@ -147,7 +146,6 @@ describe('agent.act with a hand-rolled step executor', () => {
expect(step!.metrics!.actionSteps).toBe(1);
expect(step!.explanation).toContain('the counter shows 1');
const engineEvents = step!.events.filter((event) => event.kind === 'engine');
- const observations = step!.events.filter((event) => event.kind === 'observation');
expect(engineEvents).toHaveLength(1);
const modelEvents = step!.events.filter((event) => event.kind === 'model');
expect(modelEvents).toHaveLength(1);
@@ -155,17 +153,6 @@ describe('agent.act with a hand-rolled step executor', () => {
// time than the engine event, later in the list.
expect(step!.events.indexOf(modelEvents[0]!)).toBeGreaterThan(step!.events.indexOf(engineEvents[0]!));
expect(Date.parse(modelEvents[0]!.startedAt)).toBeLessThanOrEqual(Date.parse(engineEvents[0]!.startedAt));
- // The cache baseline also supplies the executor's first look. The other
- // captures are its post-action look and the cache's passing observation,
- // whose delta from the baseline becomes the staged trace's end anchors.
- expect(observations).toHaveLength(3);
- });
-
- it('emits a schema-valid report for executor-driven steps', () => {
- const report = JSON.parse(
- readFileSync(path.join(project.dir, '.e2e', 'report.json'), 'utf8'),
- ) as object;
- assertValidReport(report);
});
});
@@ -318,39 +305,6 @@ describe('agent.act verdict mapping', () => {
}
}, 120_000);
- it('routes agent.assert through a custom executor with assert semantics', async () => {
- const seen: string[] = [];
- const executor: StepExecutor = {
- name: 'judging-executor',
- async runStep(context: StepExecutorContext) {
- seen.push(`${context.step.kind}:${context.step.instruction}`);
- if (context.step.instruction.includes('checkout')) {
- return { status: 'failed' as const, summary: 'no checkout page exists here' };
- }
- const observation = await context.observe();
- return /status "Counter" text="0"|status.*"0"/.test(observation.text)
- ? { status: 'passed' as const, summary: 'the counter reads 0' }
- : { status: 'failed' as const, summary: 'the counter is not zero' };
- },
- };
- const { outcome, project } = await runProject(
- { 'tests/assert.e2e.ts': ASSERT_SUITE },
- { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } },
- );
- try {
- const result = resultByTitle(outcome, 'custom executor judges assertions');
- expect(result.status).toBe('failed');
- // The failed assertion maps to ASSERTION_FAILED, not ACTION_FAILED.
- expect(result.attempts.at(-1)!.error?.code).toBe('ASSERTION_FAILED');
- expect(seen[0]).toBe('assert:the counter shows zero');
- const steps = result.attempts.at(-1)!.steps.filter((s) => s.api === 'agent.assert');
- expect(steps[0]!.status).toBe('passed');
- expect(steps[1]!.status).toBe('failed');
- } finally {
- project.cleanup();
- }
- }, 120_000);
-
it('fills a declared secret through typeSecret, never exposing the value', async () => {
const executor: StepExecutor = {
name: 'login-executor',
@@ -566,15 +520,6 @@ describe('agent.act engine operations are bounded by actionTimeout', () => {
}, 120_000);
});
-const ASSERT_SUITE = `import { test } from 'e2e';
-
-test('custom executor judges assertions', async ({ app, agent }) => {
- await app.open();
- await agent.assert('the counter shows zero');
- await agent.assert('the checkout page is visible');
-});
-`;
-
const SECRET_SUITE = `import { test, credentials, expect } from 'e2e';
test('executor fills a declared secret', async ({ app, agent, screen }) => {
@@ -680,10 +625,7 @@ describe('agent.act with the default ToolLoopAgent executor', () => {
if (call.lastToolResult === '') {
// First turn: the prompt carries the instruction and initial screen.
const id = nodeIdFor(call.prompt, /button "Increment"/);
- return {
- toolCalls: [{ toolName: 'tap', input: { target: id } }],
- reasoning: 'The counter starts at 0; tapping Increment once.',
- };
+ return [{ toolName: 'tap', input: { target: id } }];
}
// The tap result reports the change; conclude once the counter reads 1.
if (/text="1"/.test(call.lastToolResult)) {
@@ -775,20 +717,6 @@ describe('agent.act with the default ToolLoopAgent executor', () => {
step!.metrics!.modelCalls,
);
});
-
- it('carries a turn\'s reasoning onto its model event and into the valid report', () => {
- const attempt = resultByTitle(outcome, 'default agent increments the counter').attempts.at(
- -1,
- )!;
- const step = attempt.steps.find((candidate) => candidate.api === 'agent.act');
- const modelEvents = step!.events.filter((event) => event.kind === 'model');
- expect(modelEvents[0]!.reasoning).toBe('The counter starts at 0; tapping Increment once.');
- expect(modelEvents.at(-1)).not.toHaveProperty('reasoning');
- const report = JSON.parse(
- readFileSync(path.join(project.dir, '.e2e', 'report.json'), 'utf8'),
- ) as object;
- assertValidReport(report);
- });
});
const DELAYED_SUITE = `import { test, expect } from 'e2e';
diff --git a/packages/e2e/tests/integration/agent-ai-trace.test.ts b/packages/e2e/tests/integration/agent-ai-trace.test.ts
index 456bcf858..1dbce415d 100644
--- a/packages/e2e/tests/integration/agent-ai-trace.test.ts
+++ b/packages/e2e/tests/integration/agent-ai-trace.test.ts
@@ -115,39 +115,6 @@ describe('--ai-trace on the in-process transport', () => {
expect(steps.every((step) => step.model_id === 'scripted-loop')).toBe(true);
expect(steps.every((step) => step.provider === 'fake-loop')).toBe(true);
});
-
- it('records the prompt the model saw, its tool definitions with schemas, and its answer', () => {
- const document = readTrace(project);
- const [first, second] = document.steps;
- const input = JSON.parse(first!.input) as {
- prompt: { role: string; content: unknown }[];
- tools: { name: string; description?: string; parameters?: { type?: string } }[];
- };
- expect(input.prompt[0]!.role).toBe('system');
- expect(String(input.prompt[0]!.content)).toContain('autonomous end-to-end testing agent');
- const user = input.prompt.find((message) => message.role === 'user')!.content;
- // A pixel-mode opening prompt is content parts: the text part carries the instruction.
- const userText = Array.isArray(user)
- ? (user as { type: string; text?: string }[]).filter((part) => part.type === 'text').map((part) => part.text).join('\n')
- : String(user);
- expect(userText).toContain('increment the counter once');
- const names = input.tools.map((tool) => tool.name);
- expect(names).toContain('tap');
- expect(names).toContain('complete_step');
- for (const tool of input.tools) expect(tool.parameters?.type).toBe('object');
-
- const output = JSON.parse(first!.output!) as {
- finishReason: string;
- response: { messages: { role: string }[] };
- };
- expect(output.finishReason).toBe('tool-calls');
- expect(output.response.messages.map((message) => message.role)).toEqual(['assistant', 'tool']);
- expect(JSON.parse(first!.usage!)).toMatchObject({ inputTokens: 100, outputTokens: 20 });
-
- // The second turn's prompt carries the first turn's tool result.
- const later = JSON.parse(second!.input) as { prompt: { role: string }[] };
- expect(later.prompt.map((message) => message.role)).toEqual(['system', 'user', 'assistant', 'tool']);
- });
});
describe('--ai-trace on child-process workers', () => {
diff --git a/packages/e2e/tests/integration/agent-fallback-handoff.test.ts b/packages/e2e/tests/integration/agent-fallback-handoff.test.ts
index 57b746773..436092b9b 100644
--- a/packages/e2e/tests/integration/agent-fallback-handoff.test.ts
+++ b/packages/e2e/tests/integration/agent-fallback-handoff.test.ts
@@ -1,11 +1,12 @@
-/** Pixel-only evidence survives the cache handoff and every default-agent screen update. */
+/** Pixel-only evidence survives the cache handoff and every default-agent screen update, and no model sees it once a secret fill taints the attempt. */
-import { describe, expect, it } from 'vitest';
-import { defineEngine, EngineError, type EngineSnapshot } from '../../src/engine/index.ts';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { defineEngine, EngineError, type EngineObserveOptions, type EngineSnapshot } from '../../src/engine/index.ts';
import { buildTraceEntry, type RecordedAction } from '../../src/cache/trace.ts';
import type { StepExecutor } from '../../src/agent/executor.ts';
import { installFakeLoopModel, loopCalls } from '../helpers/fake-loop-model.ts';
-import { runProject } from '../helpers/run-project.ts';
+import { fakeCalls, installFakeModel, judgment } from '../helpers/fake-model.ts';
+import { resultByTitle, runProject, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
const SUITE = `import { test } from 'e2e';
@@ -25,7 +26,7 @@ function pixelSnapshot(value = 1): EngineSnapshot {
}
describe('semantic fallback handoff', () => {
- it.each([true, false, undefined])('requests pixels=%s after a semantic cache probe', async (pixels) => {
+ it.each([true, undefined])('requests pixels=%s after a semantic cache probe', async (pixels) => {
const captures: boolean[] = [];
const engine = defineEngine({
name: 'requested-pixels-fixture', version: '1', spiVersion: 1, platform: 'fixture',
@@ -128,7 +129,10 @@ describe('semantic fallback handoff', () => {
expect(captures).toEqual([true]);
expect(loopCalls).toHaveLength(1);
expect(loopCalls[0]?.imageParts).toBe(1);
+ expect(loopCalls[0]?.prompt).toContain('Screenshot attached: 2 by 2');
expect(loopCalls[0]?.prompt).toContain('semantic capture unavailable');
+ expect(loopCalls[0]?.prompt).toContain('never infer absence');
+ expect(outcome.report.run.results[0]!.attempts[0]!.steps.find((step) => step.api === 'agent.act')?.visionInput).toBe(true);
assertValidReport(outcome.report);
} finally {
project.cleanup();
@@ -312,3 +316,86 @@ describe('semantic fallback handoff', () => {
}
});
});
+
+const SEMANTIC_SUITE = `import { test, credentials } from 'e2e';
+test('vision judgment receives fallback', async ({ agent }) => {
+ await agent.assert('the screenshot shows the result', { vision: true });
+});
+test('tree-only judgment cannot infer absence', async ({ agent }) => {
+ await agent.assert('nothing is visible', { vision: false });
+});
+test('act cannot recover pixels after a secret fill', async ({ agent, screen }) => {
+ await screen.getByRole('textbox').fill(credentials.user('member').password);
+ await agent.act('read the tainted screenshot');
+});
+test('judgment cannot recover pixels after a secret fill', async ({ agent, screen }) => {
+ await screen.getByRole('textbox').fill(credentials.user('member').password);
+ await agent.assert('read the tainted result', { vision: true });
+});`;
+
+describe('agent semantic fallback through the engine contract', () => {
+ let project: FixtureProject;
+ let outcome: RunOutcome;
+ const captures: { filled: boolean; options: EngineObserveOptions | undefined }[] = [];
+
+ beforeAll(async () => {
+ let filled = false;
+ const engine = defineEngine({
+ name: 'masked-fixture', version: '1', spiVersion: 1, platform: 'fixture',
+ actions: ['fill'],
+ startAttempt: async () => { filled = false; },
+ locate: async () => [{ ref: { id: 'password', revision: '' }, role: 'textbox', inputPurpose: 'password', states: { secure: true } }],
+ perform: async () => { filled = true; },
+ observe: async (_operation, options) => {
+ captures.push({ filled, options });
+ if (options?.pixelFallback !== true) {
+ throw new EngineError('OPERATION_TIMEOUT', 'semantic capture timed out', { retryable: false });
+ }
+ return {
+ root: { ref: { id: 'root', revision: '' } },
+ location: 'fixture:result',
+ viewport: { width: 2, height: 2 },
+ treeUnavailable: true,
+ pixels: { data: new Uint8Array([1, 2, 3]), mediaType: 'image/png', width: 2, height: 2, scale: 1 },
+ maskedRegionCount: 0,
+ };
+ },
+ });
+ const model = installFakeLoopModel(() => [{ toolName: 'complete_step', input: { status: 'passed', summary: 'read pixels' } }]);
+ const judge = installFakeModel(() => judgment(true, 'visible in the screenshot'));
+ const result = await runProject({ 'tests/fallback.e2e.ts': SEMANTIC_SUITE }, {
+ appUrl: 'https://fixture.test',
+ config: {
+ tests: 'tests/**/*.e2e.ts',
+ targets: [{ name: 'fixture', engine }],
+ agents: { default: { model, judge } },
+ credentials: { member: { username: 'member', password: 'fixture-secret-value' } },
+ },
+ });
+ project = result.project;
+ outcome = result.outcome;
+ expect(outcome.report.run.errors).toEqual([]);
+ });
+
+ afterAll(() => project?.cleanup());
+
+ it('judges the current screenshot with vision enabled', () => {
+ expect(resultByTitle(outcome, 'vision judgment receives fallback').status).toBe('passed');
+ expect(fakeCalls).toHaveLength(1);
+ expect(fakeCalls[0]!.images).toEqual([{ mediaType: 'image/png', bytes: 3 }]);
+ expect(fakeCalls[0]!.observation).toContain('semantic capture unavailable');
+ });
+
+ it('stops tree-only and tainted judgments before a model call', () => {
+ for (const title of ['tree-only judgment cannot infer absence', 'act cannot recover pixels after a secret fill', 'judgment cannot recover pixels after a secret fill']) {
+ const result = resultByTitle(outcome, title);
+ expect(result.status).toBe('failed');
+ expect(result.attempts[0]!.error?.message).toContain('timed out');
+ }
+ const tainted = captures.filter((capture) => capture.filled);
+ expect(tainted.length).toBeGreaterThanOrEqual(2);
+ expect(tainted.every((capture) => capture.options?.pixelFallback !== true && capture.options?.pixels !== true)).toBe(true);
+ expect(loopCalls).toEqual([]);
+ assertValidReport(outcome.report);
+ });
+});
diff --git a/packages/e2e/tests/integration/agent-policy.test.ts b/packages/e2e/tests/integration/agent-policy.test.ts
index 4544bb47d..06a4c20e6 100644
--- a/packages/e2e/tests/integration/agent-policy.test.ts
+++ b/packages/e2e/tests/integration/agent-policy.test.ts
@@ -10,9 +10,8 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { OBSERVED_NAME_LIMIT, OBSERVED_TEXT_LIMIT, type SemanticNode } from '../../src/engine/contract.ts';
import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { installFakeLoopModel, loopCalls, nodeIdFor } from '../helpers/fake-loop-model.ts';
-import { extracted, installFakeModel, judgment } from '../helpers/fake-model.ts';
+import { extracted, fakeCalls, installFakeModel, judgment } from '../helpers/fake-model.ts';
import type { FakeCall } from '../helpers/fake-model.ts';
-import { assertValidReport } from '../helpers/report-schema.ts';
import { createProject, resultByTitle, runExisting, runProject, type FixtureProject } from '../helpers/run-project.ts';
import type { RunOutcome } from '../helpers/run-project.ts';
import type { SdkLanguageModel } from '../../src/agent/ai-sdk.ts';
@@ -21,10 +20,10 @@ import { createFakeEngine, FAKE_APP, FAKE_APP_URL } from '../helpers/fake-engine
const SUITE = `import { test, credentials } from 'e2e';
-test('fills a secret into a password field', async ({ app, agent, screen }) => {
- await app.open();
- await screen.getByLabel('Password').fill(credentials.user('member').password);
- await agent.assert('the password field has a value');
+test('fills a secret into a field the page echoes', async ({ app, agent, screen }) => {
+ await app.open('/echo');
+ await screen.getByLabel('Token').fill(credentials.user('member').password);
+ await agent.assert('the token echo shows a value');
});
test('repairs an extraction that fails the caller schema', async ({ app, agent }) => {
@@ -107,6 +106,7 @@ describe('agent policy and error classification', () => {
let app: FixtureApp;
let outcome: RunOutcome;
let project: FixtureProject;
+ let modelCalls: FakeCall[];
let unconfigured: RunOutcome;
let unconfiguredProject: FixtureProject;
let ghost: RunOutcome;
@@ -130,6 +130,7 @@ describe('agent policy and error classification', () => {
);
outcome = main.outcome;
project = main.project;
+ modelCalls = [...fakeCalls];
const missingCredential = await runProject(
{ 'tests/ghost.e2e.ts': UNCONFIGURED_SUITE },
@@ -189,24 +190,15 @@ describe('agent policy and error classification', () => {
expect(step.model).toMatchObject({ provider: 'fake', model: 'scripted' });
});
- it('fills a secret into a purpose-compatible secure field', () => {
- expect(resultByTitle(outcome, 'fills a secret into a password field').status).toBe('passed');
- });
-
- it('suppresses screenshot evidence once the viewport is pixel-tainted', () => {
- const attempt = resultByTitle(outcome, 'fills a secret into a password field').attempts.at(-1)!;
- const assertion = attempt.steps.find((step) => step.api === 'agent.assert')!;
- expect(assertion.artifacts).toEqual([]);
- expect(
- assertion.events.some(
- (event) => event.kind === 'policy' && event.name === 'assert.screenshot' && event.decision === 'denied',
- ),
- ).toBe(true);
- });
-
it('never sends a secret value to the model', () => {
- const result = resultByTitle(outcome, 'fills a secret into a password field');
- expect(JSON.stringify(result.attempts)).not.toContain('hunter2-secret');
+ expect(resultByTitle(outcome, 'fills a secret into a field the page echoes').status).toBe('passed');
+ const echoed = modelCalls.filter((call) => call.instruction === 'the token echo shows a value');
+ expect(echoed).toHaveLength(1);
+ expect(echoed[0]!.observation).toContain('');
+ expect(modelCalls.length).toBeGreaterThanOrEqual(2);
+ for (const call of modelCalls) {
+ expect(`${call.system}\n${call.prompt}`.toLowerCase()).not.toContain('hunter2-secret');
+ }
});
it('reports an unconfigured credential as a configuration failure', () => {
@@ -240,57 +232,6 @@ describe('agent policy and error classification', () => {
});
});
-describe('serial group artifacts', () => {
- let app: FixtureApp;
- let outcome: RunOutcome;
- let project: FixtureProject;
-
- const SERIAL_SUITE = `import { test } from 'e2e';
-
-test.describe('group', { serial: true }, () => {
- test('captures evidence from a shared session', async ({ app, agent }) => {
- await app.open();
- await agent.assert('the Home heading is visible');
- });
-});
-`;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- const model = installFakeModel(() => judgment(true, 'the heading is present'));
- const result = await runProject(
- { 'tests/serial.e2e.ts': SERIAL_SUITE },
- {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- agents: { default: { model } },
- },
- },
- );
- outcome = result.outcome;
- project = result.project;
- }, 120_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it('records a resolvable path, size, and digest for member artifacts', () => {
- // A shared session writes into the group directory, so a member that
- // registers against its own directory produces an unresolvable artifact.
- assertValidReport(outcome.report);
- const group = outcome.report.run.serialGroups[0]!;
- const screenshot = group.attempts
- .at(-1)!
- .artifacts.find((artifact) => artifact.kind === 'screenshot')!;
- expect(screenshot.path).toBeDefined();
- expect(screenshot.size).toBeGreaterThan(0);
- expect(screenshot.sha256).toMatch(/^[0-9a-f]{64}$/);
- });
-});
-
describe('tool calls outside the vocabulary', () => {
let app: FixtureApp;
diff --git a/packages/e2e/tests/integration/agent-scroll-settle.test.ts b/packages/e2e/tests/integration/agent-scroll-settle.test.ts
index 59091d35e..39abc3ef2 100644
--- a/packages/e2e/tests/integration/agent-scroll-settle.test.ts
+++ b/packages/e2e/tests/integration/agent-scroll-settle.test.ts
@@ -63,7 +63,6 @@ describe('the look after scrolling to text', () => {
expect(executorCalls).toBe(1);
const replay = resultByTitle(replayed, 'reaches and taps the row').attempts[0]!.steps.find((step) => step.api === 'agent.act');
expect(replay?.cache).toMatchObject({ mode: 'self-finalized', replayedActions: 2 });
- console.info(`scroll-to-text ${String(pages)} pages: ${String(replay?.durationMs)} ms replay, ${String(afterScroll)} captures after the live action`);
expect(afterScroll).toBe(2);
if (pages > 0) expect(beforeTap).toEqual([4, 4]);
} finally {
diff --git a/packages/e2e/tests/integration/agent-semantic-fallback.test.ts b/packages/e2e/tests/integration/agent-semantic-fallback.test.ts
deleted file mode 100644
index 109100ebc..000000000
--- a/packages/e2e/tests/integration/agent-semantic-fallback.test.ts
+++ /dev/null
@@ -1,103 +0,0 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { defineEngine, EngineError, type EngineObserveOptions } from '../../src/engine/index.ts';
-import { installFakeLoopModel, loopCalls } from '../helpers/fake-loop-model.ts';
-import { fakeCalls, installFakeModel, judgment } from '../helpers/fake-model.ts';
-import { resultByTitle, runProject, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
-import { assertValidReport } from '../helpers/report-schema.ts';
-
-const SUITE = `import { test, credentials } from 'e2e';
-test('act receives fallback on its first observation', async ({ agent }) => {
- await agent.act('read the visible screenshot');
-});
-test('vision judgment receives fallback', async ({ agent }) => {
- await agent.assert('the screenshot shows the result', { vision: true });
-});
-test('tree-only judgment cannot infer absence', async ({ agent }) => {
- await agent.assert('nothing is visible', { vision: false });
-});
-test('act cannot recover pixels after a secret fill', async ({ agent, screen }) => {
- await screen.getByRole('textbox').fill(credentials.user('member').password);
- await agent.act('read the tainted screenshot');
-});
-test('judgment cannot recover pixels after a secret fill', async ({ agent, screen }) => {
- await screen.getByRole('textbox').fill(credentials.user('member').password);
- await agent.assert('read the tainted result', { vision: true });
-});`;
-
-describe('agent semantic fallback through the engine contract', () => {
- let project: FixtureProject;
- let outcome: RunOutcome;
- const captures: { filled: boolean; options: EngineObserveOptions | undefined }[] = [];
-
- beforeAll(async () => {
- let filled = false;
- const engine = defineEngine({
- name: 'masked-fixture', version: '1', spiVersion: 1, platform: 'fixture',
- actions: ['fill'],
- startAttempt: async () => { filled = false; },
- locate: async () => [{ ref: { id: 'password', revision: '' }, role: 'textbox', inputPurpose: 'password', states: { secure: true } }],
- perform: async () => { filled = true; },
- observe: async (_operation, options) => {
- captures.push({ filled, options });
- if (options?.pixelFallback !== true) {
- throw new EngineError('OPERATION_TIMEOUT', 'semantic capture timed out', { retryable: false });
- }
- return {
- root: { ref: { id: 'root', revision: '' } },
- location: 'fixture:result',
- viewport: { width: 2, height: 2 },
- treeUnavailable: true,
- pixels: { data: new Uint8Array([1, 2, 3]), mediaType: 'image/png', width: 2, height: 2, scale: 1 },
- maskedRegionCount: 0,
- };
- },
- });
- const model = installFakeLoopModel(() => [{ toolName: 'complete_step', input: { status: 'passed', summary: 'read pixels' } }]);
- const judge = installFakeModel(() => judgment(true, 'visible in the screenshot'));
- const result = await runProject({ 'tests/fallback.e2e.ts': SUITE }, {
- appUrl: 'https://fixture.test',
- config: {
- tests: 'tests/**/*.e2e.ts',
- targets: [{ name: 'fixture', engine }],
- agents: { default: { model, judge } },
- credentials: { member: { username: 'member', password: 'fixture-secret-value' } },
- },
- });
- project = result.project;
- outcome = result.outcome;
- expect(outcome.report.run.errors).toEqual([]);
- });
-
- afterAll(() => project?.cleanup());
-
- it('sends the first fallback screenshot and the warning to the acting model', () => {
- expect(resultByTitle(outcome, 'act receives fallback on its first observation').status).toBe('passed');
- expect(loopCalls).toHaveLength(1);
- expect(loopCalls[0]!.imageParts).toBe(1);
- expect(loopCalls[0]!.prompt).toContain('Screenshot attached: 2 by 2');
- expect(loopCalls[0]!.prompt).toContain('semantic capture unavailable');
- expect(loopCalls[0]!.prompt).toContain('never infer absence');
- expect(captures[0]?.options).toEqual({ pixelFallback: true });
- const step = resultByTitle(outcome, 'act receives fallback on its first observation').attempts[0]!.steps.find((entry) => entry.api === 'agent.act');
- expect(step?.visionInput).toBe(true);
- });
-
- it('judges the current screenshot with vision enabled', () => {
- expect(resultByTitle(outcome, 'vision judgment receives fallback').status).toBe('passed');
- expect(fakeCalls).toHaveLength(1);
- expect(fakeCalls[0]!.images).toEqual([{ mediaType: 'image/png', bytes: 3 }]);
- expect(fakeCalls[0]!.observation).toContain('semantic capture unavailable');
- });
-
- it('stops tree-only and tainted judgments before a model call', () => {
- for (const title of ['tree-only judgment cannot infer absence', 'act cannot recover pixels after a secret fill', 'judgment cannot recover pixels after a secret fill']) {
- const result = resultByTitle(outcome, title);
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error?.message).toContain('timed out');
- }
- const tainted = captures.filter((capture) => capture.filled);
- expect(tainted.length).toBeGreaterThanOrEqual(2);
- expect(tainted.every((capture) => capture.options?.pixelFallback !== true && capture.options?.pixels !== true)).toBe(true);
- assertValidReport(outcome.report);
- });
-});
diff --git a/packages/e2e/tests/integration/agent-trace-cache.test.ts b/packages/e2e/tests/integration/agent-trace-cache.test.ts
index e6c0a4917..369a6e168 100644
--- a/packages/e2e/tests/integration/agent-trace-cache.test.ts
+++ b/packages/e2e/tests/integration/agent-trace-cache.test.ts
@@ -99,88 +99,6 @@ function entryFileState(project: FixtureProject): { bytes: string; mtimeMs: numb
return { bytes: readFileSync(file, 'utf8'), mtimeMs: statSync(file).mtimeMs };
}
-describe('trace cache: record then zero-turn replay', () => {
- let app: FixtureApp;
- let project: FixtureProject;
- let firstRun: RunOutcome;
- let secondRun: RunOutcome;
- let recordedFile: { bytes: string; mtimeMs: number };
- const first: ExecutorRecord = { calls: 0, prefixes: [] };
- const second: ExecutorRecord = { calls: 0, prefixes: [] };
-
- beforeAll(async () => {
- app = await startFixtureApp();
- project = createProject({ 'tests/act.e2e.ts': SUITE });
- const options = (record: ExecutorRecord) => ({
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- agents: { default: { executor: twoTapExecutor(record) } },
- cache: 'read-write' as const,
- },
- });
- firstRun = await runExisting(project, options(first));
- recordedFile = entryFileState(project);
- secondRun = await runExisting(project, options(second));
- }, 240_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it('runs the executor on the first pass and records one entry', () => {
- expect(firstRun.exitCode).toBe(0);
- expect(first.calls).toBe(1);
- expect(first.prefixes).toEqual([undefined]);
- const step = actStep(firstRun);
- expect(step.cache).toEqual({
- mode: 'missed',
- reason: 'no-entry',
- replayedActions: 0,
- totalActions: 0,
- });
- expect(readdirSync(cacheDir(project))).toHaveLength(1);
- const { entry } = readOnlyEntry(project);
- expect(entry.schemaVersion).toBe('trace-1');
- expect(entry.payload.actions).toHaveLength(2);
- expect(entry.payload.executor.name).toBe('two-tap-executor');
- expect(entry.payload.startPath).toBe('/');
- // The recording's own check, kept as data: the counter reading 2 appeared
- // during the step, so a replay must show it again before passing alone.
- expect(entry.payload.endAnchors).toContainEqual({ role: 'status', name: 'Counter', text: '2' });
- });
-
- it('replays the second run zero-turn without invoking the executor', () => {
- expect(secondRun.exitCode).toBe(0);
- expect(second.calls).toBe(0);
- const step = actStep(secondRun);
- expect(step.status).toBe('passed');
- expect(step.cache).toEqual({
- mode: 'self-finalized',
- replayedActions: 2,
- totalActions: 2,
- });
- expect(step.metrics!.modelCalls).toBe(0);
- expect(step.metrics!.actionSteps).toBe(2);
- expect(step.explanation).toContain('zero-turn');
- // A step the cache replayed whole never rewrites its entry: the file
- // keeps its bytes and its modification time, so a committed cache
- // directory stays clean across local runs.
- expect(entryFileState(project)).toEqual(recordedFile);
- const { entry } = readOnlyEntry(project);
- expect(entry.payload.summary).toBe('the counter shows 2');
- expect(entry.payload.endPath).toBe('/');
- });
-
- it('emits schema-valid reports for both cached and uncached runs', () => {
- const report = JSON.parse(
- readFileSync(path.join(project.dir, '.e2e', 'report.json'), 'utf8'),
- ) as object;
- assertValidReport(report);
- });
-});
-
describe('trace cache: divergence hands the step over mid-step', () => {
let app: FixtureApp;
let project: FixtureProject;
@@ -243,7 +161,7 @@ const WRONG_EXPECT_SUITE = `import { test, expect } from 'e2e';
test('cached step increments twice', async ({ app, agent, screen }) => {
await app.open();
await agent.act('increment the counter twice');
- await expect(screen.getByRole('status')).toHaveText('3');
+ await expect(screen.getByRole('status')).toHaveText('3', { timeout: 500 });
});
`;
@@ -256,7 +174,7 @@ test.afterEach(async ({ app }) => {
test('cached step increments twice', async ({ app, agent, screen }) => {
await app.open();
await agent.act('increment the counter twice');
- await expect(screen.getByRole('status')).toHaveText('3');
+ await expect(screen.getByRole('status')).toHaveText('3', { timeout: 500 });
});
`;
@@ -287,16 +205,6 @@ describe('trace cache: unconfirmed traces are withheld and poisoned entries evic
await app?.close();
});
- it('never writes a trace whose following assertion failed', async () => {
- project = createProject({ 'tests/act.e2e.ts': WRONG_EXPECT_SUITE });
- const outcome = await runExisting(project, options());
- expect(outcome.exitCode).not.toBe(0);
- // The act passed (the executor saw 2), the expect demanded 3: the staged
- // trace is unconfirmed and nothing may reach the store.
- expect(existsSync(cacheDir(project))).toBe(false);
- project.cleanup();
- }, 120_000);
-
it('teardown steps passing after the failure cannot confirm the implicated trace', async () => {
project = createProject({ 'tests/act.e2e.ts': WRONG_EXPECT_WITH_TEARDOWN_SUITE });
const outcome = await runExisting(project, options());
@@ -330,20 +238,10 @@ test('cached step increments twice', async ({ app, agent }) => {
});
`;
-const CONFIRMED_THEN_PROVIDER_DOWN_SUITE = `import { test, expect } from 'e2e';
-
-test('cached step increments twice', async ({ app, agent, screen }) => {
- await app.open();
- await agent.act('increment the counter twice');
- await expect(screen.getByRole('status')).toHaveText('2');
- await agent.assert('the counter shows 2', { agent: 'judge' });
-});
-`;
-
const PROVIDER_DOWN_THEN_WRONG_AFTER_EACH_SUITE = `import { test, expect } from 'e2e';
test.afterEach(async ({ screen }) => {
- await expect(screen.getByRole('status')).toHaveText('3', { timeout: 1_000 });
+ await expect(screen.getByRole('status')).toHaveText('3', { timeout: 500 });
});
test('cached step increments twice', async ({ app, agent }) => {
@@ -409,20 +307,26 @@ describe('trace cache: a model that never answered implicates nothing', () => {
project.cleanup();
}, 240_000);
- it('still writes a fresh recording a check confirmed before a judgment found no model', async () => {
- project = createProject({ 'tests/act.e2e.ts': CONFIRMED_THEN_PROVIDER_DOWN_SUITE });
+ it('keeps an entry whose replay handed off to a model that never answered', async () => {
+ await recordEntry();
+ rewriteOnlyEntry(project, (payload) => {
+ const [firstTap, secondTap] = payload.actions;
+ if (firstTap === undefined || secondTap?.name !== 'tap') throw new Error(`expected two taps, got ${JSON.stringify(payload.actions)}`);
+ return { ...payload, actions: [firstTap, { ...secondTap, target: { ...secondTap.target, name: 'No Such Button' } }] };
+ });
+ const handedOff = entryFileState(project);
const outcome = await runExisting(project, {
appUrl: app.url,
config: {
tests: 'tests/**/*.e2e.ts',
- agents: { default: { executor: twoTapExecutor(record) }, judge: { model: unreachableModel } },
+ agents: { default: { model: unreachableModel } },
cache: 'read-write' as const,
},
});
expect(outcome.exitCode).not.toBe(0);
- expect(actStep(outcome).cache?.mode).toBe('missed');
- expect(stepErrorCode(outcome, 'agent.assert')).toBe('MODEL_PROVIDER_FAILED');
- expect(readOnlyEntry(project).entry.payload.actions).toHaveLength(2);
+ expect(actStep(outcome).cache).toMatchObject({ mode: 'agent-concluded', reason: 'target-not-found', replayedActions: 1 });
+ expect(stepErrorCode(outcome, 'agent.act')).toBe('MODEL_PROVIDER_FAILED');
+ expect(entryFileState(project)).toEqual(handedOff);
project.cleanup();
}, 240_000);
@@ -443,28 +347,6 @@ describe('trace cache: a model that never answered implicates nothing', () => {
expect(readEntries(project)).toHaveLength(0);
project.cleanup();
}, 240_000);
-
- it('keeps an entry whose replay handed off to a model that never answered', async () => {
- await recordEntry();
- rewriteOnlyEntry(project, (payload) => {
- const [firstTap, secondTap] = payload.actions;
- if (firstTap === undefined || secondTap?.name !== 'tap') throw new Error(`expected two taps, got ${JSON.stringify(payload.actions)}`);
- return { ...payload, actions: [firstTap, { ...secondTap, target: { ...secondTap.target, name: 'No Such Button' } }] };
- });
- const handedOff = entryFileState(project);
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- agents: { default: { model: unreachableModel } },
- cache: 'read-write' as const,
- },
- });
- expect(outcome.exitCode).not.toBe(0);
- expect(actStep(outcome).cache).toMatchObject({ mode: 'agent-concluded', reason: 'target-not-found', replayedActions: 1 });
- expect(stepErrorCode(outcome, 'agent.act')).toBe('MODEL_PROVIDER_FAILED');
- expect(entryFileState(project)).toEqual(handedOff);
- }, 240_000);
});
const STORAGE_SUITE = `import { test, expect } from 'e2e';
@@ -476,12 +358,8 @@ test('saves the marker', async ({ app, agent, screen }) => {
});
`;
-/**
- * Taps "Save marker" once (unless a replayed prefix already did) and checks
- * the marker. With `repair`, an end-mismatch hand-off makes it tap again — the
- * shape of an executor that found the replayed flow had not taken effect.
- */
-function saveMarkerExecutor(record: ExecutorRecord, options: { repair?: boolean } = {}): StepExecutor {
+/** Taps "Save marker" once (unless a replayed prefix already did) and checks the marker. */
+function saveMarkerExecutor(record: ExecutorRecord): StepExecutor {
return {
name: 'save-marker-executor',
version: 'test',
@@ -489,11 +367,7 @@ function saveMarkerExecutor(record: ExecutorRecord, options: { repair?: boolean
record.calls += 1;
record.prefixes.push(context.replayedPrefix);
let observation = await context.observe();
- const prefix = context.replayedPrefix;
- const mustAct =
- (prefix?.replayedActions.length ?? 0) === 0 ||
- (options.repair === true && prefix?.stopReason === 'end-mismatch');
- if (mustAct) {
+ if ((context.replayedPrefix?.replayedActions.length ?? 0) === 0) {
await context.actions.tap({ id: nodeIdFor(observation.text, /button "Save marker"/) });
observation = await context.observe();
}
@@ -510,7 +384,7 @@ describe('trace cache: the recorded end state gates self-finalization', () => {
let project: FixtureProject;
const records: ExecutorRecord[] = [];
- const options = (executor: { repair?: boolean } = {}) => {
+ const options = () => {
const record: ExecutorRecord = { calls: 0, prefixes: [] };
records.push(record);
return {
@@ -518,7 +392,7 @@ describe('trace cache: the recorded end state gates self-finalization', () => {
config: {
tests: 'tests/**/*.e2e.ts',
reporters: ['json'] as const,
- agents: { default: { executor: saveMarkerExecutor(record, executor) } },
+ agents: { default: { executor: saveMarkerExecutor(record) } },
cache: 'read-write' as const,
},
};
@@ -548,88 +422,6 @@ describe('trace cache: the recorded end state gates self-finalization', () => {
const step = resultByTitle(second, 'saves the marker').attempts.at(-1)!.steps.find((s) => s.api === 'agent.act')!;
expect(step.cache).toMatchObject({ mode: 'self-finalized', replayedActions: 1, totalActions: 1 });
}, 240_000);
-
- it('hands off with end-mismatch when the recorded effect is not on screen after a full replay', async () => {
- // Every recorded action still replays; only the recorded end state is
- // made unreachable. Mechanics alone must not pass the step.
- rewriteOnlyEntry(project, (payload) => ({ ...payload, endAnchors: [{ role: 'status', name: 'Marker', text: 'never-saved' }] }));
-
- const outcome = await runExisting(project, options());
- expect(outcome.exitCode).toBe(0);
- const record = records.at(-1)!;
- expect(record.calls).toBe(1);
- expect(record.prefixes[0]).toMatchObject({
- stopReason: 'end-mismatch',
- replayedActions: ['tap button "Save marker"'],
- totalActions: 1,
- });
- const step = resultByTitle(outcome, 'saves the marker').attempts.at(-1)!.steps.find((s) => s.api === 'agent.act')!;
- expect(step.cache).toEqual({
- mode: 'agent-concluded',
- reason: 'end-mismatch',
- replayedActions: 1,
- totalActions: 1,
- });
- // The executor's pass re-stages the entry with the live anchors: healed.
- expect(readOnlyEntry(project).entry.payload.endAnchors).toContainEqual({
- role: 'status',
- name: 'Marker',
- text: 'saved',
- });
- }, 240_000);
-
- it('evicts the entry when the executor had to act again after an end-mismatch', async () => {
- rewriteOnlyEntry(project, (payload) => ({ ...payload, endAnchors: [{ role: 'status', name: 'Marker', text: 'never-saved' }] }));
-
- const outcome = await runExisting(project, options({ repair: true }));
- expect(outcome.exitCode).toBe(0);
- const record = records.at(-1)!;
- expect(record.prefixes[0]?.stopReason).toBe('end-mismatch');
- // The replayed flow plus its repair is not a flow worth replaying: the
- // entry is gone, and the next passing run records a clean one.
- expect(readdirSync(cacheDir(project)).filter((name) => name.endsWith('.json'))).toHaveLength(0);
- }, 240_000);
-});
-
-const TRAILING_ACT_SUITE = `import { test } from 'e2e';
-
-test('cached step increments twice', async ({ app, agent }) => {
- await app.open();
- await agent.act('increment the counter twice');
-});
-`;
-
-describe('trace cache: only a verification step confirms a write', () => {
- let app: FixtureApp;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- }, 60_000);
-
- afterAll(async () => {
- await app?.close();
- });
-
- it('never writes the trace of a trailing act nothing asserted on, even though the attempt passed', async () => {
- const project = createProject({ 'tests/act.e2e.ts': TRAILING_ACT_SUITE });
- try {
- const record: ExecutorRecord = { calls: 0, prefixes: [] };
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- reporters: ['json'] as const,
- agents: { default: { executor: twoTapExecutor(record) } },
- cache: 'read-write' as const,
- },
- });
- expect(outcome.exitCode).toBe(0);
- expect(record.calls).toBe(1);
- expect(existsSync(cacheDir(project))).toBe(false);
- } finally {
- project.cleanup();
- }
- }, 120_000);
});
const TOOLS_ONLY_SUITE = `import { test } from 'e2e';
@@ -639,30 +431,7 @@ test('tools-only step', async ({ agent }) => {
});
`;
-/** Never observes, never acts: the shape of an executor that only uses its own tools. */
-function toolsOnlyExecutor(record: ExecutorRecord): StepExecutor {
- return {
- name: 'tools-only-executor',
- version: 'test',
- async runStep(context: StepExecutorContext) {
- record.calls += 1;
- record.prefixes.push(context.replayedPrefix);
- return { status: 'passed' as const, summary: 'done without the screen' };
- },
- };
-}
-
describe('trace cache: an engine-independent executor is not gated by the cache', () => {
- let app: FixtureApp;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- }, 60_000);
-
- afterAll(async () => {
- await app?.close();
- });
-
it('skips the end-state observation when the executor recorded no actions', async () => {
const project = createProject({ 'tests/tools.e2e.ts': TOOLS_ONLY_SUITE });
let executorFinished = false;
@@ -702,31 +471,9 @@ describe('trace cache: an engine-independent executor is not gated by the cache'
project.cleanup();
}
});
-
- it('runs the executor with caching on, without an opened app, and stages nothing', async () => {
- const project = createProject({ 'tests/tools.e2e.ts': TOOLS_ONLY_SUITE });
- try {
- const record: ExecutorRecord = { calls: 0, prefixes: [] };
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- reporters: ['json'] as const,
- agents: { default: { executor: toolsOnlyExecutor(record) } },
- cache: 'read-write' as const,
- },
- });
- expect(outcome.exitCode).toBe(0);
- expect(record.calls).toBe(1);
- expect(record.prefixes).toEqual([undefined]);
- expect(existsSync(cacheDir(project))).toBe(false);
- } finally {
- project.cleanup();
- }
- }, 120_000);
});
-describe('trace cache: modes that never write', () => {
+describe('trace cache: modes', () => {
let app: FixtureApp;
beforeAll(async () => {
@@ -737,24 +484,6 @@ describe('trace cache: modes that never write', () => {
await app?.close();
});
- it('caches by default: a config without a cache key records entries', async () => {
- const project = createProject({ 'tests/act.e2e.ts': SUITE });
- try {
- const record: ExecutorRecord = { calls: 0, prefixes: [] };
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- agents: { default: { executor: twoTapExecutor(record) } },
- },
- });
- expect(outcome.exitCode).toBe(0);
- expect(readdirSync(cacheDir(project))).toHaveLength(1);
- } finally {
- project.cleanup();
- }
- }, 120_000);
-
it('--no-cache overrides the config: nothing is recorded, and an entry recorded without it is left untouched', async () => {
const project = createProject({ 'tests/act.e2e.ts': SUITE });
const options = (record: ExecutorRecord, noCache: boolean) => ({
@@ -790,68 +519,36 @@ describe('trace cache: modes that never write', () => {
}
}, 180_000);
- it('read-only mode never creates the store', async () => {
- const project = createProject({ 'tests/act.e2e.ts': SUITE });
- try {
- const record: ExecutorRecord = { calls: 0, prefixes: [] };
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- agents: { default: { executor: twoTapExecutor(record) } },
- cache: 'read-only' as const,
- },
- });
- expect(outcome.exitCode).toBe(0);
- expect(record.calls).toBe(1);
- expect(existsSync(cacheDir(project))).toBe(false);
- } finally {
- project.cleanup();
- }
- }, 120_000);
-
- it('CI demotes an unset cache mode to read-only', async () => {
+ it('caches by default, and CI demotes an unset mode to read-only: it replays the entry and never rewrites it', async () => {
const project = createProject({ 'tests/act.e2e.ts': SUITE });
+ const options = (record: ExecutorRecord, env?: NodeJS.ProcessEnv) => ({
+ appUrl: app.url,
+ config: {
+ tests: 'tests/**/*.e2e.ts',
+ agents: { default: { executor: twoTapExecutor(record) } },
+ },
+ ...(env === undefined ? {} : { runOptions: { env } }),
+ });
try {
- const record: ExecutorRecord = { calls: 0, prefixes: [] };
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- agents: { default: { executor: twoTapExecutor(record) } },
- },
- runOptions: {
- env: { ...process.env, APP_URL: app.url, CI: '1' },
- },
- });
- expect(outcome.exitCode).toBe(0);
- expect(existsSync(cacheDir(project))).toBe(false);
- } finally {
- project.cleanup();
- }
- }, 120_000);
+ const first = await runExisting(project, options({ calls: 0, prefixes: [] }));
+ expect(first.exitCode).toBe(0);
+ expect(readdirSync(cacheDir(project))).toHaveLength(1);
- it('CI honors an explicit read-write as the project stating its trust', async () => {
- const project = createProject({ 'tests/act.e2e.ts': SUITE });
- try {
- const record: ExecutorRecord = { calls: 0, prefixes: [] };
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- agents: { default: { executor: twoTapExecutor(record) } },
- cache: 'read-write' as const,
- },
- runOptions: {
- env: { ...process.env, APP_URL: app.url, CI: '1' },
- },
- });
- expect(outcome.exitCode).toBe(0);
- expect(existsSync(cacheDir(project))).toBe(true);
+ // An entry from before the occurrence fields: a read-write replay would complete and rewrite it.
+ const { file, entry } = readOnlyEntry(project);
+ const { testId, targetId, instructionDigest } = entry.payload.recordedFor!;
+ writeFileSync(file, JSON.stringify({ ...entry, payload: { ...entry.payload, recordedFor: { testId, targetId, instructionDigest } } }), 'utf8');
+ const legacy = entryFileState(project);
+ const inCi: ExecutorRecord = { calls: 0, prefixes: [] };
+ const second = await runExisting(project, options(inCi, { ...process.env, APP_URL: app.url, CI: '1' }));
+ expect(second.exitCode).toBe(0);
+ expect(inCi.calls).toBe(0);
+ expect(actStep(second).cache).toEqual({ mode: 'self-finalized', replayedActions: 2, totalActions: 2 });
+ expect(entryFileState(project)).toEqual(legacy);
} finally {
project.cleanup();
}
- }, 120_000);
+ }, 180_000);
});
const PIN_SUITE = `import { test, expect } from 'e2e';
@@ -923,25 +620,6 @@ describe('trace cache: a bare-point tap replays like a coordinate-driven tool',
expect(step.cache).toMatchObject({ mode: 'self-finalized', replayedActions: 1, totalActions: 1 });
expect(step.events.filter((event) => event.kind === 'engine').map((event) => event.name)).toEqual(['tapAt']);
}, 240_000);
-
- it('hands the step to the executor when the recorded viewport is not the live one', async () => {
- rewriteOnlyEntry(project, (payload) => {
- const [tapAt] = payload.actions;
- if (tapAt?.name !== 'tapAt') throw new Error(`expected a tapAt, got ${JSON.stringify(payload.actions)}`);
- return { ...payload, actions: [{ ...tapAt, viewport: { width: 390, height: 844 } }] };
- });
-
- const outcome = await runExisting(project, options());
- expect(outcome.exitCode).toBe(0);
- const record = records.at(-1)!;
- expect(record.calls).toBe(1);
- // Nothing replayed, so the executor starts from the top with no prefix; the miss carries the reason.
- expect(record.prefixes[0]).toBeUndefined();
- const step = resultByTitle(outcome, 'picks the red pin').attempts.at(-1)!.steps.find((s) => s.api === 'agent.act')!;
- expect(step.cache).toMatchObject({ mode: 'missed', reason: 'viewport-changed', replayedActions: 0, totalActions: 1 });
- // The pass re-stages the entry with the live viewport: healed.
- expect(readOnlyEntry(project).entry.payload.actions[0]).toMatchObject({ viewport: { width: 1280, height: 720 } });
- }, 240_000);
});
const TODOS_SUITE = `import { test, expect } from 'e2e';
diff --git a/packages/e2e/tests/integration/agent.test.ts b/packages/e2e/tests/integration/agent.test.ts
index 7ae71f86d..1e3e1aa7f 100644
--- a/packages/e2e/tests/integration/agent.test.ts
+++ b/packages/e2e/tests/integration/agent.test.ts
@@ -420,51 +420,6 @@ describe('agent judgment tier', () => {
});
});
-describe('the built-in agent with a judge', () => {
- const SUITE = `import { test } from 'e2e';
-
-test('the judge judges the built-in agent assertions', async ({ app, agent }) => {
- await app.open();
- await agent.assert('the Home heading is visible');
- await agent.assert('the checkout page is visible');
-});
-`;
- let app: FixtureApp;
- let outcome: RunOutcome;
- let project: FixtureProject;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- // The built-in agent's assertions must reach the judgment tier and the
- // judge, not its own act loop on the actor model.
- const actor = installFakeModel(respond, { modelId: 'actor' });
- const judge = installFakeModel(respond, { modelId: 'judge' });
- const result = await runProject(
- { 'tests/judge.e2e.ts': SUITE },
- { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model: actor, judge } } } },
- );
- outcome = result.outcome;
- project = result.project;
- }, 120_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it('sends both assertions to the judge as single judgments and fails on the false one', () => {
- const result = resultByTitle(outcome, 'the judge judges the built-in agent assertions');
- expect(result.status).toBe('failed');
- expect(result.attempts.at(-1)!.error?.code).toBe('ASSERTION_FAILED');
- const judged = fakeCalls.filter((call) => call.schemaName === 'agent-judgment-2');
- expect(judged).toHaveLength(2);
- for (const call of judged) expect(call.modelId).toBe('judge');
- const steps = result.attempts.at(-1)!.steps.filter((step) => step.api === 'agent.assert');
- expect(steps.map((step) => step.status)).toEqual(['passed', 'failed']);
- for (const step of steps) expect(step.model).toMatchObject({ model: 'judge', calls: 1 });
- });
-});
-
describe('an inconclusive judgment after a secret fill', () => {
const SUITE = `import { test, credentials } from 'e2e';
diff --git a/packages/e2e/tests/integration/artifact-store-serial.test.ts b/packages/e2e/tests/integration/artifact-store-serial.test.ts
index b1e7b7811..2d90848f4 100644
--- a/packages/e2e/tests/integration/artifact-store-serial.test.ts
+++ b/packages/e2e/tests/integration/artifact-store-serial.test.ts
@@ -7,9 +7,12 @@
* report record carries. Both refs land on the group attempt's record.
*/
+import { existsSync } from 'node:fs';
+import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type { ArtifactStore, StoredArtifact } from '../../src/types.ts';
import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
+import { assertValidReport } from '../helpers/report-schema.ts';
import { createProject, runExisting, type FixtureProject } from '../helpers/run-project.ts';
const SERIAL_SUITE = `import { test, expect } from 'e2e';
@@ -91,7 +94,13 @@ describe('ArtifactStore across a serial group', () => {
for (const put of shots) expect(put.attemptId).toBe(groupAttempt.id);
const shotRecords = groupAttempt.artifacts.filter((artifact) => artifact.kind === 'screenshot');
expect(shotRecords).toHaveLength(2);
- for (const record of shotRecords) expect(record.ref).toMatch(/^store:\/\/screenshot\//);
+ for (const record of shotRecords) {
+ expect(record.ref).toMatch(/^store:\/\/screenshot\//);
+ expect(existsSync(path.join(project.dir, '.e2e', 'artifacts', record.path!))).toBe(true);
+ expect(record.size).toBeGreaterThan(0);
+ expect(record.sha256).toMatch(/^[0-9a-f]{64}$/);
+ }
+ assertValidReport(outcome.report);
// Every store identity is a real report identity.
const reportedIds = new Set(groups.flatMap((group) => group.attempts.map((attempt) => attempt.id)));
diff --git a/packages/e2e/tests/integration/cli-boot.test.ts b/packages/e2e/tests/integration/cli-boot.test.ts
index 70018a74d..9f2f11c4d 100644
--- a/packages/e2e/tests/integration/cli-boot.test.ts
+++ b/packages/e2e/tests/integration/cli-boot.test.ts
@@ -34,13 +34,10 @@ const HOOK = [
const WITHOUT_AI = ['--import', `data:text/javascript,${encodeURIComponent(HOOK)}`];
describe('the CLI without the ai package', () => {
- it('is really without it: the hook fails the import the way a missing package does', async () => {
+ it('answers --version and --help', async () => {
const script = "await import('ai').then(() => process.stdout.write('resolved'), (error) => process.stdout.write(error.code));";
const { stdout } = await execFileAsync(process.execPath, [...WITHOUT_AI, '--input-type=module', '-e', script]);
expect(stdout).toBe('ERR_MODULE_NOT_FOUND');
- });
-
- it('answers --version and --help', async () => {
const version = await execFileAsync(process.execPath, [...WITHOUT_AI, CLI, '--version']);
expect(version.stdout.trim()).toBe(VERSION);
const help = await execFileAsync(process.execPath, [...WITHOUT_AI, CLI, '--help']);
diff --git a/packages/e2e/tests/integration/cli-init.test.ts b/packages/e2e/tests/integration/cli-init.test.ts
index 047059926..ddcbe75e5 100644
--- a/packages/e2e/tests/integration/cli-init.test.ts
+++ b/packages/e2e/tests/integration/cli-init.test.ts
@@ -154,7 +154,7 @@ describe('initializing standalone projects', () => {
});
it.each([undefined, '{}', '{"type":"commonjs"}'])(
- 'loads a .ts config and runs .ts tests with their helpers when package.json is %s',
+ 'loads a .ts config and collects .ts tests with their helpers when package.json is %s, and runs them under commonjs',
async (manifest) => {
if (manifest !== undefined) writeFileSync(path.join(dir, 'package.json'), manifest);
writeFileSync(path.join(dir, 'e2e.config.ts'), CONFIG);
@@ -170,6 +170,7 @@ describe('initializing standalone projects', () => {
const config = resolveConfig(raw, { projectRoot: dir, env: {} });
const collection = await collect(config);
expect(collection.tests.map((test) => test.title)).toEqual(['helpers load as ES modules']);
+ if (manifest !== '{"type":"commonjs"}') return;
const { stdout } = await execFileAsync(process.execPath, [CLI, 'run', '--workers', '1', '--no-cache'], { cwd: dir });
expect(stdout).toContain('1 passed');
@@ -246,25 +247,6 @@ describe('initializing standalone projects', () => {
});
});
- it('explains a removed export and a wrong subpath in the config', async () => {
- await execFileAsync(process.execPath, [CLI, 'init', '--yes'], { cwd: dir });
- linkPackages('e2e');
- writeFileSync(
- path.join(dir, 'e2e.config.ts'),
- "import { defineConfig } from 'e2e';\nexport default defineConfig({ targets: [{ name: 'local', platform: 'test' }] });\n",
- );
- await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({
- code: 'CONFIG_LOAD_FAILED',
- message: expect.stringContaining('defineConfig was removed in e2e 0.5'),
- });
- // The import must be used, or the TypeScript transform elides it.
- writeFileSync(path.join(dir, 'e2e.config.ts'), "import { test } from 'e2e/test';\nexport default { marker: test };\n");
- await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({
- code: 'CONFIG_LOAD_FAILED',
- message: expect.stringContaining('e2e exports e2e, e2e/agent, e2e/engine'),
- });
- });
-
it('names look-alike test files when the globs match nothing', async () => {
await execFileAsync(process.execPath, [CLI, 'init', '--yes'], { cwd: dir });
linkPackages('e2e', 'web');
@@ -279,48 +261,4 @@ describe('initializing standalone projects', () => {
});
});
- it('preserves the original load error', async () => {
- writeFileSync(path.join(dir, 'e2e.config.ts'), "throw new Error('config setup failed');\n");
- await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({
- code: 'CONFIG_LOAD_FAILED',
- message: `failed to load config ${path.join(dir, 'e2e.config.ts')}: config setup failed`,
- });
- });
-
- it('keeps the code of a configuration error the config throws while it evaluates', async () => {
- linkPackages('e2e');
- writeFileSync(
- path.join(dir, 'e2e.config.ts'),
- "import { ConfigurationError } from 'e2e/engine';\nthrow new ConfigurationError('INVALID_CONFIG', 'web({ video }) was renamed web({ screencast })');\n",
- );
- await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({
- code: 'INVALID_CONFIG',
- message: 'web({ video }) was renamed web({ screencast })',
- });
- });
-
- it('keeps the code of a configuration error from another copy of e2e, and only a configuration one', async () => {
- const foreign = (category: string) =>
- `const error = new Error('refused by a second copy');\nObject.assign(error, { [Symbol.for('e2e.error.v1')]: true, category: '${category}', code: 'INVALID_CONFIG', retryable: false });\nthrow error;\n`;
- writeFileSync(path.join(dir, 'e2e.config.ts'), foreign('configuration'));
- await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({
- code: 'INVALID_CONFIG',
- message: 'refused by a second copy',
- });
- writeFileSync(path.join(dir, 'e2e.config.ts'), foreign('test'));
- await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({ code: 'CONFIG_LOAD_FAILED' });
- });
-
- it('loads a config through a symlink', async () => {
- const sourceDir = path.join(dir, 'source');
- mkdirSync(sourceDir);
- const source = path.join(sourceDir, 'e2e.config.ts');
- writeFileSync(source, CONFIG);
- const linked = path.join(dir, 'e2e.config.ts');
- symlinkSync(source, linked, 'file');
-
- await expect(loadConfigModule(linked)).resolves.toMatchObject({
- targets: [{ name: 'local', platform: 'test' }],
- });
- });
});
diff --git a/packages/e2e/tests/integration/cli-run.test.ts b/packages/e2e/tests/integration/cli-run.test.ts
new file mode 100644
index 000000000..b0e7bd2c2
--- /dev/null
+++ b/packages/e2e/tests/integration/cli-run.test.ts
@@ -0,0 +1,147 @@
+/**
+ * `e2e run` and `e2e list` through the built CLI on the fake engine, for what
+ * only the process boundary decides: the exit code a crashed worker maps to,
+ * shards that split one selection between two machines, and the defaults a
+ * worker resolves for itself when `CI` is set.
+ */
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterEach, describe, expect, it } from 'vitest';
+import type { Report1Document } from '../../src/report/build.ts';
+import { assertValidReport } from '../helpers/report-schema.ts';
+import { createProject, type FixtureProject } from '../helpers/run-project.ts';
+
+const CLI = fileURLToPath(new URL('../../dist/cli/bin.js', import.meta.url));
+const projects: FixtureProject[] = [];
+
+afterEach(() => {
+ for (const project of projects.splice(0)) project.cleanup();
+});
+
+/** A config on the fake engine, `behavior` its `createFakeEngine` options, `extra` more config keys. */
+function fakeConfig(behavior = '{}', extra = ''): string {
+ return `import type { E2EConfig } from 'e2e';
+import { createFakeEngine, FAKE_APP } from '../../helpers/fake-engine.ts';
+
+export default {
+ tests: 'tests/**/*.e2e.ts',
+ targets: [{ name: 'fake', platform: 'fake', engine: createFakeEngine(${behavior}).engine, app: FAKE_APP }],
+ workers: 1,
+ cache: 'off',${extra}
+} satisfies E2EConfig;
+`;
+}
+
+/** A fixture project removed after the test. */
+function fixture(files: Record): FixtureProject {
+ const created = createProject(files);
+ projects.push(created);
+ return created;
+}
+
+/** Writes a project and runs one CLI command in it; `CI` is unset unless `env` sets it. */
+function cli(files: Record, args: readonly string[], env: Record = {}) {
+ const created = fixture(files);
+ return { project: created, ...rerun(created, args, env) };
+}
+
+/** Runs one more CLI command in an existing project. */
+function rerun(existing: FixtureProject, args: readonly string[], env: Record = {}) {
+ const result = spawnSync(process.execPath, [CLI, ...args], {
+ cwd: existing.dir,
+ env: { ...process.env, CI: '', NO_COLOR: '1', ...env },
+ encoding: 'utf8',
+ timeout: 60_000,
+ });
+ expect(result.error).toBeUndefined();
+ return { exitCode: result.status, output: result.stdout + result.stderr, stdout: result.stdout };
+}
+
+/** The report the last run wrote. */
+function reportOf(ran: FixtureProject): Report1Document {
+ const report = JSON.parse(readFileSync(path.join(ran.dir, '.e2e', 'report.json'), 'utf8')) as Report1Document;
+ assertValidReport(report);
+ return report;
+}
+
+const test = (title: string, body = '') => `import { test } from 'e2e';\ntest('${title}', async ({ app }) => {\n await app.open();\n ${body}\n});\n`;
+
+describe('the run command through the CLI', () => {
+ it('exits 3 when a worker dies under a test', () => {
+ const { project, exitCode } = cli(
+ { 'e2e.config.ts': fakeConfig(), 'tests/crash.e2e.ts': test('crashes the worker', 'process.exit(7);') },
+ ['run'],
+ );
+ expect(exitCode).toBe(3);
+ const report = reportOf(project);
+ expect(report.run.exitCode).toBe(3);
+ expect(report.run.results.map((result) => result.attempts[0]?.error?.code)).toEqual(['WORKER_CRASH']);
+ });
+
+ it('splits one selection into shards that cover every test once, each with the setup its tests need', () => {
+ const files: Record = {
+ 'e2e.config.ts': fakeConfig('{ state: true }'),
+ 'tests/auth.setup.e2e.ts': `import { test } from 'e2e';\ntest.setup('sign in', { sessions: ['member'] }, async ({ app, session }) => {\n await app.open();\n await session.save('member');\n});\n`,
+ };
+ for (const name of ['a', 'b', 'c', 'd', 'e']) {
+ files[`tests/${name}.e2e.ts`] = `import { test } from 'e2e';\ntest('${name}', { session: 'member' }, async ({ app }) => {\n await app.open();\n});\n`;
+ }
+ const sharded = fixture(files);
+ const listed = (shard: string) => {
+ const { exitCode, stdout } = rerun(sharded, ['list', '--reporter', 'json', '--shard', shard]);
+ expect(exitCode).toBe(0);
+ const { pairs } = JSON.parse(stdout) as { pairs: { title: string; kind: string }[] };
+ return pairs;
+ };
+ const [first, second] = [listed('1/2'), listed('2/2')];
+ const tests = (pairs: { title: string; kind: string }[]) => pairs.filter((pair) => pair.kind === 'test').map((pair) => pair.title);
+ expect([...tests(first), ...tests(second)].toSorted()).toEqual(['a', 'b', 'c', 'd', 'e']);
+ expect(tests(first).filter((title) => tests(second).includes(title))).toEqual([]);
+ expect(tests(first).length).toBeGreaterThan(0);
+ expect(tests(second).length).toBeGreaterThan(0);
+ for (const pairs of [first, second]) {
+ expect(pairs.filter((pair) => pair.kind === 'setup').map((pair) => pair.title)).toEqual(['sign in']);
+ }
+
+ const ran = rerun(sharded, ['run', '--shard', '2/2']);
+ expect(ran.exitCode).toBe(0);
+ const executed = reportOf(sharded).run.results.filter((result) => result.attempts.length > 0);
+ expect(executed.map((result) => result.titlePath.at(-1)).toSorted()).toEqual(['sign in', ...tests(second)].toSorted());
+ });
+
+ it('under CI, refuses a focused test and records a trace only on the first retry', () => {
+ const focused = cli(
+ { 'e2e.config.ts': fakeConfig('{ trace: true }'), 'tests/only.e2e.ts': `import { test } from 'e2e';\ntest.only('focused', async () => {});\ntest('other', async () => {});\n` },
+ ['run'],
+ { CI: 'true' },
+ );
+ expect(focused.exitCode).toBe(2);
+ expect(focused.output).toContain('ONLY_IN_CI');
+
+ const flaky = cli(
+ {
+ 'e2e.config.ts': fakeConfig('{ trace: true }'),
+ 'tests/flaky.e2e.ts': `import { existsSync, writeFileSync } from 'node:fs';
+import { test } from 'e2e';
+test('fails once', { retries: 1 }, async ({ app }) => {
+ await app.open();
+ const marker = new URL('./failed-once', import.meta.url);
+ if (!existsSync(marker)) {
+ writeFileSync(marker, '');
+ throw new Error('first attempt fails');
+ }
+});
+`,
+ },
+ ['run'],
+ { CI: 'true' },
+ );
+ expect(flaky.exitCode).toBe(0);
+ const [result] = reportOf(flaky.project).run.results;
+ expect(result?.status).toBe('flaky');
+ expect(result?.attempts.map((attempt) => attempt.artifacts.some((artifact) => artifact.kind === 'trace'))).toEqual([false, true]);
+ });
+});
diff --git a/packages/e2e/tests/integration/config-load.test.ts b/packages/e2e/tests/integration/config-load.test.ts
index 173a87a2f..8f37e3d73 100644
--- a/packages/e2e/tests/integration/config-load.test.ts
+++ b/packages/e2e/tests/integration/config-load.test.ts
@@ -1,6 +1,6 @@
-/** Loading a config module: a config error it raises while it evaluates keeps its code, and any other throw is CONFIG_LOAD_FAILED. */
+/** Loading a config module: a config error it raises while it evaluates keeps its code, any other throw is CONFIG_LOAD_FAILED with the original message, and a symlinked config loads. */
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
@@ -50,8 +50,46 @@ describe('loadConfigModule', () => {
await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({ category: 'infrastructure', code: 'BROWSER_MISSING', message: 'no browser' });
});
- it('reports any other throw while the config evaluates as CONFIG_LOAD_FAILED', async () => {
- writeFileSync(path.join(dir, 'e2e.config.ts'), `throw new Error('boom');\nexport default {};\n`);
+ it('preserves the original load error', async () => {
+ writeFileSync(path.join(dir, 'e2e.config.ts'), "throw new Error('config setup failed');\n");
+ await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({
+ code: 'CONFIG_LOAD_FAILED',
+ message: `failed to load config ${path.join(dir, 'e2e.config.ts')}: config setup failed`,
+ });
+ });
+
+ it('keeps the code of a configuration error the config throws while it evaluates', async () => {
+ writeFileSync(
+ path.join(dir, 'e2e.config.ts'),
+ `import { ConfigurationError } from '${ENGINE}';\nthrow new ConfigurationError('INVALID_CONFIG', 'web({ video }) was renamed web({ screencast })');\n`,
+ );
+ await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({
+ code: 'INVALID_CONFIG',
+ message: 'web({ video }) was renamed web({ screencast })',
+ });
+ });
+
+ it('keeps the code of a configuration error from another copy of e2e, and only a configuration one', async () => {
+ const foreign = (category: string) =>
+ `const error = new Error('refused by a second copy');\nObject.assign(error, { [Symbol.for('e2e.error.v1')]: true, category: '${category}', code: 'INVALID_CONFIG', retryable: false });\nthrow error;\n`;
+ writeFileSync(path.join(dir, 'e2e.config.ts'), foreign('configuration'));
+ await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({
+ code: 'INVALID_CONFIG',
+ message: 'refused by a second copy',
+ });
+ writeFileSync(path.join(dir, 'e2e.config.ts'), foreign('test'));
await expect(loadConfigModule(path.join(dir, 'e2e.config.ts'))).rejects.toMatchObject({ code: 'CONFIG_LOAD_FAILED' });
});
+
+ it('loads a config through a symlink', async () => {
+ const sourceDir = path.join(dir, 'source');
+ mkdirSync(sourceDir);
+ const source = path.join(sourceDir, 'e2e.config.ts');
+ writeFileSync(source, "export default { targets: [{ name: 'local', platform: 'test' }] };\n");
+ const linked = path.join(dir, 'e2e.config.ts');
+ symlinkSync(source, linked, 'file');
+ await expect(loadConfigModule(linked)).resolves.toMatchObject({
+ targets: [{ name: 'local', platform: 'test' }],
+ });
+ });
});
diff --git a/packages/e2e/tests/integration/config-names.test.ts b/packages/e2e/tests/integration/config-names.test.ts
index 804519223..4bbfda564 100644
--- a/packages/e2e/tests/integration/config-names.test.ts
+++ b/packages/e2e/tests/integration/config-names.test.ts
@@ -1,4 +1,4 @@
-/** Target and agent names are artifact path segments: a name that is only dots is refused before anything runs. */
+/** Target names are artifact path segments: a name that is only dots is refused before anything runs. */
import { existsSync } from 'node:fs';
import path from 'node:path';
@@ -32,20 +32,4 @@ describe('all-dot names', () => {
project.cleanup();
}
});
-
- it('refuses an agent named `.` as INVALID_CONFIG', async () => {
- const { outcome, project } = await runProject(SUITE, {
- appUrl: APP_URL,
- config: { targets: [{ name: 'fake', platform: 'custom', engine: fakeEngine() }], agents: { '.': {} } },
- });
- try {
- expect(outcome.exitCode).toBe(2);
- expect(outcome.report.run.errors.map((error) => [error.code, error.message])).toEqual([
- ['INVALID_CONFIG', 'invalid agent name ".": names are ASCII letters, numbers, "_", "-", or ".", and cannot be only dots'],
- ]);
- expect(outcome.results).toHaveLength(0);
- } finally {
- project.cleanup();
- }
- });
});
diff --git a/packages/e2e/tests/integration/engine-contract.test.ts b/packages/e2e/tests/integration/engine-contract.test.ts
index 2362854e0..9b0af062d 100644
--- a/packages/e2e/tests/integration/engine-contract.test.ts
+++ b/packages/e2e/tests/integration/engine-contract.test.ts
@@ -383,7 +383,7 @@ test.describe('wizard', { serial: true, retries: 1 }, () => {
);
it(
- 'classifies EngineErrors from fixture surfaces (app.open) with the canonical mapping',
+ 'classifies EngineErrors from fixture surfaces (app.open) with the canonical mapping, and a plain Error as ENGINE_FAILURE',
async () => {
const failure = createFakeEngine({
onNavigate() {
@@ -416,6 +416,23 @@ test.describe('wizard', { serial: true, retries: 1 }, () => {
expect(secondResult.attempts[0]!.error?.code).toBe('UNSUPPORTED_CAPABILITY');
expect(second.outcome.exitCode).toBe(2);
second.project.cleanup();
+
+ const plain = createFakeEngine({
+ onNavigate() {
+ throw new TypeError('renderer gone');
+ },
+ });
+ const third = await runProject(
+ { 'tests/plain-error.e2e.ts': PASSING_TEST },
+ { appUrl: APP_URL, config: engineConfig(plain.engine) },
+ );
+ const thirdResult = resultByTitle(third.outcome, 'taps a node');
+ expect(thirdResult.status).toBe('failed');
+ expect(thirdResult.attempts[0]!.error?.category).toBe('infrastructure');
+ expect(thirdResult.attempts[0]!.error?.code).toBe('ENGINE_FAILURE');
+ expect(thirdResult.attempts[0]!.error?.message).toContain('renderer gone');
+ expect(third.outcome.exitCode).toBe(3);
+ third.project.cleanup();
},
60_000,
);
@@ -664,46 +681,6 @@ test('consumes session', { session: 'acct' }, async ({ app }) => {
60_000,
);
- it(
- 'retries retryable stale nodes against the engine and never repeats a possibly committed action',
- async () => {
- let performCalls = 0;
- const fake = createFakeEngine({
- perform() {
- performCalls += 1;
- if (performCalls === 1) {
- throw engineFailure('NODE_STALE', 'node went stale', true);
- }
- },
- });
- const { outcome, project } = await runProject(
- { 'tests/stale.e2e.ts': PASSING_TEST },
- { appUrl: APP_URL, config: engineConfig(fake.engine) },
- );
- expect(resultByTitle(outcome, 'taps a node').status).toBe('passed');
- expect(performCalls).toBe(2);
-
- let committedCalls = 0;
- const committed = createFakeEngine({
- perform() {
- committedCalls += 1;
- throw engineFailure('ACTION_MAY_HAVE_COMMITTED', 'maybe committed');
- },
- });
- const second = await runProject(
- { 'tests/committed.e2e.ts': PASSING_TEST },
- { appUrl: APP_URL, config: engineConfig(committed.engine) },
- );
- const result = resultByTitle(second.outcome, 'taps a node');
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error?.code).toBe('ACTION_FAILED');
- expect(committedCalls).toBe(1);
- project.cleanup();
- second.project.cleanup();
- },
- 60_000,
- );
-
it(
're-observes when an engine reports a retryable observation failure',
async () => {
@@ -769,27 +746,6 @@ test('consumes session', { session: 'acct' }, async ({ app }) => {
60_000,
);
- it(
- 'words the vision hint as a condition on an engine that declares screenshots but has produced no pixels',
- async () => {
- // The SPI has no pixel-capture declaration: an engine with the artifacts
- // capability may still return no pixels from observe, as this one does.
- const fake = createFakeEngine({ artifacts: true });
- const model = installFakeModel(() => judgment('inconclusive', 'the tree lists no state for the Submit button'));
- const { outcome, project } = await runProject(
- { 'tests/observe-inconclusive-artifacts.e2e.ts': OBSERVE_TEST },
- { appUrl: APP_URL, config: engineConfig(fake.engine, { agents: { default: { model } } }) },
- );
- const error = resultByTitle(outcome, 'asserts a node').attempts.at(-1)!.error!;
- expect(error.code).toBe('ASSERTION_INCONCLUSIVE');
- expect(error.message).toBe(
- 'the tree lists no state for the Submit button; the judge saw the semantic tree only; if the engine captures pixels, pass vision: true when the answer is in pixels',
- );
- project.cleanup();
- },
- 60_000,
- );
-
it(
'leaves vision out of an inconclusive judgment once a pixel request of the attempt was degraded',
async () => {
@@ -833,12 +789,13 @@ test('asserts after a degraded pixel request', async ({ app, agent }) => {
async () => {
// An observation is handed whatever remains of the invocation deadline,
// so one starting near the end cannot finish. The engine failure that
- // follows describes a truncated budget, not a broken app.
+ // follows describes a truncated budget, not a broken app. Only the
+ // second poll stalls, so the failure evidence after it observes at once.
let observeCalls = 0;
const fake = createFakeEngine({
async observe(operation) {
observeCalls += 1;
- if (observeCalls === 1) return;
+ if (observeCalls !== 2) return;
await new Promise((resolve) => setTimeout(resolve, operation.timeoutMs + 50));
throw engineFailure('ENGINE_FAILURE', 'observation ran out of budget');
},
@@ -875,29 +832,6 @@ test('asserts after a degraded pixel request', async ({ app, agent }) => {
60_000,
);
- it(
- 'normalizes a plain Error thrown by an engine member to infrastructure ENGINE_FAILURE',
- async () => {
- const fake = createFakeEngine({
- onNavigate() {
- throw new TypeError('renderer gone');
- },
- });
- const { outcome, project } = await runProject(
- { 'tests/plain-error.e2e.ts': PASSING_TEST },
- { appUrl: APP_URL, config: engineConfig(fake.engine) },
- );
- const result = resultByTitle(outcome, 'taps a node');
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error?.category).toBe('infrastructure');
- expect(result.attempts[0]!.error?.code).toBe('ENGINE_FAILURE');
- expect(result.attempts[0]!.error?.message).toContain('renderer gone');
- expect(outcome.exitCode).toBe(3);
- project.cleanup();
- },
- 60_000,
- );
-
it(
'ends the attempt when session restore fails after it started',
async () => {
@@ -1020,7 +954,7 @@ test('needs web', { requires: ['web'] }, async () => {});
);
it(
- 'registers an engine screenshot as an attempt artifact and reports unsupported gestures honestly',
+ 'registers an engine screenshot as an attempt artifact',
async () => {
const fake = createFakeEngine({ artifacts: true });
const file = `import { test } from 'e2e';
@@ -1029,10 +963,6 @@ test('takes evidence', async ({ app }) => {
await app.open('/');
await app.screenshot('after-open');
});
-
-test('swipes without a swipe capability', async ({ screen }) => {
- await screen.swipe({ direction: 'down' });
-});
`;
const { outcome, project } = await runProject(
{ 'tests/evidence.e2e.ts': file },
@@ -1042,10 +972,6 @@ test('swipes without a swipe capability', async ({ screen }) => {
expect(evidence.status).toBe('passed');
expect(evidence.attempts[0]!.artifacts.some((artifact) => artifact.kind === 'screenshot')).toBe(true);
expect(fake.operations.some((op) => op.method === 'artifacts.screenshot(after-open)')).toBe(true);
- const swipes = resultByTitle(outcome, 'swipes without a swipe capability');
- expect(swipes.status).toBe('failed');
- expect(swipes.attempts[0]!.error?.code).toBe('UNSUPPORTED_CAPABILITY');
- expect(swipes.attempts[0]!.error?.message).toContain('swipe');
project.cleanup();
},
60_000,
@@ -1223,20 +1149,6 @@ test('fails on purpose', async ({ app }) => {
60_000,
);
- it(
- 'records every retry with on-all-retries',
- async () => {
- const fake = createFakeEngine({ video: true });
- const { outcome, project } = await runProject(
- { 'tests/fail.e2e.ts': FAILING_TEST },
- { appUrl: APP_URL, config: engineConfig(fake.engine, { video: 'on-all-retries', retries: 2 }) },
- );
- expect([0, 1, 2].map((attempt) => videosOf(outcome, 'fails on purpose', attempt).length)).toEqual([0, 1, 1]);
- project.cleanup();
- },
- 60_000,
- );
-
it(
"lets a test's own video win over --video, and a target's win over the config",
async () => {
diff --git a/packages/e2e/tests/integration/engine-target.test.ts b/packages/e2e/tests/integration/engine-target.test.ts
index 12933eca3..5ea94571b 100644
--- a/packages/e2e/tests/integration/engine-target.test.ts
+++ b/packages/e2e/tests/integration/engine-target.test.ts
@@ -6,7 +6,7 @@
*/
import { describe, expect, it } from 'vitest';
-import { defineEngine, type EngineFixtureContext } from '../../src/engine/index.ts';
+import { defineEngine } from '../../src/engine/index.ts';
import type { StepExecutor } from '../../src/agent/executor.ts';
import type { SemanticNode } from '../../src/engine/surface.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
@@ -67,41 +67,15 @@ export default {
} satisfies E2EConfig;
`;
-const FIXTURE_SUITE = `import { test, expect } from 'e2e';
-
-test('a contributed fixture runs with harness discipline', async (fixtures) => {
- const device = (fixtures as unknown as { device: { reset(): Promise; shake(): Promise } }).device;
- await expect(await device.reset()).toBe('reset-done');
- await expect(await device.shake()).toBe('shaken');
-});
-`;
-
-const STATE_SUITE = `import { test, expect } from 'e2e';
-
-test.setup('seed the counter', { sessions: ['seeded'] }, async ({ screen, session }) => {
- await screen.getByRole('button', { name: 'Increment' }).tap();
- await session.save('seeded');
-});
-
-test('restores the seeded counter', { session: 'seeded' }, async ({ screen }) => {
- await expect(screen.getByRole('status')).toHaveText('1');
-});
-`;
-
-
/** A two-node screen: a counter value and a button that increments it. */
function toyEngine(
options: {
withLocate?: boolean;
- withFixtures?: boolean;
- withState?: boolean;
- withIsolation?: boolean;
withoutInit?: boolean;
withPrepare?: 'ok' | 'fail' | 'hang-finish';
} = {},
) {
const lifecycle: string[] = [];
- const fixtureCalls: string[] = [];
let initEnv: Record = {};
let count = 0;
const nodes = (): SemanticNode[] => [
@@ -146,17 +120,6 @@ function toyEngine(
info.log(`booted ${info.env['TOY_POOL']?.split(',')[0] ?? 'no device'}`);
},
}),
- ...(options.withIsolation !== true
- ? {}
- : {
- async startAttempt(context: { attemptId: string; artifactsDir: string }) {
- lifecycle.push(`startAttempt:${context.artifactsDir.length > 0 ? 'dir' : 'nodir'}`);
- count = 0;
- },
- async endAttempt() {
- lifecycle.push('endAttempt');
- },
- }),
async dispose() {
lifecycle.push('dispose');
},
@@ -188,37 +151,8 @@ function toyEngine(
);
},
}),
- ...(options.withFixtures !== true
- ? {}
- : {
- fixtures: {
- device: (context: EngineFixtureContext) => context.fixture('device', {
- async reset() {
- fixtureCalls.push(`reset:${context.targetName}`);
- count = 0;
- return 'reset-done';
- },
- async shake() {
- fixtureCalls.push('shake');
- return 'shaken';
- },
- }, { reset: { kind: 'resource' }, shake: { kind: 'resource' } }),
- },
- }),
- ...(options.withState !== true
- ? {}
- : {
- state: {
- async capture() {
- return { format: 'toy', version: 1, data: { count } };
- },
- async restore(state: { data: unknown }) {
- count = (state.data as { count: number }).count;
- },
- },
- }),
});
- return { initEnv: () => initEnv, engine, lifecycle, fixtureCalls, current: () => count };
+ return { initEnv: () => initEnv, engine, lifecycle, current: () => count };
}
/** Observes, taps until the counter reads the goal, verifies, concludes. */
@@ -287,116 +221,8 @@ describe('engine targets', () => {
env: { ...process.env, APP_URL: '', CI: '' },
quiet: true,
});
- expect(outcome.exitCode).not.toBe(0);
- const result = outcome.results[0];
- const message = result?.attempts.at(-1)?.error?.message ?? '';
- expect(message).toMatch(/no engine capability|UNSUPPORTED/i);
- } finally {
- project.cleanup();
- }
- });
-
- it('runs the deterministic screen/expect tier over engine.locate', async () => {
- const toy = toyEngine({ withLocate: true });
- const project = createProject({ 'tests/screen.e2e.ts': DETERMINISTIC_SUITE });
- try {
- const outcome = await run({
- cwd: project.dir,
- rawConfig: {
- targets: [{ name: 'toy-sim', platform: 'ios', engine: toy.engine }],
- cache: 'off',
- },
- env: { ...process.env, APP_URL: '', CI: '' },
- quiet: true,
- });
- expect(outcome.exitCode).toBe(0);
- expect(toy.current()).toBe(2);
- assertValidReport(outcome.report);
- } finally {
- project.cleanup();
- }
- });
-
- it('runs a contributed fixture with harness step discipline', async () => {
- const toy = toyEngine({ withFixtures: true });
- const project = createProject({ 'tests/fixture.e2e.ts': FIXTURE_SUITE });
- try {
- const outcome = await run({
- cwd: project.dir,
- rawConfig: {
- targets: [{ name: 'toy-sim', platform: 'ios', engine: toy.engine }],
- cache: 'off',
- },
- env: { ...process.env, APP_URL: '', CI: '' },
- quiet: true,
- });
- expect(outcome.exitCode).toBe(0);
- expect(outcome.exitCode).toBe(0);
- expect(toy.fixtureCalls).toEqual(['reset:toy-sim', 'shake']);
- // Every contributed call is a recorded step, named ..
- const steps = outcome.results[0]?.attempts[0]?.steps ?? [];
- const names = steps.map((step) => step.api);
- expect(names).toContain('device.reset');
- expect(names).toContain('device.shake');
- assertValidReport(outcome.report);
- } finally {
- project.cleanup();
- }
- });
-
- it('captures and restores opaque engine state across a session', async () => {
- const toy = toyEngine({ withLocate: true, withState: true });
- const project = createProject({ 'tests/state.e2e.ts': STATE_SUITE });
- try {
- const outcome = await run({
- cwd: project.dir,
- rawConfig: {
- targets: [{ name: 'toy-sim', platform: 'ios', engine: toy.engine }],
- cache: 'off',
- },
- env: { ...process.env, APP_URL: '', CI: '' },
- quiet: true,
- });
- expect(outcome.exitCode).toBe(0);
- assertValidReport(outcome.report);
- } finally {
- project.cleanup();
- }
- });
-
-
- it('resets per-attempt state via startAttempt/endAttempt isolation', async () => {
- const toy = toyEngine({ withLocate: true, withIsolation: true });
- const suite = `import { test, expect } from 'e2e';
-
-test('first attempt starts fresh', async ({ screen }) => {
- await screen.getByRole('button', { name: 'Increment' }).tap();
- await expect(screen.getByRole('status')).toHaveText('1');
-});
-
-test('second attempt also starts fresh', async ({ screen }) => {
- await screen.getByRole('button', { name: 'Increment' }).tap();
- await expect(screen.getByRole('status')).toHaveText('1');
-});
-`;
- const project = createProject({ 'tests/iso.e2e.ts': suite });
- try {
- const outcome = await run({
- cwd: project.dir,
- rawConfig: {
- targets: [{ name: 'toy-sim', platform: 'ios', engine: toy.engine }],
- cache: 'off',
- },
- env: { ...process.env, APP_URL: '', CI: '' },
- quiet: true,
- });
- expect(outcome.exitCode).toBe(0);
- expect(toy.lifecycle.filter((e) => e.startsWith('startAttempt'))).toEqual([
- 'startAttempt:dir',
- 'startAttempt:dir',
- ]);
- expect(toy.lifecycle.filter((e) => e === 'endAttempt')).toHaveLength(2);
- assertValidReport(outcome.report);
+ expect(outcome.exitCode).toBe(2);
+ expect(outcome.results[0]?.attempts.at(-1)?.error).toMatchObject({ code: 'UNSUPPORTED_CAPABILITY', category: 'configuration' });
} finally {
project.cleanup();
}
@@ -577,31 +403,4 @@ test('second attempt also starts fresh', async ({ screen }) => {
project.cleanup();
}
});
-
- it('gates an undeclared fixture at selection via requires', async () => {
- const toy = toyEngine(); // no fixtures declared
- const project = createProject({
- 'tests/req.e2e.ts': `import { test } from 'e2e';
-
-test('needs device', { requires: ['device'] }, async () => {});
-`,
- });
- try {
- const outcome = await run({
- cwd: project.dir,
- rawConfig: {
- targets: [{ name: 'toy-sim', platform: 'ios', engine: toy.engine }],
- cache: 'off',
- },
- env: { ...process.env, APP_URL: '', CI: '' },
- quiet: true,
- passWithNoTests: true,
- });
- const result = outcome.results[0];
- expect(result?.status).toBe('skipped');
- expect(result?.skip?.cause).toBe('capability-unavailable');
- } finally {
- project.cleanup();
- }
- });
});
diff --git a/packages/e2e/tests/integration/expect-poll-scope.test.ts b/packages/e2e/tests/integration/expect-poll-scope.test.ts
index bb72e508c..21c9180ca 100644
--- a/packages/e2e/tests/integration/expect-poll-scope.test.ts
+++ b/packages/e2e/tests/integration/expect-poll-scope.test.ts
@@ -1,10 +1,9 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
import {
resultByTitle,
runProjectWithConfigFile,
- workerConfigSource,
+ workerFakeConfigSource,
type FixtureProject,
type RunOutcome,
} from '../helpers/run-project.ts';
@@ -22,13 +21,8 @@ test('an unawaited poll fails the body that started it', async () => {
.toBe(true);
});
-test('an unawaited poll that would pass still fails the body', async () => {
- let reads = 0;
- expect.poll(() => ++reads > 3, { interval: 50 }).toBe(true);
-});
-
test('a body error stays primary with the unawaited poll beside it', async () => {
- expect.poll(() => false, { timeout: 400 }).toBe(true);
+ expect.poll(() => false, { timeout: 200 }).toBe(true);
throw new Error('the body gave up');
});
@@ -43,7 +37,7 @@ test('an awaited passing poll passes', async () => {
test('the test after them is never charged, and the abandoned poll read no more', async () => {
const before = counters.abandonedReads;
- await new Promise((resolve) => setTimeout(resolve, 800));
+ await new Promise((resolve) => setTimeout(resolve, 500));
expect(counters.abandonedReads).toBe(before);
});
`;
@@ -52,27 +46,27 @@ const HOOKS_SUITE = `import { test, expect } from 'e2e';
test.describe('afterEach', () => {
test.afterEach(async () => {
- expect.poll(() => false, { timeout: 400 }).toBe(true);
+ expect.poll(() => false, { timeout: 200 }).toBe(true);
});
test('owns the afterEach poll', async () => {});
});
test.describe('beforeEach', () => {
test.beforeEach(async () => {
- expect.poll(() => false, { timeout: 400 }).toBe(true);
+ expect.poll(() => false, { timeout: 200 }).toBe(true);
});
test('owns the beforeEach poll', async () => {});
});
test.describe('afterAll', () => {
test.afterAll(async () => {
- expect.poll(() => false, { timeout: 400 }).toBe(true);
+ expect.poll(() => false, { timeout: 200 }).toBe(true);
});
test('runs before the afterAll poll', async () => {});
});
test('the test after the hooks is never charged', async () => {
- await new Promise((resolve) => setTimeout(resolve, 800));
+ await new Promise((resolve) => setTimeout(resolve, 500));
});
`;
@@ -82,7 +76,7 @@ const LEAKY_FIXTURE_SUITE = `import { test as base, expect } from 'e2e';
const test = base.extend<{ leaky: number }>({
leaky: async (_fixtures, use) => {
await use(1);
- expect.poll(() => false, { timeout: 400 }).toBe(true);
+ expect.poll(() => false, { timeout: 200 }).toBe(true);
},
});
@@ -96,7 +90,7 @@ const TIDY_FIXTURE_SUITE = `import { test as base, expect } from 'e2e';
const test = base.extend<{ tidy: number }>({
tidy: async (_fixtures, use) => {
await use(1);
- await expect.poll(() => true, { timeout: 400 }).toBe(true);
+ await expect.poll(() => true, { timeout: 200 }).toBe(true);
},
});
@@ -109,38 +103,48 @@ const BEFORE_ALL_SUITE = `import { test, expect } from 'e2e';
test.describe('beforeAll', () => {
test.beforeAll(async () => {
- expect.poll(() => false, { timeout: 400 }).toBe(true);
+ expect.poll(() => false, { timeout: 200 }).toBe(true);
});
test('sits under the beforeAll poll', async () => {});
});
test('the test after the beforeAll is never charged', async () => {
- await new Promise((resolve) => setTimeout(resolve, 800));
+ await new Promise((resolve) => setTimeout(resolve, 500));
});
`;
/** A body the timeout cut keeps running; the poll it starts later is its own, never the next test's. */
const TIMED_OUT_SUITE = `import { test, expect } from 'e2e';
-test('times out, then polls from its leftover body', { timeout: 1000 }, async () => {
- await new Promise((resolve) => setTimeout(resolve, 1500));
- await expect.poll(() => false, { timeout: 2000 }).toBe(true);
+test('times out, then polls from its leftover body', { timeout: 300 }, async () => {
+ await new Promise((resolve) => setTimeout(resolve, 500));
+ await expect.poll(() => false, { timeout: 500 }).toBe(true);
});
test('runs while the leftover body polls, and is never charged', async () => {
- await new Promise((resolve) => setTimeout(resolve, 1500));
+ await new Promise((resolve) => setTimeout(resolve, 1000));
+});
+`;
+
+const STEPS_SUITE = `import { test } from 'e2e';
+
+test('a step call without await', async ({ app }) => {
+ app.open();
+});
+
+test('a step call without await before the body throws', async ({ app }) => {
+ app.open();
+ throw new Error('the body gave up');
});
`;
const NOT_AWAITED = /returned before expect\.poll\(\.\.\.\)\.toBe\(\.\.\.\) finished; put `await` in front of every expect\.poll call$/;
describe('expect.poll not awaited, in a run', () => {
- let app: FixtureApp;
let outcome: RunOutcome;
let project: FixtureProject;
beforeAll(async () => {
- app = await startFixtureApp();
({ outcome, project } = await runProjectWithConfigFile(
{
'tests/a-body.e2e.ts': BODY_SUITE,
@@ -149,14 +153,14 @@ describe('expect.poll not awaited, in a run', () => {
'tests/b-tidy-fixture.e2e.ts': TIDY_FIXTURE_SUITE,
'tests/c-before-all.e2e.ts': BEFORE_ALL_SUITE,
'tests/d-timed-out.e2e.ts': TIMED_OUT_SUITE,
+ 'tests/e-steps.e2e.ts': STEPS_SUITE,
},
- { appUrl: app.url, configSource: workerConfigSource(1) },
+ { configSource: workerFakeConfigSource(1) },
));
}, 180_000);
afterAll(async () => {
project?.cleanup();
- await app?.close();
});
it('produces a schema-valid report and fails the run', () => {
@@ -175,17 +179,28 @@ describe('expect.poll not awaited, in a run', () => {
expect(attempt.secondaryErrors).toEqual([]);
});
- it('fails the body for an unawaited poll that would have passed', () => {
- const attempt = resultByTitle(outcome, 'an unawaited poll that would pass still fails the body').attempts[0]!;
- expect(attempt.error?.code).toBe('STEP_NOT_AWAITED');
- });
-
it('keeps the body error primary and notes the unawaited poll beside it', () => {
const attempt = resultByTitle(outcome, 'a body error stays primary with the unawaited poll beside it').attempts[0]!;
expect(attempt.error).toMatchObject({ code: 'ERROR', message: 'the body gave up' });
expect(attempt.secondaryErrors.map((error) => [error.code, error.phase])).toEqual([['STEP_NOT_AWAITED', 'body']]);
});
+ it('fails a body that returns before a step it started finished with STEP_NOT_AWAITED', () => {
+ const attempt = resultByTitle(outcome, 'a step call without await').attempts[0]!;
+ expect(attempt.error).toMatchObject({ code: 'STEP_NOT_AWAITED', phase: 'body' });
+ expect(attempt.error?.message).toContain('app.open');
+ expect(attempt.error?.source?.file).toBe('tests/e-steps.e2e.ts');
+ expect(attempt.steps).toHaveLength(1);
+ expect(attempt.steps[0]).toMatchObject({ api: 'app.open', status: 'failed', error: { code: 'STEP_NOT_AWAITED' } });
+ });
+
+ it('keeps the body error primary and notes the un-awaited step beside it', () => {
+ const attempt = resultByTitle(outcome, 'a step call without await before the body throws').attempts[0]!;
+ expect(attempt.error).toMatchObject({ code: 'ERROR', message: 'the body gave up' });
+ expect(attempt.secondaryErrors.map((error) => error.code)).toEqual(['STEP_NOT_AWAITED']);
+ expect(attempt.steps[0]).toMatchObject({ api: 'app.open', status: 'failed', error: { code: 'STEP_NOT_AWAITED' } });
+ });
+
it('leaves awaited polls alone', () => {
const failing = resultByTitle(outcome, 'an awaited failing poll fails with its own error').attempts[0]!;
expect(failing.error?.code).toBe('ASSERTION_FAILED');
diff --git a/packages/e2e/tests/integration/expect-soft.test.ts b/packages/e2e/tests/integration/expect-soft.test.ts
index a8f7a723d..9916baef3 100644
--- a/packages/e2e/tests/integration/expect-soft.test.ts
+++ b/packages/e2e/tests/integration/expect-soft.test.ts
@@ -1,24 +1,15 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
import { resultByTitle, runProject, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
-const SOFT_SUITE = `import { test } from '@e2e-dev/web';
-import { expect } from 'e2e';
+const SOFT_SUITE = `import { test, expect } from 'e2e';
-test('soft failures let the body run on', async ({ app, screen, browser }) => {
+test('soft failures let the body run on', async ({ app, screen }) => {
await app.open();
expect.soft(1, 'the count').toBe(2);
- await expect.soft(screen.getByRole('heading', { name: 'Nowhere' })).toBeVisible({ timeout: 300 });
- await expect.soft(browser).toHaveURL('/nowhere', { timeout: 300 });
+ await expect.soft(screen.getByRole('button', { name: 'Submit' })).toBeHidden({ timeout: 300 });
expect.soft('ok').toBe('ok');
- await expect(screen.getByRole('heading', { name: 'Home' })).toBeVisible();
-});
-
-test('every soft matcher passing keeps the attempt green', async ({ app, screen }) => {
- await app.open();
- expect.soft({ id: 1, name: 'Ada' }).toMatchObject({ id: expect.any(Number) });
- await expect.soft(screen.getByRole('heading', { name: 'Home' })).toBeVisible();
+ await expect(screen.getByRole('button', { name: 'Submit' })).toBeVisible();
});
test('a body error stays primary over the soft failures', async ({ app }) => {
@@ -27,7 +18,7 @@ test('a body error stays primary over the soft failures', async ({ app }) => {
throw new Error('the body gave up');
});
-test('a timed-out body still settles its soft failures', { timeout: 1000 }, async ({ app }) => {
+test('a timed-out body still settles its soft failures', { timeout: 300 }, async ({ app }) => {
await app.open();
expect.soft(1).toBe(2);
await new Promise(() => {});
@@ -45,18 +36,15 @@ test.describe('soft in a hook', () => {
`;
describe('expect.soft in a run', () => {
- let app: FixtureApp;
let outcome: RunOutcome;
let project: FixtureProject;
beforeAll(async () => {
- app = await startFixtureApp();
- ({ outcome, project } = await runProject({ 'tests/soft.e2e.ts': SOFT_SUITE }, { appUrl: app.url }));
+ ({ outcome, project } = await runProject({ 'tests/soft.e2e.ts': SOFT_SUITE }, {}));
}, 180_000);
afterAll(async () => {
project?.cleanup();
- await app?.close();
});
it('produces a schema-valid report', () => {
@@ -70,29 +58,21 @@ describe('expect.soft in a run', () => {
const attempt = result.attempts[0]!;
expect(attempt.error).toMatchObject({ code: 'ASSERTION_FAILED', phase: 'body' });
const lines = attempt.error!.message.split('\n');
- expect(lines[0]).toBe('3 soft assertions failed');
+ expect(lines[0]).toBe('2 soft assertions failed');
expect(lines[1]).toBe('1. the count: expected 1 to be 2');
- expect(lines[2]).toBe('2. expect.toBeVisible failed');
- expect(lines.filter((line) => line.startsWith(' locator: ') && line.includes('Nowhere'))).toHaveLength(1);
- expect(attempt.error!.message).toMatch(/\n3\. expect\.toHaveURL failed/);
+ expect(lines[2]).toBe('2. expect.toBeHidden failed');
+ expect(lines.filter((line) => line.startsWith(' locator: ') && line.includes('Submit'))).toHaveLength(1);
expect(attempt.error?.source?.file).toBe('tests/soft.e2e.ts');
- // The failed locator and web matchers are recorded steps; the hard matcher after them still ran and passed.
+ // The failed locator matcher is a recorded step; the hard matcher after it still ran and passed.
expect(attempt.steps.map((step) => [step.api, step.status])).toEqual([
['app.open', 'passed'],
- ['expect.toBeVisible', 'failed'],
- ['expect.toHaveURL', 'failed'],
+ ['expect.toBeHidden', 'failed'],
['expect.toBeVisible', 'passed'],
]);
expect(attempt.steps[1]?.error?.code).toBe('ASSERTION_FAILED');
expect(attempt.secondaryErrors).toEqual([]);
});
- it('passes when every soft matcher passes', () => {
- const result = resultByTitle(outcome, 'every soft matcher passing keeps the attempt green');
- expect(result.status).toBe('passed');
- expect(result.attempts[0]!.steps.map((step) => step.status)).toEqual(['passed', 'passed']);
- });
-
it('keeps the body error primary and notes the soft failures beside it', () => {
const result = resultByTitle(outcome, 'a body error stays primary over the soft failures');
expect(result.status).toBe('failed');
diff --git a/packages/e2e/tests/integration/explore.test.ts b/packages/e2e/tests/integration/explore.test.ts
index 05834ecc9..ffe03385c 100644
--- a/packages/e2e/tests/integration/explore.test.ts
+++ b/packages/e2e/tests/integration/explore.test.ts
@@ -11,7 +11,6 @@ import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { defineTool } from '../../src/agent/public.ts';
import type { StepExecutor } from '../../src/agent/executor.ts';
-import type { AgentConfig } from '../../src/types.ts';
import { explore, type ExploreOptions, type ExploreOutcome } from '../../src/explore/index.ts';
import { FINDING_TOOL_NAME } from '../../src/explore/executor.ts';
import type { ModelInstance } from '../../src/types.ts';
@@ -65,15 +64,14 @@ async function runExplore(
project: FixtureProject,
app: FixtureApp,
model: ModelInstance,
- options: Partial & { projectAgent?: AgentConfig | undefined } = {},
+ options: Partial = {},
): Promise {
- const { projectAgent, ...rest } = options;
const notices: string[] = [];
const outcome = await explore({
cwd: project.dir,
rawConfig: {
targets: [{ name: 'web', engine: web(), app: { url: app.url } }] as never,
- agents: { default: projectAgent ?? { model } },
+ agents: { default: { model } },
// The scripted loop answers instantly; the deterministic engine budget is the one that matters.
actionTimeout: 10_000,
},
@@ -81,7 +79,7 @@ async function runExplore(
maxSteps: 2,
timeoutMs: 180_000,
notice: (message) => notices.push(message),
- ...rest,
+ ...options,
});
return Object.assign(outcome, { notices });
}
@@ -191,9 +189,8 @@ describe('e2e explore', () => {
expect(written.run.explore).toEqual(record);
}, 120_000);
- it('passes with warnings only, and offers the project tools of the agents entry', async () => {
+ it('passes with warnings only', async () => {
let plans = 0;
- const seen: string[][] = [];
const model = installExploreModel({
plan: () => {
plans += 1;
@@ -202,24 +199,13 @@ describe('e2e explore', () => {
: { decision: 'finish', summary: 'Only polish items.' };
},
loop: (call) => {
- seen.push([...call.toolNames]);
- if (call.turn === 1) return [{ toolName: 'ping', input: {} }];
- if (call.turn === 2) return [{ toolName: FINDING_TOOL_NAME, input: { ...COUNTER_FINDING, kind: 'warning', severity: 1 } }];
+ if (call.turn === 1) return [{ toolName: FINDING_TOOL_NAME, input: { ...COUNTER_FINDING, kind: 'warning', severity: 1 } }];
return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'Menu toggles' } }];
},
});
- const ping = defineTool(
- { description: 'answers pong', inputSchema: z.object({}), execute: async () => 'pong' },
- { mutates: false },
- );
- const outcome = await runExplore(project, app, model, {
- projectAgent: { model, tools: { ping }, system: 'Project guidance line.' },
- });
+ const outcome = await runExplore(project, app, model);
expect(outcome.exitCode).toBe(0);
expect(outcome.status).toBe('passed');
- expect(seen[0]).toContain('ping');
- expect(seen[0]).toContain(FINDING_TOOL_NAME);
- expect(loopCalls.at(-1)!.toolResults).toContain('pong');
expect(outcome.report.run.explore!.findings[0]).toMatchObject({ kind: 'warning', severity: 1 });
expect((outcome as unknown as { notices: string[] }).notices).toEqual([]);
}, 120_000);
@@ -298,26 +284,6 @@ describe('e2e explore', () => {
]);
}, 120_000);
- it('is blocked when the agent explores nothing, and replaces a custom executor with a notice', async () => {
- const model = installExploreModel({
- plan: () => ({ decision: 'finish', summary: 'Nothing here.' }),
- loop: () => [{ toolName: 'complete_step', input: { status: 'passed', summary: 'unused' } }],
- });
- const custom: StepExecutor = {
- name: 'house-brain',
- runStep: async () => ({ status: 'passed', summary: 'never runs' }),
- };
- const outcome = await runExplore(project, app, model, {
- projectAgent: { model, executor: custom },
- });
- expect(outcome.status).toBe('blocked');
- expect(outcome.report.run.explore).toMatchObject({ ended: 'finished', steps: [], findings: [] });
- expect(outcome.report.run.results[0]!.attempts[0]!.error?.code).toBe('AUTOMATION_UNSUPPORTED');
- expect((outcome as unknown as { notices: string[] }).notices).toEqual([
- 'agents.default is a custom executor ("house-brain"); explore runs the built-in agent instead',
- ]);
- }, 120_000);
-
it.each([
{ firstStatus: 'blocked', expectedStatus: 'blocked', exitCode: 3 },
{ firstStatus: 'passed', expectedStatus: 'passed', exitCode: 0 },
@@ -395,6 +361,7 @@ describe('e2e explore', () => {
},
loop: (call) => {
seen.push([...call.toolNames]);
+ if (call.turn === 1) return [{ toolName: 'ping', input: {} }];
return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'looked' } }];
},
});
@@ -421,6 +388,7 @@ describe('e2e explore', () => {
// The ux agent's tool is in the explorer's vocabulary.
expect(seen[0]).toContain('ping');
expect(seen[0]).toContain(FINDING_TOOL_NAME);
+ expect(loopCalls.at(-1)!.toolResults).toContain('pong');
await expect(
explore({ cwd: project.dir, rawConfig: { targets: [{ name: 'web', engine: web(), app: { url: app.url } }] as never, agents: { default: { model } } }, agent: 'nope' }),
@@ -441,6 +409,27 @@ describe('e2e explore', () => {
expect(notices).toContain('agents.custom is a custom executor ("math-brain"); explore runs the built-in agent instead');
}, 60_000);
+ it('replaces a custom executor with the built-in agent on the model it brought, and is blocked when nothing is explored', async () => {
+ const model = installExploreModel({
+ plan: () => ({ decision: 'finish', summary: 'Nothing here.' }),
+ loop: () => [{ toolName: 'complete_step', input: { status: 'passed', summary: 'unused' } }],
+ });
+ const custom: StepExecutor = { name: 'house-brain', model, runStep: async () => ({ status: 'passed', summary: 'never runs' }) };
+ const notices: string[] = [];
+ const outcome = await explore({
+ cwd: project.dir,
+ rawConfig: { targets: [{ name: 'web', engine: web(), app: { url: app.url } }] as never, agents: { default: { executor: custom } } },
+ goal: 'Explore the home page and find bugs',
+ maxSteps: 2,
+ timeoutMs: 180_000,
+ notice: (message) => notices.push(message),
+ });
+ expect(outcome.status).toBe('blocked');
+ expect(outcome.report.run.explore).toMatchObject({ ended: 'finished', steps: [], findings: [] });
+ expect(outcome.report.run.results[0]!.attempts[0]!.error?.code).toBe('AUTOMATION_UNSUPPORTED');
+ expect(notices).toEqual(['agents.default is a custom executor ("house-brain"); explore runs the built-in agent instead, with the model that executor brought']);
+ }, 120_000);
+
it('lets the explorer sign in with a configured credential through type_secret, never seeing the password', async () => {
const planPrompts: string[] = [];
let fillResult: string | undefined;
diff --git a/packages/e2e/tests/integration/extended-fixtures.test.ts b/packages/e2e/tests/integration/extended-fixtures.test.ts
index aadb8532e..05920a548 100644
--- a/packages/e2e/tests/integration/extended-fixtures.test.ts
+++ b/packages/e2e/tests/integration/extended-fixtures.test.ts
@@ -1,21 +1,12 @@
import { readFileSync } from 'node:fs';
import path from 'node:path';
-import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
+import { describe, expect, it, vi } from 'vitest';
+import { createFakeEngine } from '../helpers/fake-engine.ts';
+import { engineConfig } from '../helpers/fixture-config.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
import { resultByTitle, runProject } from '../helpers/run-project.ts';
describe('test.extend fixtures', () => {
- let app: FixtureApp;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- });
-
- afterAll(async () => {
- await app?.close();
- });
-
it(
'sets fixtures up in order, hands them to hooks and the body, and tears them down last first',
async () => {
@@ -77,18 +68,21 @@ silent('never calls use', async ({ silent }) => {
log('unreachable:' + silent);
});
-const clash = base.extend<{ browser: string }>({
- browser: async (_fixtures, use) => {
- await use('not the browser');
+const clash = base.extend<{ gadget: string }>({
+ gadget: async (_fixtures, use) => {
+ await use('not the gadget');
},
});
-clash('redefines an engine fixture', async ({ browser }) => {
- log('unreachable:' + String(browser));
+clash('redefines an engine fixture', async ({ gadget }) => {
+ log('unreachable:' + String(gadget));
});
`;
const logPath = path.join('/tmp', `e2e-extend-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/extend.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject(
+ { 'tests/extend.e2e.ts': file },
+ { config: engineConfig(createFakeEngine({ fixtures: true }).engine) },
+ );
assertValidReport(outcome.report);
const passes = resultByTitle(outcome, 'sees the values');
@@ -115,17 +109,17 @@ clash('redefines an engine fixture', async ({ browser }) => {
expect(clash.status).toBe('failed');
expect(clash.attempts[0]!.error?.phase).toBe('beforeEach');
expect(clash.attempts[0]!.error?.code).toBe('TEST_SETUP_FAILED');
- expect(clash.attempts[0]!.error?.message).toContain('fixture "browser" is contributed by engine web');
+ expect(clash.attempts[0]!.error?.message).toContain('fixture "gadget" is contributed by engine fake');
expect(readFileSync(logPath, 'utf8').trim().split('\n')).toEqual([
- 'setup:first:web',
+ 'setup:first:fake',
'setup:second:one',
'beforeEach:one-two',
'body:one:one-two',
'afterEach:one',
'teardown:second',
'teardown:first',
- 'setup:first:web',
+ 'setup:first:fake',
'setup:second:one',
'beforeEach:one-two',
'body:failing:one-two',
@@ -151,19 +145,19 @@ const log = (entry: string) => appendFileSync(process.env.HOOK_LOG!, entry + '\\
const test = base.extend<{ slow: string }>({
slow: async (_fixtures, use) => {
log('setup:slow');
- await new Promise((resolve) => setTimeout(resolve, 2500));
+ await new Promise((resolve) => setTimeout(resolve, 1000));
await use('slow');
log('teardown:slow');
},
});
-test('times out in setup', { timeout: 1000 }, async ({ slow }) => {
+test('times out in setup', { timeout: 500 }, async ({ slow }) => {
log('unreachable:' + slow);
});
`;
const logPath = path.join('/tmp', `e2e-extend-late-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/late.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/late.e2e.ts': file }, {});
const result = resultByTitle(outcome, 'times out in setup');
expect(result.status).toBe('timed-out');
expect(result.attempts[0]!.error?.phase).toBe('beforeEach');
diff --git a/packages/e2e/tests/integration/interrupt-signals.test.ts b/packages/e2e/tests/integration/interrupt-signals.test.ts
index f9612b3da..9e103be8c 100644
--- a/packages/e2e/tests/integration/interrupt-signals.test.ts
+++ b/packages/e2e/tests/integration/interrupt-signals.test.ts
@@ -2,7 +2,8 @@
* Ctrl-C against the built CLI, the way a terminal delivers it: SIGINT to the
* whole process group, runner and workers alike. The engine is a file-logging
* fake declared in the fixture config, so the test can see, from outside,
- * whether the engine was disposed and whether the worker outlived the runner.
+ * whether the engine was disposed, whether the runner released what its
+ * `prepare` acquired, and whether the worker outlived the runner.
*/
import { spawn, type ChildProcess } from 'node:child_process';
@@ -41,6 +42,8 @@ export default {
name: 'signal-fake',
version: '1.0.0',
spiVersion: 1,
+ async prepare() { log('prepare'); },
+ async finish() { log('finish'); },
async init() { log('init'); },
async startAttempt() { log('startAttempt'); },
async endAttempt() { log('endAttempt'); },
@@ -270,7 +273,7 @@ describe('interrupt signals against the CLI', () => {
expect(code).toBe(130);
// Well inside the 60 s test timeout: the interrupt, not the timeout, ended the body.
expect(elapsed).toBeLessThan(20_000);
- expect(run.events()).toEqual(['init', 'startAttempt', 'test.start', 'endAttempt', 'dispose']);
+ expect(run.events()).toEqual(['prepare', 'init', 'startAttempt', 'test.start', 'endAttempt', 'dispose', 'finish']);
expect(run.output()).toContain('interrupted: stopping the running test');
expect(alive(run.workerPid)).toBe(false);
}),
@@ -322,6 +325,7 @@ describe('interrupt signals against the CLI', () => {
const pid = run.workerPid;
await waitFor(() => !alive(pid), CLEANUP_TIMEOUT_MS + 5_000, 'the worker to exit');
expect(run.events()).toContain('dispose');
+ expect(run.events().filter((event) => event === 'finish')).toEqual(['finish']);
}),
60_000,
);
diff --git a/packages/e2e/tests/integration/list-reporter.test.ts b/packages/e2e/tests/integration/list-reporter.test.ts
index 59b0a1db1..bc4e95aff 100644
--- a/packages/e2e/tests/integration/list-reporter.test.ts
+++ b/packages/e2e/tests/integration/list-reporter.test.ts
@@ -4,56 +4,11 @@ import { readFileSync } from 'node:fs';
import path from 'node:path';
import { stripVTControlCharacters } from 'node:util';
import { describe, expect, it, vi } from 'vitest';
-import type { StepExecutor } from '../../src/agent/executor.ts';
import { defineEngine } from '../../src/engine/index.ts';
-import { visibleWidth } from '../../src/report/format.ts';
import { runProject, type FixtureProject } from '../helpers/run-project.ts';
import { snapshot } from '../helpers/snapshot.ts';
-/** Forty CJK glyphs: eighty columns, over the step label's budget of seventy-two. */
-const WIDE_LABEL = '日'.repeat(40);
-
-/** An executor that passes every step without a model, so the run needs neither a browser nor a key. */
-const passing: StepExecutor = { name: 'passing', async runStep() { return { status: 'passed', summary: 'done' }; } };
-
describe('list reporter output', () => {
- it('clips a wide-glyph agent step label to its 72-column budget on the permanent step line', async () => {
- const written: string[] = [];
- const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
- written.push(String(chunk));
- return true;
- });
- let project: FixtureProject | undefined;
- try {
- const run = await runProject(
- { 'tests/wide.e2e.ts': `import { test } from 'e2e';
- test('acts on a wide label', async ({ agent }) => { await agent.act(${JSON.stringify(WIDE_LABEL)}); });` },
- {
- appUrl: 'http://127.0.0.1:4599',
- config: {
- targets: [{ name: 'fake', platform: 'custom', engine: defineEngine({ name: 'fake', version: '1', spiVersion: 1, observe: async () => snapshot([]) }) }],
- agents: { default: { executor: passing } },
- cache: 'off',
- },
- runOptions: { quiet: false },
- },
- );
- project = run.project;
- stdoutWrite.mockRestore();
- expect(run.outcome.status).toBe('passed');
- const stepLines = stripVTControlCharacters(written.join('')).split('\n').filter((line) => line.includes('agent.act "'));
- expect(stepLines.length).toBeGreaterThan(0);
- for (const line of stepLines) {
- const label = /"([^"]*)"/.exec(line)?.[1];
- expect(label).toBe(`${'日'.repeat(35)}…`);
- expect(visibleWidth(label!)).toBe(71);
- }
- } finally {
- stdoutWrite.mockRestore();
- project?.cleanup();
- }
- });
-
it('tallies a --repeat-each run: the tests that passed every run, and the flaky runs of the one that did not with the code each retry recovered from', async () => {
const written: string[] = [];
const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
diff --git a/packages/e2e/tests/integration/managed-process.test.ts b/packages/e2e/tests/integration/managed-process.test.ts
index ba2dc3ef9..c78c0eb2c 100644
--- a/packages/e2e/tests/integration/managed-process.test.ts
+++ b/packages/e2e/tests/integration/managed-process.test.ts
@@ -41,20 +41,12 @@ async function isReachable(url: string): Promise {
}
describe('ManagedProcess', () => {
- it('starts the app, waits for readiness, and stops the process group', async () => {
- const port = await freePort();
- const app = nodeApp(port);
- await app.start();
- expect(await isReachable(`http://127.0.0.1:${port}/`)).toBe(true);
- await app.stop();
- expect(await isReachable(`http://127.0.0.1:${port}/`)).toBe(false);
- });
-
- it('stop is idempotent and safe before start', async () => {
+ it('starts the app, waits for readiness, and stops the process group, idempotently and safely before start', async () => {
const port = await freePort();
const app = nodeApp(port);
await app.stop();
await app.start();
+ expect(await isReachable(`http://127.0.0.1:${port}/`)).toBe(true);
await app.stop();
await app.stop();
expect(await isReachable(`http://127.0.0.1:${port}/`)).toBe(false);
@@ -106,7 +98,7 @@ describe('ManagedProcess', () => {
{
executable: process.execPath,
args: ['-e', 'setInterval(() => {}, 1000)'],
- startupTimeout: 1_500,
+ startupTimeout: 500,
shutdownTimeout: 2_000,
},
os.tmpdir(),
@@ -127,37 +119,4 @@ describe('ManagedProcess', () => {
expect(await isReachable(`http://127.0.0.1:${port}/`)).toBe(false);
});
- it('without reuseExisting, a server another process already runs fails the launch before spawning', async () => {
- const port = await freePort();
- const devServer = nodeApp(port);
- await devServer.start();
- try {
- const failure = await nodeApp(port)
- .start()
- .catch((error: unknown) => error);
- expect(failure).toBeInstanceOf(InfrastructureError);
- expect((failure as InfrastructureError).code).toBe('APP_ALREADY_RUNNING');
- expect((failure as InfrastructureError).message).toContain(`http://127.0.0.1:${port}/ already answered before app.command started`);
- expect(await isReachable(`http://127.0.0.1:${port}/`)).toBe(true);
- } finally {
- await devServer.stop();
- }
- });
-
- it('with reuseExisting, attaches to a server another process already runs and never stops it', async () => {
- const port = await freePort();
- const devServer = nodeApp(port);
- await devServer.start();
- try {
- // Without reuse this spawn would lose the port; with it, the command is never started.
- const app = nodeApp(port, { reuseExisting: true });
- await app.start();
- expect(app.spawned).toBe(false);
- await app.stop();
- expect(await isReachable(`http://127.0.0.1:${port}/`)).toBe(true);
- } finally {
- await devServer.stop();
- }
- expect(await isReachable(`http://127.0.0.1:${port}/`)).toBe(false);
- });
});
diff --git a/packages/e2e/tests/integration/matcher-options.test.ts b/packages/e2e/tests/integration/matcher-options.test.ts
deleted file mode 100644
index 47f7e8a31..000000000
--- a/packages/e2e/tests/integration/matcher-options.test.ts
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Playwright's matcher options through the real CLI and browser. The CLI
- * transpiles without type-checking, so an option a matcher does not take
- * reaches the runtime: the ones e2e implements must change the verdict the
- * way Playwright's do, and every other one must fail the call instead of
- * leaving the matcher to pass on its default.
- */
-
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
-import { resultByTitle, runProject, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
-
-const SUITE = `import { test } from '@e2e-dev/web';
-import { expect } from 'e2e';
-
-const soon = { timeout: 300 };
-// Built at runtime, the way a JavaScript test or a shared helper passes options.
-const flag = (name, value) => JSON.parse(JSON.stringify({ [name]: value, ...soon }));
-
-test('state flags expect the opposite state', async ({ app, screen }) => {
- await app.open();
- await expect(screen.getByLabel('Notifications')).toBeChecked({ checked: false });
- await expect(screen.getByLabel('Notifications')).toBeChecked(flag('checked', false));
- await expect(screen.getByText('Hidden content')).toBeVisible({ visible: false });
- await expect(screen.getByText('Never rendered')).toBeAttached({ attached: false });
- await expect(screen.getByRole('button', { name: 'Disabled action' })).toBeEnabled({ enabled: false });
- await screen.getByLabel('Notifications').check();
- await expect(screen.getByLabel('Notifications')).toBeChecked({ checked: true });
-});
-
-test('ignoreCase matches text in any case', async ({ app, screen }) => {
- await app.open();
- await expect(screen.getByRole('heading')).toHaveText('HOME', { ignoreCase: true });
- await expect(screen.getByRole('heading')).toContainText('OM', flag('ignoreCase', true));
- await expect(screen.getByRole('heading')).not.toContainText('home', soon);
-});
-
-test('checked false fails on a checked box', async ({ app, screen }) => {
- await app.open();
- await screen.getByLabel('Notifications').check();
- await expect(screen.getByLabel('Notifications')).toBeChecked(flag('checked', false));
-});
-
-test('visible false fails on a visible heading', async ({ app, screen }) => {
- await app.open();
- await expect(screen.getByRole('heading')).toBeVisible(flag('visible', false));
-});
-
-test('enabled false fails on an enabled button', async ({ app, screen }) => {
- await app.open();
- await expect(screen.getByRole('button', { name: 'Increment' })).toBeEnabled(flag('enabled', false));
-});
-
-test('attached false fails on a hidden node', async ({ app, screen }) => {
- await app.open();
- await expect(screen.getByText('Hidden content')).toBeAttached(flag('attached', false));
-});
-
-test('a negated ignoreCase match fails on the same text in another case', async ({ app, screen }) => {
- await app.open();
- await expect(screen.getByRole('heading')).not.toContainText('home', flag('ignoreCase', true));
-});
-
-test('an option toBeChecked does not take is refused', async ({ app, screen }) => {
- await app.open();
- await expect(screen.getByLabel('Notifications')).toBeChecked(flag('indeterminate', true));
-});
-
-test('an option toHaveTitle does not take is refused', async ({ app, browser }) => {
- await app.open();
- await expect(browser).toHaveTitle('', flag('ignoreCase', true));
-});
-
-test('an option waitForURL does not take is refused', async ({ app, browser }) => {
- await app.open();
- await browser.waitForURL('/', flag('waitUntil', 'load'));
-});
-`;
-
-describe('matcher options in a browser', () => {
- let app: FixtureApp;
- let outcome: RunOutcome;
- let project: FixtureProject;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- ({ outcome, project } = await runProject(
- { 'tests/options.e2e.ts': SUITE },
- { appUrl: app.url, config: { actionTimeout: 5_000, assertionTimeout: 4_000, timeout: 30_000 } },
- ));
- }, 240_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it.each(['state flags expect the opposite state', 'ignoreCase matches text in any case'])('%s', (title) => {
- const result = resultByTitle(outcome, title);
- expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe('passed');
- });
-
- it.each([
- ['checked false fails on a checked box', 'expected: unchecked'],
- ['visible false fails on a visible heading', 'expected: hidden or absent'],
- ['enabled false fails on an enabled button', 'expected: disabled'],
- ['attached false fails on a hidden node', 'expected: detached'],
- ['a negated ignoreCase match fails on the same text in another case', 'expected: not text containing "home" (ignoring case)'],
- ])('%s', (title, expected) => {
- const result = resultByTitle(outcome, title);
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error).toMatchObject({ code: 'ASSERTION_FAILED', message: expect.stringContaining(expected) });
- });
-
- it.each([
- ['an option toBeChecked does not take is refused', 'expect.toBeChecked options has no key "indeterminate"'],
- ['an option toHaveTitle does not take is refused', 'expect.toHaveTitle options has no key "ignoreCase"'],
- ['an option waitForURL does not take is refused', 'browser.waitForURL options has no key "waitUntil"'],
- ])('%s', (title, message) => {
- const result = resultByTitle(outcome, title);
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error).toMatchObject({ code: 'INVALID_ARGUMENT', message: expect.stringContaining(message) });
- });
-});
diff --git a/packages/e2e/tests/integration/mcp-custom.test.ts b/packages/e2e/tests/integration/mcp-custom.test.ts
deleted file mode 100644
index 04eacaf40..000000000
--- a/packages/e2e/tests/integration/mcp-custom.test.ts
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * `e2e mcp` with a project that has two targets, a custom engine with fewer
- * verbs, and project tools from `agents.default.tools`: the server's tool
- * list stays the same four, each session's catalog is what its own target
- * can do (the engine's verbs, the project tools for its platform), and a
- * second config in the same project opens without restarting the server. A
- * project tool named like a built-in never gets here: config load refuses it.
- */
-
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { Client } from '@modelcontextprotocol/client';
-import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
-import { createProject, type FixtureProject } from '../helpers/run-project.ts';
-
-const PACKAGE_ROOT = fileURLToPath(new URL('../../', import.meta.url));
-const CLI = path.join(PACKAGE_ROOT, 'dist', 'cli', 'bin.js');
-
-const KIOSK = `import { createFakeEngine } from '../../helpers/fake-engine.ts';
-
-export const kiosk = createFakeEngine({
- tree: { ref: { id: 'root', revision: '' }, role: 'root', children: [
- { ref: { id: 'k-1', revision: '' }, role: 'button', name: 'Start order', states: { hidden: false } },
- { ref: { id: 'k-2', revision: '' }, role: 'textbox', name: 'Table', states: { hidden: false } },
- ] },
-});
-`;
-
-const CONFIG = `import type { E2EConfig } from 'e2e';
-import { defineTool, getToolContext } from 'e2e/agent';
-import { web } from '@e2e-dev/web';
-import { z } from 'zod';
-import { kiosk } from './kiosk.ts';
-
-export default {
- targets: [
- { name: 'web', platform: 'web', engine: web(), app: { url: process.env.APP_URL! } },
- { name: 'kiosk', platform: 'kiosk', engine: kiosk.engine },
- ],
- agents: { default: {
- tools: {
- seed_data: defineTool(
- { description: 'Seed a tenant with demo data.', inputSchema: z.object({ tenant: z.string() }), execute: async ({ tenant }: { tenant: string }) => \`Seeded \${tenant}.\` },
- { mutates: true },
- ),
- count_nodes: defineTool(
- { description: 'Count the nodes on screen.', inputSchema: z.object({}), execute: async (_input: unknown, options: object) => \`\${(await getToolContext(options).observe()).text.split('\\\\n').length} nodes\` },
- { mutates: false },
- ),
- shake: defineTool(
- { description: 'Shake the phone.', inputSchema: z.object({}), execute: async () => 'shaken' },
- { mutates: true, platforms: ['ios'] },
- ),
- kiosk_reset: defineTool(
- { description: 'Reset the kiosk.', inputSchema: z.object({}), execute: async () => 'kiosk reset' },
- { mutates: true, platforms: ['kiosk'] },
- ),
- },
- } },
-} satisfies E2EConfig;
-`;
-
-/** A second project config beside the first: one target, no project tools. */
-const KIOSK_ONLY_CONFIG = `import type { E2EConfig } from 'e2e';
-import { kiosk } from './kiosk.ts';
-
-export default {
- targets: [{ name: 'kiosk-only', platform: 'kiosk', engine: kiosk.engine }],
-} satisfies E2EConfig;
-`;
-
-interface ToolText {
- readonly text: string;
- readonly isError: boolean;
-}
-
-describe('e2e mcp with project tools and a custom engine', { timeout: 120_000 }, () => {
- let app: FixtureApp;
- let project: FixtureProject;
- let client: Client;
-
- const invoke = async (name: string, args: Record = {}): Promise => {
- const result = (await client.callTool({ name, arguments: args }, { timeout: 110_000 })) as {
- content: { type: string; text?: string }[];
- isError?: boolean;
- };
- return {
- text: result.content.filter((part) => part.type === 'text').map((part) => part.text ?? '').join('\n'),
- isError: result.isError === true,
- };
- };
- const call = (tool: string, args?: Record): Promise => invoke('call', { tool, ...(args === undefined ? {} : { args }) });
- const catalogNames = (text: string): string[] =>
- text
- .split('\n')
- .filter((line) => line.startsWith('- '))
- .map((line) => /^- (\S+?)(?: \{|:)/.exec(line)![1]!);
-
- beforeAll(async () => {
- app = await startFixtureApp();
- project = createProject({ 'e2e.config.ts': CONFIG, 'kiosk.ts': KIOSK, 'kiosk.config.ts': KIOSK_ONLY_CONFIG });
- const transport = new StdioClientTransport({
- command: process.execPath,
- args: [CLI, 'mcp', '--headless'],
- cwd: project.dir,
- env: { ...(process.env as Record), APP_URL: app.url, CI: '' },
- stderr: 'ignore',
- });
- client = new Client({ name: 'e2e-mcp-custom-test', version: '0.0.0' });
- await client.connect(transport);
- }, 60_000);
-
- afterAll(async () => {
- await client?.close().catch(() => undefined);
- project?.cleanup();
- await app?.close();
- });
-
- it('serves the same four tools whatever the project declares', async () => {
- const { tools } = await client.listTools();
- expect(tools.map((tool) => tool.name)).toEqual(['open_session', 'tools', 'call', 'close_session']);
- });
-
- it('needs a target name when the config declares several', async () => {
- const result = await invoke('open_session');
- expect(result.isError).toBe(true);
- expect(result.text).toContain('TARGET_REQUIRED');
- expect(result.text).toContain('web, kiosk');
- });
-
- it('catalogs the project tools of a web session, runs them, and skips the foreign ones', async () => {
- const opened = await invoke('open_session', { target: 'web' });
- expect(opened.isError, opened.text).toBe(false);
- expect(catalogNames(opened.text)).toEqual([
- 'observe',
- 'tap',
- 'double_tap',
- 'long_press',
- 'right_click',
- 'hover',
- 'scroll_to',
- 'type',
- 'press',
- 'select',
- 'check',
- 'drag',
- 'upload',
- 'scroll',
- 'navigate',
- 'back',
- 'screenshot',
- 'tap_at',
- 'hover_at',
- 'type_at',
- 'press_at',
- 'select_at',
- 'locate',
- 'start_recording',
- 'stop_recording',
- 'seed_data',
- 'count_nodes',
- ]);
- expect(opened.text).toContain('- seed_data {tenant}: Seed a tenant with demo data.');
- expect(opened.text).toContain('- count_nodes: Count the nodes on screen. [read-only]');
- const detail = await invoke('tools', { tool: 'locate' });
- expect(detail.text).toContain('locator');
-
- const seeded = await call('seed_data', { tenant: 'acme' });
- expect(seeded.isError, seeded.text).toBe(false);
- expect(seeded.text).toBe('Seeded acme.');
- const counted = await call('count_nodes');
- expect(counted.isError, counted.text).toBe(false);
- expect(counted.text).toMatch(/^\d+ nodes$/);
-
- const foreign = await call('kiosk_reset');
- expect(foreign.isError).toBe(true);
- expect(foreign.text).toContain('UNKNOWN_TOOL');
-
- const closed = await invoke('close_session');
- expect(closed.isError, closed.text).toBe(false);
- });
-
- it('drives a custom engine with the verbs it declares and nothing more', async () => {
- const opened = await invoke('open_session', { target: 'kiosk' });
- expect(opened.isError, opened.text).toBe(false);
- expect(opened.text).toContain('platform kiosk, engine fake');
- expect(opened.text).toContain('button "Start order"');
- const names = catalogNames(opened.text);
- expect(names).toContain('kiosk_reset');
- expect(names).not.toContain('scroll');
-
- const observed = await call('observe');
- const id = /#(\S+) button "Start order"/.exec(observed.text)?.[1];
- expect(id).toBeDefined();
- const tapped = await call('tap', { target: id! });
- expect(tapped.isError, tapped.text).toBe(false);
- expect(tapped.text).toContain(`Tapped #${id}.`);
- expect(tapped.text).toMatch(/Screen changes since revision|The screen did not change|No listed node changed/);
-
- // The fake declares no viewport swipe, so scroll is a verb this target lacks.
- const scrolled = await call('scroll', { direction: 'down' });
- expect(scrolled.isError).toBe(true);
- expect(scrolled.text).toContain('UNSUPPORTED_CAPABILITY');
- expect(scrolled.text).toContain('declares no such action');
-
- const reset = await call('kiosk_reset');
- expect(reset.isError, reset.text).toBe(false);
- expect(reset.text).toBe('kiosk reset');
-
- const closed = await invoke('close_session');
- expect(closed.isError, closed.text).toBe(false);
- expect(closed.text).not.toContain('Cleanup:');
- });
-
- it('opens another config of the same project without a restart', async () => {
- const opened = await invoke('open_session', { config: 'kiosk.config.ts' });
- expect(opened.isError, opened.text).toBe(false);
- expect(opened.text).toContain('open on target "kiosk-only"');
- expect(opened.text).toContain('config ');
- expect(opened.text).toContain('kiosk.config.ts');
- // The fake declares every node action but no swipe, no back hook, and no pointer kinds: no scroll, no back, and the point tools ride the node verbs.
- expect(catalogNames(opened.text)).toEqual([
- 'observe',
- 'tap',
- 'double_tap',
- 'long_press',
- 'right_click',
- 'hover',
- 'scroll_to',
- 'type',
- 'press',
- 'select',
- 'check',
- 'drag',
- 'upload',
- 'navigate',
- 'screenshot',
- 'tap_at',
- 'hover_at',
- 'type_at',
- 'press_at',
- 'select_at',
- 'locate',
- ]);
- const closed = await invoke('close_session');
- expect(closed.isError, closed.text).toBe(false);
- });
-});
diff --git a/packages/e2e/tests/integration/mcp-server.test.ts b/packages/e2e/tests/integration/mcp-server.test.ts
index 5f3ddf6e4..c1d09f291 100644
--- a/packages/e2e/tests/integration/mcp-server.test.ts
+++ b/packages/e2e/tests/integration/mcp-server.test.ts
@@ -3,13 +3,13 @@
* a real MCP client. The server's tool list is four tools; everything the
* session can do is a catalog behind `call`. Covers the catalog, a live
* session driven through `call`, argument validation, the secret and pixel
- * invariants, a video recording, a second session open beside the first, a
- * second session after the first closed, an explicit config path, and stdout
- * hygiene: a config that prints to stdout must not corrupt
- * the protocol.
+ * invariants, a second session open beside the first, a second session after
+ * the first closed, an explicit config path, a config with two targets, a
+ * custom engine, and project tools (each session's catalog is what its own
+ * target can do), and stdout hygiene: a config that prints to stdout must not
+ * corrupt the protocol.
*/
-import { readdirSync, statSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { Client } from '@modelcontextprotocol/client';
@@ -55,6 +55,49 @@ export default {
} satisfies E2EConfig;
`;
+const KIOSK = `import { createFakeEngine } from '../../helpers/fake-engine.ts';
+
+export const kiosk = createFakeEngine({
+ tree: { ref: { id: 'root', revision: '' }, role: 'root', children: [
+ { ref: { id: 'k-1', revision: '' }, role: 'button', name: 'Start order', states: { hidden: false } },
+ { ref: { id: 'k-2', revision: '' }, role: 'textbox', name: 'Table', states: { hidden: false } },
+ ] },
+});
+`;
+
+// A third config: two targets, one a custom engine with fewer verbs, and
+// project tools scoped by platform.
+const CUSTOM_CONFIG = `import type { E2EConfig } from 'e2e';
+import { defineTool, getToolContext } from 'e2e/agent';
+import { z } from 'zod';
+import { targets } from './targets.ts';
+import { kiosk } from './kiosk.ts';
+
+export default {
+ targets: [...targets, { name: 'kiosk', platform: 'kiosk', engine: kiosk.engine }],
+ agents: { default: {
+ tools: {
+ seed_data: defineTool(
+ { description: 'Seed a tenant with demo data.', inputSchema: z.object({ tenant: z.string() }), execute: async ({ tenant }: { tenant: string }) => \`Seeded \${tenant}.\` },
+ { mutates: true },
+ ),
+ count_nodes: defineTool(
+ { description: 'Count the nodes on screen.', inputSchema: z.object({}), execute: async (_input: unknown, options: object) => \`\${(await getToolContext(options).observe()).text.split('\\\\n').length} nodes\` },
+ { mutates: false },
+ ),
+ shake: defineTool(
+ { description: 'Shake the phone.', inputSchema: z.object({}), execute: async () => 'shaken' },
+ { mutates: true, platforms: ['ios'] },
+ ),
+ kiosk_reset: defineTool(
+ { description: 'Reset the kiosk.', inputSchema: z.object({}), execute: async () => 'kiosk reset' },
+ { mutates: true, platforms: ['kiosk'] },
+ ),
+ },
+ } },
+} satisfies E2EConfig;
+`;
+
interface ToolText {
readonly text: string;
readonly isError: boolean;
@@ -92,10 +135,17 @@ describe('e2e mcp', { timeout: 120_000 }, () => {
};
const catalogLines = (text: string): string[] => text.split('\n').filter((line) => line.startsWith('- '));
+ const catalogNames = (text: string): string[] => catalogLines(text).map((line) => /^- (\S+?)(?: \{|:)/.exec(line)![1]!);
beforeAll(async () => {
app = await startFixtureApp();
- project = createProject({ 'e2e.config.ts': CONFIG, 'targets.ts': TARGETS, 'protected.config.ts': PROTECTED_CONFIG });
+ project = createProject({
+ 'e2e.config.ts': CONFIG,
+ 'targets.ts': TARGETS,
+ 'protected.config.ts': PROTECTED_CONFIG,
+ 'kiosk.ts': KIOSK,
+ 'custom.config.ts': CUSTOM_CONFIG,
+ });
transport = new StdioClientTransport({
command: process.execPath,
args: [CLI, 'mcp', '--headless'],
@@ -169,34 +219,36 @@ describe('e2e mcp', { timeout: 120_000 }, () => {
expect(opened.text).toContain(`App: ${app.url}/`);
expect(opened.text).toContain('Credentials: "admin" (username "admin", password secret "admin.password")');
expect(opened.text).toContain('Tools (run one with call {tool, args}; tools {tool} shows a tool\'s arguments):');
- expect(catalogLines(opened.text)).toEqual([
- expect.stringMatching(/^- observe: Look at the whole current screen.* \[read-only\]$/),
- expect.stringMatching(/^- tap \{target\}: Tap or click one node: the gesture for a button, link, menu item, tab, checkbox, row, or field\.$/),
- expect.stringMatching(/^- double_tap \{target\}: Double-click one node: only for an item that opens or enters editing on the second click/),
- expect.stringMatching(/^- long_press \{target\}: Press one node and hold: only for a control with a long-press menu or action\.$/),
- expect.stringMatching(/^- right_click \{target\}: Right-click one node to open its context menu, and only for that\.$/),
- expect.stringMatching(/^- hover \{target\}: Move the pointer over one node without clicking/),
- expect.stringMatching(/^- scroll_to \{target\?, text\?, direction\?\}: Scroll until a node is inside the viewport/),
- expect.stringMatching(/^- type \{target\?, value, replace\?\}: Type a plain-text value into one input node, or into whatever has focus when target is omitted\.$/),
- expect.stringMatching(/^- press \{target\?, key, times\?\}: /),
- expect.stringMatching(/^- select \{target, value\}: /),
- expect.stringMatching(/^- check \{target, checked\?\}: Set a checkbox, switch, or radio to a state/),
- expect.stringMatching(/^- drag \{target, to\}: Drag one node and drop it on another/),
- expect.stringMatching(/^- upload \{target, files\}: Attach one or more files to a file input node\.$/),
- expect.stringMatching(/^- scroll \{direction, target\?, times\?\}: /),
- expect.stringMatching(/^- navigate \{url\}: /),
- expect.stringMatching(/^- back: Go back one step/),
- expect.stringMatching(/^- screenshot: Attach a screenshot of the current viewport.* \[read-only\]$/),
- expect.stringMatching(/^- tap_at \{x, y\}: Tap a point in the latest screenshot/),
- expect.stringMatching(/^- hover_at \{x, y\}: Move the pointer to a point in the latest screenshot/),
- expect.stringMatching(/^- type_at \{x, y, value, replace\?\}: Type a plain-text value into the field at a point/),
- expect.stringMatching(/^- press_at \{x, y, key\}: Send one key/),
- expect.stringMatching(/^- select_at \{x, y, value\}: Pick one option/),
- expect.stringMatching(/^- type_secret \{target, name\}: /),
- expect.stringMatching(/^- locate \{role\?, name\?, text\?, label\?, placeholder\?, testId\?, exact\?\}: .* \[read-only\]$/),
- expect.stringMatching(/^- start_recording \{name\?\}: Start recording a video of the app, for a person to watch: .*\.$/),
- expect.stringMatching(/^- stop_recording: Stop the running recording and save it: .*\.$/),
+ expect(catalogNames(opened.text)).toEqual([
+ 'observe',
+ 'tap',
+ 'double_tap',
+ 'long_press',
+ 'right_click',
+ 'hover',
+ 'scroll_to',
+ 'type',
+ 'press',
+ 'select',
+ 'check',
+ 'drag',
+ 'upload',
+ 'scroll',
+ 'navigate',
+ 'back',
+ 'screenshot',
+ 'tap_at',
+ 'hover_at',
+ 'type_at',
+ 'press_at',
+ 'select_at',
+ 'type_secret',
+ 'locate',
+ 'start_recording',
+ 'stop_recording',
]);
+ expect(opened.text).toMatch(/^- tap \{target\}: Tap or click one node: the gesture for a button, link, menu item, tab, checkbox, row, or field\.$/m);
+ expect(opened.text).toMatch(/^- locate \{role\?, name\?, text\?, label\?, placeholder\?, testId\?, exact\?\}: .* \[read-only\]$/m);
expect(opened.text).toMatch(/Current screen \(revision b\d+, path \/, \d+ nodes\):/);
expect(opened.text).toContain('button "Increment"');
const sessionId = /^Session (\S+) open/.exec(opened.text)![1]!;
@@ -319,32 +371,6 @@ describe('e2e mcp', { timeout: 120_000 }, () => {
expect(gone.text).toContain('the previous session ended: closed by the agent');
});
- it('records the app between start_recording and stop_recording, and saves a recording still running at close', async () => {
- const opened = await invoke('open_session');
- expect(opened.isError, opened.text).toBe(false);
- const sessionId = /^Session (\S+) open/.exec(opened.text)![1]!;
- const recordings = path.join(project.dir, '.e2e', 'videos', sessionId);
-
- const started = await call('start_recording', { name: 'counter' });
- expect(started.isError, started.text).toBe(false);
- const observed = await call('observe');
- await call('tap', { target: nodeId(observed.text, /button "Increment"/) });
- const stopped = await call('stop_recording');
- expect(stopped.isError, stopped.text).toBe(false);
- expect(stopped.text).toMatch(/^Recording 1 "counter" stopped after \d+\.\d s\.\n- \S+\.webm$/);
- const file = path.join(recordings, '1-counter.webm');
- expect(stopped.text.endsWith(`- ${file}`)).toBe(true);
- expect(statSync(file).size).toBeGreaterThan(0);
-
- await call('start_recording');
- const closed = await invoke('close_session');
- expect(closed.isError, closed.text).toBe(false);
- expect(closed.text).toMatch(/\nRecording 2 stopped after \d+\.\d s\.\n- \S+$/);
- expect(closed.text.endsWith(`- ${path.join(recordings, '2.webm')}`)).toBe(true);
- expect(closed.text).not.toContain('Cleanup:');
- expect(readdirSync(recordings).toSorted()).toEqual(['1-counter.webm', '2.webm']);
- });
-
it('opens a second session after the first closed, on an explicit config path, and names an unknown target', async () => {
const unknown = await invoke('open_session', { target: 'nope' });
expect(unknown.isError).toBe(true);
@@ -404,6 +430,57 @@ describe('e2e mcp', { timeout: 120_000 }, () => {
expect(closed.isError, closed.text).toBe(false);
});
+ it('needs a target name when the config declares several', async () => {
+ const result = await invoke('open_session', { config: 'custom.config.ts' });
+ expect(result.isError).toBe(true);
+ expect(result.text).toContain('TARGET_REQUIRED');
+ expect(result.text).toContain('web, kiosk');
+ });
+
+ it('catalogs and runs the project tools of a custom engine target, with only the verbs it declares', async () => {
+ const opened = await invoke('open_session', { config: 'custom.config.ts', target: 'kiosk' });
+ expect(opened.isError, opened.text).toBe(false);
+ expect(opened.text).toContain('platform kiosk, engine fake');
+ expect(opened.text).toContain('button "Start order"');
+ expect(opened.text).toContain('- seed_data {tenant}: Seed a tenant with demo data.');
+ expect(opened.text).toContain('- count_nodes: Count the nodes on screen. [read-only]');
+ const names = catalogNames(opened.text);
+ expect(names).toEqual(expect.arrayContaining(['seed_data', 'count_nodes', 'kiosk_reset']));
+ expect(names).not.toContain('shake');
+ expect(names).not.toContain('scroll');
+
+ const seeded = await call('seed_data', { tenant: 'acme' });
+ expect(seeded.isError, seeded.text).toBe(false);
+ expect(seeded.text).toBe('Seeded acme.');
+ const counted = await call('count_nodes');
+ expect(counted.isError, counted.text).toBe(false);
+ expect(counted.text).toMatch(/^\d+ nodes$/);
+ const reset = await call('kiosk_reset');
+ expect(reset.isError, reset.text).toBe(false);
+ expect(reset.text).toBe('kiosk reset');
+ const foreign = await call('shake');
+ expect(foreign.isError).toBe(true);
+ expect(foreign.text).toContain('UNKNOWN_TOOL');
+
+ const observed = await call('observe');
+ const id = /#(\S+) button "Start order"/.exec(observed.text)?.[1];
+ expect(id).toBeDefined();
+ const tapped = await call('tap', { target: id! });
+ expect(tapped.isError, tapped.text).toBe(false);
+ expect(tapped.text).toContain(`Tapped #${id}.`);
+ expect(tapped.text).toMatch(/Screen changes since revision|The screen did not change|No listed node changed/);
+
+ // The fake declares no viewport swipe, so scroll is a verb this target lacks.
+ const scrolled = await call('scroll', { direction: 'down' });
+ expect(scrolled.isError).toBe(true);
+ expect(scrolled.text).toContain('UNSUPPORTED_CAPABILITY');
+ expect(scrolled.text).toContain('declares no such action');
+
+ const closed = await invoke('close_session');
+ expect(closed.isError, closed.text).toBe(false);
+ expect(closed.text).not.toContain('Cleanup:');
+ });
+
it('kept stdout for the protocol: the config\'s console.log landed on stderr', () => {
expect(stderr).toContain('config loaded');
expect(stderr).toContain('e2e mcp: [info] e2e mcp');
diff --git a/packages/e2e/tests/integration/mcp-session-host.test.ts b/packages/e2e/tests/integration/mcp-session-host.test.ts
index 2d386e2d0..3b721c7cb 100644
--- a/packages/e2e/tests/integration/mcp-session-host.test.ts
+++ b/packages/e2e/tests/integration/mcp-session-host.test.ts
@@ -14,6 +14,7 @@ import { existsSync, mkdtempSync, readdirSync, rmSync } from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { z } from 'zod';
import { createFakeEngine, FAKE_APP, type FakeEngineBehavior, type FakeEngineHandle } from '../helpers/fake-engine.ts';
import { freePort } from '../helpers/free-port.ts';
import { gate } from '../helpers/gate.ts';
@@ -27,6 +28,8 @@ const resolveModule = new URL('../../dist/config/resolve.js', import.meta.url).h
const { resolveConfig } = (await import(resolveModule)) as typeof import('../../src/config/resolve.ts');
const secretsModule = new URL('../../dist/secrets.js', import.meta.url).href;
const { credentials, secrets } = (await import(secretsModule)) as typeof import('../../src/secrets.ts');
+const agentModule = new URL('../../dist/agent/public.js', import.meta.url).href;
+const { defineTool } = (await import(agentModule)) as typeof import('../../src/agent/public.ts');
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -62,6 +65,8 @@ describe('SessionHost', { timeout: 60_000 }, () => {
/** Runs while a config file evaluates, as its top-level code would. */
readonly evaluate?: (configPath: string) => void;
readonly onSessionEnd?: (summary: McpSessionSummary) => void;
+ /** Project tools under `agents.default.tools`. */
+ readonly tools?: Record>;
}
/**
@@ -81,6 +86,7 @@ describe('SessionHost', { timeout: 60_000 }, () => {
credentials: { admin: { username: 'admin', password: 'kiosk-pw' } },
...(options.trace === undefined ? {} : { trace: options.trace }),
...(options.video === undefined ? {} : { video: options.video }),
+ ...(options.tools === undefined ? {} : { agents: { default: { tools: options.tools } } }),
} as never,
{ projectRoot: dir, env: {}, configPath },
);
@@ -120,41 +126,22 @@ describe('SessionHost', { timeout: 60_000 }, () => {
expect(fake.stats()).toMatchObject({ attemptsStarted: 2, attemptsEnded: 2, disposes: 2 });
});
- it('closes the schema of every fixed tool: an argument it does not declare fails validation', () => {
- const specs = host(engines().next).toolSpecs();
- expect(specs.map((spec) => spec.name)).toEqual(['open_session', 'tools', 'call', 'close_session']);
- const accepted: Record> = {
- open_session: { target: 'kiosk' },
- tools: { tool: 'tap' },
- call: { tool: 'tap', args: { target: 'n1' } },
- close_session: {},
- };
- for (const spec of specs) {
- expect(spec.inputSchema.safeParse(accepted[spec.name]).success, spec.name).toBe(true);
- const decorated = spec.inputSchema.safeParse({ ...accepted[spec.name], force: true });
- expect(decorated.success, spec.name).toBe(false);
- if (!decorated.success) expect(decorated.error.issues.map((issue) => issue.code)).toEqual(['unrecognized_keys']);
- }
- });
-
it('closes an idle session and disposes the engine', async () => {
const fakes = engines();
- const idle = host(fakes.next, { idleMs: 300 });
+ const idle = host(fakes.next, { idleMs: 100 });
await idle.open({});
expect(idle.isOpen).toBe(true);
- await sleep(1_000);
- expect(idle.isOpen).toBe(false);
+ await expect.poll(() => idle.isOpen, { timeout: 5_000 }).toBe(false);
expect(logs.some((line) => line.includes('idle for'))).toBe(true);
expect(fakes.made[0]!.stats()).toMatchObject({ attemptsStarted: 1, attemptsEnded: 1, disposes: 1 });
});
it('ends a session at its TTL through the step deadline', async () => {
const fakes = engines();
- const short = host(fakes.next, { ttlMs: 1_000 });
+ const short = host(fakes.next, { ttlMs: 500 });
await short.open({});
- await sleep(3_000);
- expect(short.isOpen).toBe(false);
- expect(logs.some((line) => line.includes('exceeded its 1000 ms timeout'))).toBe(true);
+ await expect.poll(() => short.isOpen, { timeout: 5_000 }).toBe(false);
+ await expect.poll(() => logs.some((line) => line.includes('exceeded its 500 ms timeout')), { timeout: 5_000 }).toBe(true);
expect(fakes.made[0]!.stats()).toMatchObject({ attemptsEnded: 1, disposes: 1 });
});
@@ -203,7 +190,9 @@ describe('SessionHost', { timeout: 60_000 }, () => {
await navigating.promise;
const second = sessionId(await pair.open({}));
held.open();
- await pair.close('done', sessionId(await first));
+ const firstId = sessionId(await first);
+ expect(second).not.toBe(firstId);
+ await pair.close('done', firstId);
await pair.close('done', second);
});
@@ -467,9 +456,8 @@ describe('SessionHost', { timeout: 60_000 }, () => {
await counted.close('closed by the agent', id);
await expect(counted.open({ target: 'nowhere' })).rejects.toMatchObject({ code: 'UNKNOWN_TARGET' });
await counted.open({});
- await sleep(1_000);
+ await expect.poll(() => summaries, { timeout: 5_000 }).toHaveLength(3);
- expect(summaries).toHaveLength(3);
const [closed, failed, idle] = summaries as [McpSessionSummary, McpSessionSummary, McpSessionSummary];
expect(closed).toMatchObject({
outcome: 'closed',
@@ -488,6 +476,29 @@ describe('SessionHost', { timeout: 60_000 }, () => {
expect(idle).toMatchObject({ outcome: 'closed', endedBy: 'idle', platform: 'kiosk' });
});
+ it('redacts a secret from what a project tool returns or throws', async () => {
+ const echo = defineTool(
+ { description: 'Echo the admin password.', inputSchema: z.object({}), execute: async () => 'pw=kiosk-pw' },
+ { mutates: false },
+ );
+ const leak = defineTool(
+ { description: 'Fail with the admin password.', inputSchema: z.object({}), execute: async () => { throw new Error('denied kiosk-pw'); } },
+ { mutates: false },
+ );
+ const extra = { signal: new AbortController().signal };
+ const text = (result: { content: { type: string; text?: string }[] }) => result.content.map((part) => part.text ?? '').join('\n');
+ const session = host(engines().next, { tools: { echo, leak } });
+ const id = sessionId(await session.open({}));
+
+ const echoed = await session.call(id, 'echo', {}, extra);
+ expect(text(echoed)).toBe('pw=');
+ const thrown = await session.call(id, 'leak', {}, extra);
+ expect(thrown.isError).toBe(true);
+ expect(text(thrown)).toContain('denied ');
+ expect(text(thrown)).not.toContain('kiosk-pw');
+ await session.close('done', id);
+ });
+
it('counts an action that failed and then could not look at the screen as one failed call', async () => {
let broken = false;
const fake = createFakeEngine({
diff --git a/packages/e2e/tests/integration/mcp-stdout-backpressure.test.ts b/packages/e2e/tests/integration/mcp-stdout-backpressure.test.ts
index a4cdd4adf..a05e70a82 100644
--- a/packages/e2e/tests/integration/mcp-stdout-backpressure.test.ts
+++ b/packages/e2e/tests/integration/mcp-stdout-backpressure.test.ts
@@ -66,7 +66,10 @@ describe('e2e mcp stdout backpressure', { timeout: 60_000 }, () => {
for (let index = 0; index < READS; index += 1) {
send({ jsonrpc: '2.0', id: 100 + index, method: 'resources/read', params: { uri: 'e2e://guide' } });
}
- await sleep(2_000);
+ await expect
+ .poll(() => server.exitCode !== null || server.stdout.readableLength >= server.stdout.readableHighWaterMark, { timeout: 10_000, interval: 5 })
+ .toBe(true);
+ await sleep(250);
expect(server.exitCode, `server exited early:\n${stderr}`).toBeNull();
let output = '';
diff --git a/packages/e2e/tests/integration/model-usage.test.ts b/packages/e2e/tests/integration/model-usage.test.ts
index ec55996ed..48457f30b 100644
--- a/packages/e2e/tests/integration/model-usage.test.ts
+++ b/packages/e2e/tests/integration/model-usage.test.ts
@@ -1,7 +1,6 @@
import { readFileSync } from 'node:fs';
import path from 'node:path';
-import { stripVTControlCharacters } from 'node:util';
-import { describe, expect, it, vi } from 'vitest';
+import { describe, expect, it } from 'vitest';
import { defineEngine } from '../../src/engine/index.ts';
import { judgment } from '../helpers/fake-model.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
@@ -10,37 +9,6 @@ import { createScriptedInstance, scriptedResult } from '../helpers/scripted-mode
import { snapshot } from '../helpers/snapshot.ts';
describe('reported model usage', () => {
- it('names the model the steps reported on the list reporter’s AI row', async () => {
- const project = createProject({ 'tests/usage.e2e.ts': `import { test } from 'e2e';
- test('judge twice', async ({ agent }) => {
- await agent.assert('ready');
- await agent.assert('still ready');
- });` });
- const model = createScriptedInstance('fake-loop', 'scripted-loop', async () =>
- scriptedResult([{ type: 'text', text: JSON.stringify(judgment(true, 'ready')) }], 'stop'));
- const written: string[] = [];
- const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
- written.push(String(chunk));
- return true;
- });
- try {
- const outcome = await runExisting(project, { appUrl: 'http://127.0.0.1:4599', config: {
- targets: [{ name: 'fake', platform: 'custom', engine: defineEngine({
- name: 'fake', version: '1', spiVersion: 1, observe: async () => snapshot([]),
- }) }], agents: { default: { model } }, cache: 'off',
- }, runOptions: { quiet: false } });
- stdoutWrite.mockRestore();
- expect(outcome.status).toBe('passed');
- const printed = stripVTControlCharacters(written.join(''));
- expect(printed).toContain('model fake-loop/scripted-loop');
- expect(printed).toMatch(/AI {2}240 tokens · 2 model calls · fake-loop\/scripted-loop\n/);
- expect(printed).not.toContain('undefined');
- } finally {
- stdoutWrite.mockRestore();
- project.cleanup();
- }
- });
-
it('keeps run totals representable when separate steps overflow their sum', async () => {
const project = createProject({ 'tests/usage.e2e.ts': `import { test } from 'e2e';
test('judge twice', async ({ agent }) => {
@@ -70,7 +38,7 @@ describe('reported model usage', () => {
}
});
- it.each([-1, 0.5, NaN, Infinity])('sanitizes malformed judgment counters (%s) before recording events', async (invalid) => {
+ it.each([-1, 0.5, NaN])('sanitizes malformed judgment counters (%s) before recording events', async (invalid) => {
const project = createProject({ 'tests/usage.e2e.ts': `import { test } from 'e2e';
test('judge', async ({ agent }) => { await agent.assert('ready'); });` });
const model = createScriptedInstance('test', 'usage', async () => ({
diff --git a/packages/e2e/tests/integration/parallel-workers.test.ts b/packages/e2e/tests/integration/parallel-workers.test.ts
index 60fadf9e8..eeb3e24e5 100644
--- a/packages/e2e/tests/integration/parallel-workers.test.ts
+++ b/packages/e2e/tests/integration/parallel-workers.test.ts
@@ -1,4 +1,4 @@
-import { mkdtempSync, readdirSync } from 'node:fs';
+import { existsSync, mkdtempSync, readdirSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
@@ -8,6 +8,7 @@ import {
resultByTitle,
runProjectWithConfigFile,
workerConfigSource,
+ workerFakeConfigSource,
} from '../helpers/run-project.ts';
describe('parallel worker execution', () => {
@@ -36,7 +37,7 @@ test('${name} runs', async ({ app }) => {
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/one.e2e.ts': testFile('one'), 'tests/two.e2e.ts': testFile('two') },
- { appUrl: app.url, configSource: workerConfigSource(2) },
+ { configSource: workerFakeConfigSource(2) },
);
expect(resultByTitle(outcome, 'one runs').status).toBe('passed');
expect(resultByTitle(outcome, 'two runs').status).toBe('passed');
@@ -72,6 +73,11 @@ test('starts with the seeded state', { session: 'seeded' }, async ({ app, screen
await app.open('/storage');
await expect(screen.getByRole('status', { name: 'Marker' })).toHaveText('saved');
});
+
+test('without a session starts clean', async ({ app, screen }) => {
+ await app.open('/storage');
+ await expect(screen.getByRole('status', { name: 'Marker' })).toHaveText('empty');
+});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/auth.setup.e2e.ts': setupFile, 'tests/consumer.e2e.ts': consumerFile },
@@ -79,7 +85,10 @@ test('starts with the seeded state', { session: 'seeded' }, async ({ app, screen
);
expect(resultByTitle(outcome, 'seed storage').status).toBe('passed');
expect(resultByTitle(outcome, 'starts with the seeded state').status).toBe('passed');
+ expect(resultByTitle(outcome, 'without a session starts clean').status).toBe('passed');
expect(outcome.exitCode).toBe(0);
+ const sessionsRoot = path.join(project.dir, '.e2e', 'sessions');
+ if (existsSync(sessionsRoot)) expect(readdirSync(sessionsRoot)).toEqual([]);
project.cleanup();
},
120_000,
@@ -113,7 +122,7 @@ test('unrelated still runs', async ({ app }) => {
'tests/dependent.e2e.ts': dependentFile,
'tests/unrelated.e2e.ts': unrelatedFile,
},
- { appUrl: app.url, configSource: workerConfigSource(2) },
+ { configSource: workerFakeConfigSource(2) },
);
const dependent = resultByTitle(outcome, 'depends on broken');
expect(dependent.status).toBe('skipped');
@@ -128,7 +137,7 @@ test('unrelated still runs', async ({ app }) => {
it(
'runs serial groups as one unit on one worker',
async () => {
- const serialFile = `import { test, expect } from 'e2e';
+ const serialFile = `import { test } from 'e2e';
test.describe('wizard', { serial: true }, () => {
let shared = 0;
@@ -136,13 +145,12 @@ test.describe('wizard', { serial: true }, () => {
test('step 1', async ({ app, screen }) => {
await app.open();
shared += 1;
- await screen.getByRole('button', { name: 'Increment' }).tap();
- await expect(screen.getByRole('status')).toHaveText('1');
+ await screen.getByRole('button', { name: 'Submit' }).tap();
});
test('step 2 shares state', async ({ screen }) => {
if (shared !== 1) throw new Error('module state was not preserved: ' + shared);
- await screen.getByRole('button', { name: 'Increment' }).tap();
+ await screen.getByRole('button', { name: 'Submit' }).tap();
});
});
`;
@@ -154,7 +162,7 @@ test('parallel neighbor', async ({ app }) => {
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/wizard.e2e.ts': serialFile, 'tests/other.e2e.ts': otherFile },
- { appUrl: app.url, configSource: workerConfigSource(2) },
+ { configSource: workerFakeConfigSource(2) },
);
expect(resultByTitle(outcome, 'step 1').status).toBe('passed');
expect(resultByTitle(outcome, 'step 2 shares state').status).toBe('passed');
@@ -193,7 +201,7 @@ test('survivor passes', async ({ app }) => {
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/crash.e2e.ts': crashFile, 'tests/survivor.e2e.ts': survivorFile },
- { appUrl: app.url, configSource: workerConfigSource(2) },
+ { configSource: workerFakeConfigSource(2) },
);
const crashed = resultByTitle(outcome, 'crashes the worker');
expect(crashed.status).toBe('failed');
@@ -207,6 +215,36 @@ test('survivor passes', async ({ app }) => {
const run = outcome.report['run'] as unknown as Record;
const errors = run['errors'] as Record[];
expect(errors.some((error) => error['code'] === 'WORKER_EXIT')).toBe(true);
+ assertValidReport(JSON.parse(readFileSync(outcome.reportPath!, 'utf8')));
+ project.cleanup();
+ },
+ 120_000,
+ );
+
+ it(
+ 'times out a body that never calls the harness and runs the next test of its file',
+ async () => {
+ const file = `import { test } from 'e2e';
+
+test('sleeps past its timeout', { timeout: 1000 }, async ({ app }) => {
+ await app.open();
+ await new Promise((resolve) => setTimeout(resolve, 60_000));
+});
+
+test('runs after the timeout', async ({ app }) => {
+ await app.open();
+});
+`;
+ const { outcome, project } = await runProjectWithConfigFile(
+ { 'tests/timeout.e2e.ts': file },
+ { configSource: workerFakeConfigSource(1) },
+ );
+ const timedOut = resultByTitle(outcome, 'sleeps past its timeout');
+ expect(timedOut.status).toBe('timed-out');
+ expect(timedOut.attempts[0]?.error?.message).toContain('test timed out after 1000 ms');
+ expect(resultByTitle(outcome, 'runs after the timeout').status).toBe('passed');
+ expect(outcome.exitCode).toBe(1);
+ expect(outcome.report.run.errors).toEqual([]);
project.cleanup();
},
120_000,
@@ -264,16 +302,18 @@ test('sleeps a long time', { timeout: 8000 }, async ({ app }) => {
test('waits in the queue', async () => {});
`;
const controller = new AbortController();
- const timer = setTimeout(() => controller.abort(), 4_000);
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/slow.e2e.ts': slowFile, 'tests/unstarted.e2e.ts': queuedFile },
{
- appUrl: app.url,
- configSource: workerConfigSource(1),
- runOptions: { interruptSignal: controller.signal },
+ configSource: workerFakeConfigSource(1),
+ runOptions: {
+ interruptSignal: controller.signal,
+ onEvent: (event) => {
+ if (event.type === 'test-started') controller.abort();
+ },
+ },
},
);
- clearTimeout(timer);
expect(outcome.exitCode).toBe(130);
expect(outcome.status).toBe('interrupted');
const result = resultByTitle(outcome, 'sleeps a long time');
@@ -307,7 +347,7 @@ test('would run last', async () => {});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/limit.e2e.ts': file },
- { appUrl: app.url, configSource: workerConfigSource(1), runOptions: { maxFailures: 1 } },
+ { configSource: workerFakeConfigSource(1), runOptions: { maxFailures: 1 } },
);
expect(outcome.exitCode).toBe(1);
const byDeclaration = outcome.results.toSorted((a, b) => a.test.declarationIndex - b.test.declarationIndex);
@@ -336,9 +376,8 @@ test('${name} never runs', async ({ app }) => {
const { outcome, project } = await runProjectWithConfigFile(
Object.fromEntries(['one', 'two', 'three', 'four'].map((name) => [`tests/${name}.e2e.ts`, testFile(name)])),
{
- appUrl: app.url,
// Every worker resolves its own project id, so none agrees with the runner's digest.
- configSource: workerConfigSource(4, "\n projectId: 'p-' + Math.random().toString(36).slice(2),"),
+ configSource: workerFakeConfigSource(4, "\n projectId: 'p-' + Math.random().toString(36).slice(2),"),
},
);
const run = outcome.report['run'] as unknown as Record;
@@ -372,7 +411,7 @@ test('runs after the rejection', async ({ app }) => {
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/stray.e2e.ts': strayFile },
- { appUrl: app.url, configSource: workerConfigSource(1) },
+ { configSource: workerFakeConfigSource(1) },
);
const stray = resultByTitle(outcome, 'leaves a rejection behind');
expect(stray.status).toBe('failed');
@@ -391,25 +430,31 @@ test('runs after the rejection', async ({ app }) => {
it(
'charges a rejection that surfaces late to the test running at the time',
async () => {
- const leakingFile = `import { test } from 'e2e';
+ const leakingFile = `import { existsSync } from 'node:fs';
+import { test } from 'e2e';
test('leaves a timer behind', async ({ app }) => {
await app.open();
- setTimeout(() => {
+ const marker = new URL('./running', import.meta.url);
+ const timer = setInterval(() => {
+ if (!existsSync(marker)) return;
+ clearInterval(timer);
void Promise.reject(new Error('late'));
- }, 1_500);
+ }, 20);
});
`;
- const sleepingFile = `import { test } from 'e2e';
+ const sleepingFile = `import { writeFileSync } from 'node:fs';
+import { test } from 'e2e';
test('is running when it surfaces', async ({ app }) => {
await app.open();
- await new Promise((resolve) => setTimeout(resolve, 6_000));
+ writeFileSync(new URL('./running', import.meta.url), '');
+ await new Promise((resolve) => setTimeout(resolve, 2_000));
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/a-leaks.e2e.ts': leakingFile, 'tests/b-sleeps.e2e.ts': sleepingFile },
- { appUrl: app.url, configSource: workerConfigSource(1) },
+ { configSource: workerFakeConfigSource(1) },
);
expect(resultByTitle(outcome, 'leaves a timer behind').status).toBe('passed');
const charged = resultByTitle(outcome, 'is running when it surfaces');
@@ -445,7 +490,7 @@ test('runs on the same worker afterwards', async ({ app }) => {
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/a-leaks.e2e.ts': leakingFile, 'tests/b-next.e2e.ts': nextFile },
- { appUrl: app.url, configSource: workerConfigSource(1) },
+ { configSource: workerFakeConfigSource(1) },
);
const finished = resultByTitle(outcome, 'finishes before the leak');
expect(finished.status).toBe('passed');
@@ -479,7 +524,7 @@ test('runs after both', async ({ app }) => {
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/twice.e2e.ts': file },
- { appUrl: app.url, configSource: workerConfigSource(1) },
+ { configSource: workerFakeConfigSource(1) },
);
const attempt = resultByTitle(outcome, 'fails twice').attempts[0]!;
expect(attempt.status).toBe('failed');
@@ -505,7 +550,7 @@ test('throws off the stack', async ({ app }) => {
setTimeout(() => {
throw new Error('boom-uncaught');
});
- await new Promise((resolve) => setTimeout(resolve, 2_000));
+ await new Promise((resolve) => setTimeout(resolve, 300));
});
test('never starts', async ({ app }) => {
@@ -514,7 +559,7 @@ test('never starts', async ({ app }) => {
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/fatal.e2e.ts': file },
- { appUrl: app.url, configSource: workerConfigSource(1) },
+ { configSource: workerFakeConfigSource(1) },
);
const run = outcome.report['run'] as unknown as Record;
const errors = run['errors'] as Record[];
@@ -558,7 +603,7 @@ test('runs on the next worker', async ({ app }) => {
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/a-early.e2e.ts': earlyFile, 'tests/b-next.e2e.ts': nextFile },
- { appUrl: app.url, configSource: workerConfigSource(1) },
+ { configSource: workerFakeConfigSource(1) },
);
const run = outcome.report['run'] as unknown as Record;
const errors = run['errors'] as Record[];
diff --git a/packages/e2e/tests/integration/press-sequentially.test.ts b/packages/e2e/tests/integration/press-sequentially.test.ts
deleted file mode 100644
index c43e3dd89..000000000
--- a/packages/e2e/tests/integration/press-sequentially.test.ts
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * `locator.pressSequentially` against a real browser: the fixture's City
- * field renders suggestions on key events only, so `fill` leaves the list
- * empty and `pressSequentially` populates it.
- */
-
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
-import { resultByTitle, runProject, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
-
-const SUITE = `import { test } from '@e2e-dev/web';
-import { expect } from 'e2e';
-
-test('fill sets the value without key events', async ({ app, screen }) => {
- await app.open('/verbs');
- await screen.getByLabel('City').fill('Wa');
- await expect(screen.getByLabel('City')).toHaveValue('Wa');
- await expect(screen.getByRole('status', { name: 'Keys' })).toHaveText('0');
- await expect(screen.getByRole('list', { name: 'Cities' }).getByRole('listitem')).toHaveCount(0);
-});
-
-test('pressSequentially types the same text as keystrokes', async ({ app, screen }) => {
- await app.open('/verbs');
- await screen.getByLabel('City').pressSequentially('Wa');
- await expect(screen.getByLabel('City')).toHaveValue('Wa');
- await expect(screen.getByLabel('City')).toBeFocused();
- await expect(screen.getByRole('status', { name: 'Keys' })).toHaveText('2');
- const cities = screen.getByRole('list', { name: 'Cities' }).getByRole('listitem');
- await expect(cities).toHaveCount(1);
- await expect(cities.first()).toHaveText('Warsaw');
-});
-
-test('a delay spaces the keystrokes', async ({ app, screen }) => {
- await app.open('/verbs');
- await screen.getByLabel('City').pressSequentially('Wr', { delay: 200 });
- await expect(screen.getByLabel('City')).toHaveValue('Wr');
- await expect(screen.getByRole('status', { name: 'Keys' })).toHaveText('2');
- await expect(screen.getByRole('list', { name: 'Cities' }).getByRole('listitem').first()).toHaveText('Wroclaw');
-});
-`;
-
-describe('locator.pressSequentially in a browser', () => {
- let app: FixtureApp;
- let outcome: RunOutcome;
- let project: FixtureProject;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- ({ outcome, project } = await runProject(
- { 'tests/typing.e2e.ts': SUITE },
- { appUrl: app.url, config: { actionTimeout: 5_000, assertionTimeout: 4_000, timeout: 30_000 } },
- ));
- }, 240_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it('fill reaches the value but never the key handlers', () => {
- const result = resultByTitle(outcome, 'fill sets the value without key events');
- expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe('passed');
- });
-
- it('pressSequentially fires the key handlers and records one locator step', () => {
- const result = resultByTitle(outcome, 'pressSequentially types the same text as keystrokes');
- expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe('passed');
- const step = result.attempts[0]!.steps.find((candidate) => candidate.api === 'locator.pressSequentially');
- expect(step).toMatchObject({ kind: 'locator', label: 'getByLabel("City")', status: 'passed' });
- });
-
- it('honors the delay between characters', () => {
- const result = resultByTitle(outcome, 'a delay spaces the keystrokes');
- expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe('passed');
- const step = result.attempts[0]!.steps.find((candidate) => candidate.api === 'locator.pressSequentially');
- expect(step?.durationMs).toBeGreaterThanOrEqual(200);
- });
-});
diff --git a/packages/e2e/tests/integration/report-markdown.test.ts b/packages/e2e/tests/integration/report-markdown.test.ts
deleted file mode 100644
index 36d3d43ee..000000000
--- a/packages/e2e/tests/integration/report-markdown.test.ts
+++ /dev/null
@@ -1,45 +0,0 @@
-/** The markdown reporter through the built runner: `summary.md` beside the report, with app text kept from GitHub's autolinks. */
-
-import { readdirSync, readFileSync } from 'node:fs';
-import path from 'node:path';
-import { describe, expect, it } from 'vitest';
-import { defineEngine } from '../../src/engine/index.ts';
-import { runProject } from '../helpers/run-project.ts';
-import { snapshot } from '../helpers/snapshot.ts';
-
-const HOSTILE_TITLE = 'ping @octocat see https://evil.example';
-
-/** The page with its code spans removed: what GitHub's mention, issue, and URL filters get to read. */
-function outsideCodeSpans(markdown: string): string {
- return markdown.replace(/(`+)[^`]*\1/g, '');
-}
-
-describe('markdown reporter', () => {
- it('writes summary.md with a mention and a URL from a test title as code, and nothing outside a code span GitHub would link', async () => {
- const { outcome, project } = await runProject(
- { 'tests/hostile.e2e.ts': `import { test } from 'e2e';
- test(${JSON.stringify(HOSTILE_TITLE)}, async () => { throw new Error('boom'); });` },
- {
- appUrl: 'http://127.0.0.1:4599',
- config: {
- targets: [{ name: 'fake', platform: 'custom', engine: defineEngine({ name: 'fake', version: '1', spiVersion: 1, observe: async () => snapshot([]) }) }],
- reporters: ['markdown'],
- },
- },
- );
- try {
- expect(outcome.exitCode).toBe(1);
- const summary = readFileSync(path.join(project.dir, '.e2e', 'summary.md'), 'utf8');
- expect(summary).toContain('ping `@octocat` see `https://evil.example`');
- expect(outsideCodeSpans(summary)).not.toContain('@octocat');
- expect(outsideCodeSpans(summary)).not.toMatch(/https:\/\/evil/);
- const failuresDir = path.join(project.dir, '.e2e', 'failures');
- const pages = readdirSync(failuresDir).map((file) => readFileSync(path.join(failuresDir, file), 'utf8'));
- expect(pages).toHaveLength(1);
- expect(pages[0]).toContain('# ✗ ping `@octocat` see `https://evil.example`');
- expect(outsideCodeSpans(pages[0]!)).not.toContain('@octocat');
- } finally {
- project.cleanup();
- }
- });
-});
diff --git a/packages/e2e/tests/integration/reporters.test.ts b/packages/e2e/tests/integration/reporters.test.ts
index e3728b1c8..ab64ab34d 100644
--- a/packages/e2e/tests/integration/reporters.test.ts
+++ b/packages/e2e/tests/integration/reporters.test.ts
@@ -7,30 +7,28 @@
import { existsSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
+import { createFakeEngine } from '../helpers/fake-engine.ts';
+import { engineConfig } from '../helpers/fixture-config.ts';
import { createProject, runExisting, type FixtureProject } from '../helpers/run-project.ts';
import type { FinishedRun, Reporter, RunEvent } from '../../src/index.ts';
const SUITE = `import { test, expect } from 'e2e';
-test('shows the counter', async ({ app, screen }) => {
+test('shows the submit button', async ({ app, screen }) => {
await app.open();
- await expect(screen.getByRole('status')).toHaveText('0');
+ await expect(screen.getByRole('button', { name: 'Submit' })).toBeVisible();
});
`;
describe('reporter objects', () => {
- let app: FixtureApp;
let project: FixtureProject;
- beforeAll(async () => {
- app = await startFixtureApp();
+ beforeAll(() => {
project = createProject({ 'tests/reporters.e2e.ts': SUITE });
});
- afterAll(async () => {
+ afterAll(() => {
project?.cleanup();
- await app?.close();
});
afterEach(() => {
@@ -51,8 +49,12 @@ describe('reporter objects', () => {
},
};
const outcome = await runExisting(project, {
- appUrl: app.url,
- config: { tests: 'tests/**/*.e2e.ts', reporters: [recording], cache: 'off' as const },
+ config: {
+ ...engineConfig(createFakeEngine({ trace: true }).engine),
+ tests: 'tests/**/*.e2e.ts',
+ reporters: [recording],
+ cache: 'off' as const,
+ },
});
expect(outcome.exitCode).toBe(0);
@@ -116,8 +118,12 @@ describe('reporter objects', () => {
},
};
const outcome = await runExisting(project, {
- appUrl: app.url,
- config: { tests: 'tests/**/*.e2e.ts', reporters: [throwing, hanging, malformed, scalar, loud, quiet], cache: 'off' as const },
+ config: {
+ ...engineConfig(createFakeEngine({ trace: true }).engine),
+ tests: 'tests/**/*.e2e.ts',
+ reporters: [throwing, hanging, malformed, scalar, loud, quiet],
+ cache: 'off' as const,
+ },
runOptions: {
reporterTimeout: 200,
onEvent: () => {
@@ -156,8 +162,12 @@ describe('reporter objects', () => {
},
};
const outcome = await runExisting(project, {
- appUrl: app.url,
- config: { tests: 'tests/**/*.e2e.ts', reporters: [uploading], cache: 'off' as const },
+ config: {
+ ...engineConfig(createFakeEngine({ trace: true }).engine),
+ tests: 'tests/**/*.e2e.ts',
+ reporters: [uploading],
+ cache: 'off' as const,
+ },
runOptions: { reporterTimeout: 30_000, forceSignal: force.signal },
});
diff --git a/packages/e2e/tests/integration/run-events.test.ts b/packages/e2e/tests/integration/run-events.test.ts
index b1a0d6203..3524bc4d1 100644
--- a/packages/e2e/tests/integration/run-events.test.ts
+++ b/packages/e2e/tests/integration/run-events.test.ts
@@ -1,7 +1,6 @@
/**
* The `onEvent` host stream end to end: a real run emits an ordered,
- * JSON-serializable event sequence that mirrors what the report persists,
- * while a throwing sink is quarantined without affecting the run.
+ * JSON-serializable event sequence that mirrors what the report persists.
*/
import { mkdirSync, readFileSync } from 'node:fs';
@@ -14,7 +13,7 @@ import {
createProject,
runExisting,
runProjectWithConfigFile,
- workerConfigSource,
+ workerFakeConfigSource,
type FixtureProject,
type RunOutcome,
} from '../helpers/run-project.ts';
@@ -30,6 +29,14 @@ test('increments once', async ({ app, agent, screen }) => {
});
`;
+/** A test that touches no page beyond opening the app, for the fake engine. */
+const OPENS_THE_APP = `import { test } from 'e2e';
+
+test('opens the app', async ({ app }) => {
+ await app.open();
+});
+`;
+
/** Taps Increment once and passes. No AI SDK, no model. */
const oneTapExecutor: StepExecutor = {
name: 'one-tap-executor',
@@ -142,7 +149,6 @@ describe('run events', () => {
describe('run events: attempt and step identity', () => {
it.each(['in-process', 'workers'] as const)('joins retries and serial members to their report records through %s', async (execution) => {
- const app = await startFixtureApp();
const events: RunEvent[] = [];
const files = {
'tests/retry.e2e.ts': `import { test } from 'e2e';
@@ -173,8 +179,7 @@ test.describe('group', { serial: true, retries: 1 }, () => {
const runOptions = { onEvent: (event: RunEvent) => { events.push(event); } };
if (execution === 'workers') {
const result = await runProjectWithConfigFile(files, {
- appUrl: app.url,
- configSource: workerConfigSource(2, `
+ configSource: workerFakeConfigSource(2, `
cache: 'off',
agents: { default: { executor: {
name: 'retry-once',
@@ -191,7 +196,6 @@ test.describe('group', { serial: true, retries: 1 }, () => {
} else {
project = createProject(files);
outcome = await runExisting(project, {
- appUrl: app.url,
config: { cache: 'off', agents: { default: { executor } } },
runOptions,
});
@@ -227,14 +231,12 @@ test.describe('group', { serial: true, retries: 1 }, () => {
}
} finally {
project?.cleanup();
- await app.close();
}
}, 120_000);
});
describe('run events: run lifecycle hygiene', () => {
it('a test\u2019s console output arrives as output events attributed to it, and never on the runner\u2019s streams', async () => {
- const app = await startFixtureApp();
const events: RunEvent[] = [];
const written: string[] = [];
const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
@@ -258,8 +260,7 @@ test('talks', async () => {
`,
},
{
- appUrl: app.url,
- configSource: workerConfigSource(1, `
+ configSource: workerFakeConfigSource(1, `
secrets: { 'stripe-key': 'sk_live_generic_4242' },`),
runOptions: { onEvent: (event: RunEvent) => { events.push(event); } },
},
@@ -280,19 +281,17 @@ test('talks', async () => {
"hello { from: 'the test' }\ntoken leaked twice \nsplit across writes\n",
);
expect(events.some((event) => event.type === 'output' && /sk_live|eric_4242/.test(event.text))).toBe(false);
- expect(inTest.every((event) => event.target === 'web' && event.pair?.agent === 'default')).toBe(true);
+ expect(inTest.every((event) => event.target === 'fake' && event.pair?.agent === 'default')).toBe(true);
// The module's top level runs while the file loads, outside any pair.
expect(output.some((event) => event.pair === undefined && event.text === 'top level\n')).toBe(true);
expect(written.join('')).not.toContain('hello');
} finally {
stdoutWrite.mockRestore();
project?.cleanup();
- await app.close();
}
}, 120_000);
it('a serial member’s unfinished line leaves with the member that wrote it, not the one after', async () => {
- const app = await startFixtureApp();
const events: RunEvent[] = [];
let project: FixtureProject | undefined;
try {
@@ -306,9 +305,8 @@ test.describe('shared', { serial: true }, () => {
`,
},
{
- appUrl: app.url,
// A registered secret is what makes the worker hold an unfinished line.
- configSource: workerConfigSource(1, `
+ configSource: workerFakeConfigSource(1, `
secrets: { 'stripe-key': 'sk_live_generic_4242' },`),
runOptions: { onEvent: (event: RunEvent) => { events.push(event); } },
},
@@ -327,13 +325,11 @@ test.describe('shared', { serial: true }, () => {
expect(outputOf('shared > second')).toEqual([' done\n']);
} finally {
project?.cleanup();
- await app.close();
}
}, 120_000);
it('a junit-write failure is one stderr line and leaves the outcome, the report, and run-finished alone', async () => {
- const app = await startFixtureApp();
- const project = createProject({ 'tests/events.e2e.ts': SUITE });
+ const project = createProject({ 'tests/events.e2e.ts': OPENS_THE_APP });
// A directory where junit.xml must be written makes its atomic rename fail
// while report.json beside it still has a clear path.
mkdirSync(path.join(project.dir, '.e2e', 'junit.xml'), { recursive: true });
@@ -341,12 +337,9 @@ test.describe('shared', { serial: true }, () => {
const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
try {
const outcome = await runExisting(project, {
- appUrl: app.url,
config: {
tests: 'tests/**/*.e2e.ts',
reporters: ['junit'] as const,
- agents: { default: { executor: oneTapExecutor } },
- cache: 'off' as const,
},
runOptions: { onEvent: (event) => {
events.push(event);
@@ -368,24 +361,19 @@ test.describe('shared', { serial: true }, () => {
} finally {
vi.restoreAllMocks();
project.cleanup();
- await app.close();
}
}, 120_000);
it('a report-write failure fails the run, withholds reportPath, and precedes run-finished', async () => {
- const app = await startFixtureApp();
- const project = createProject({ 'tests/events.e2e.ts': SUITE });
+ const project = createProject({ 'tests/events.e2e.ts': OPENS_THE_APP });
// A directory where report.json must be written makes the atomic rename fail.
mkdirSync(path.join(project.dir, '.e2e', 'report.json'), { recursive: true });
const events: RunEvent[] = [];
try {
const outcome = await runExisting(project, {
- appUrl: app.url,
config: {
tests: 'tests/**/*.e2e.ts',
reporters: ['json'] as const,
- agents: { default: { executor: oneTapExecutor } },
- cache: 'off' as const,
},
runOptions: { onEvent: (event) => {
events.push(event);
@@ -406,22 +394,17 @@ test.describe('shared', { serial: true }, () => {
expect(outcome.report.run.errors.some((entry) => entry.phase === 'report')).toBe(true);
} finally {
project.cleanup();
- await app.close();
}
}, 120_000);
it('a cancellation before any test starts ends the run as interrupted without collecting', async () => {
- const app = await startFixtureApp();
- const project = createProject({ 'tests/events.e2e.ts': SUITE });
+ const project = createProject({ 'tests/events.e2e.ts': OPENS_THE_APP });
const events: RunEvent[] = [];
try {
const outcome = await runExisting(project, {
- appUrl: app.url,
config: {
tests: 'tests/**/*.e2e.ts',
reporters: ['json'] as const,
- agents: { default: { executor: oneTapExecutor } },
- cache: 'off' as const,
},
runOptions: {
interruptSignal: AbortSignal.abort(),
@@ -439,58 +422,22 @@ test.describe('shared', { serial: true }, () => {
expect(outcome.reportPath).toBeUndefined();
} finally {
project.cleanup();
- await app.close();
}
}, 120_000);
it('clears the credential registry when the run resolves', async () => {
- const app = await startFixtureApp();
- const project = createProject({ 'tests/events.e2e.ts': SUITE });
+ const project = createProject({ 'tests/events.e2e.ts': OPENS_THE_APP });
try {
await runExisting(project, {
- appUrl: app.url,
config: {
tests: 'tests/**/*.e2e.ts',
reporters: ['json'] as const,
- agents: { default: { executor: oneTapExecutor } },
- cache: 'off' as const,
credentials: { member: { username: 'member', password: 'hunter2' } },
},
});
expect(() => credentials.user('member')).toThrow(/only available while the e2e runner/);
} finally {
project.cleanup();
- await app.close();
- }
- }, 120_000);
-});
-
-describe('run events: quarantined sink', () => {
- it('a throwing sink never affects the run outcome', async () => {
- const app = await startFixtureApp();
- const project = createProject({ 'tests/events.e2e.ts': SUITE });
- let calls = 0;
- try {
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: {
- tests: 'tests/**/*.e2e.ts',
- reporters: ['json'] as const,
- agents: { default: { executor: oneTapExecutor } },
- cache: 'off' as const,
- },
- runOptions: {
- onEvent: () => {
- calls += 1;
- throw new Error('broken sink');
- },
- },
- });
- expect(outcome.exitCode).toBe(0);
- expect(calls).toBe(1);
- } finally {
- project.cleanup();
- await app.close();
}
}, 120_000);
});
diff --git a/packages/e2e/tests/integration/runner-lifecycle.test.ts b/packages/e2e/tests/integration/runner-lifecycle.test.ts
index a845dc4db..16660b462 100644
--- a/packages/e2e/tests/integration/runner-lifecycle.test.ts
+++ b/packages/e2e/tests/integration/runner-lifecycle.test.ts
@@ -4,7 +4,17 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
import { resultId } from '../../src/internal/ids.ts';
-import { createProject, listProject, resultByTitle, runExisting, runProject, type RunOutcome } from '../helpers/run-project.ts';
+import {
+ createProject,
+ listProject,
+ resultByTitle,
+ runExisting,
+ runProject,
+ runExistingWithConfigFile,
+ runProjectWithConfigFile,
+ workerFakeConfigSource,
+ type RunOutcome,
+} from '../helpers/run-project.ts';
import type { FinishedRun } from '../../src/index.ts';
describe('runner lifecycle', () => {
@@ -52,10 +62,7 @@ test.afterEach(() => log('afterEach:file-late'));
`;
const logPath = path.join('/tmp', `e2e-hooks-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject(
- { 'tests/hooks.e2e.ts': hooksFile },
- { appUrl: app.url },
- );
+ const { outcome, project } = await runProject({ 'tests/hooks.e2e.ts': hooksFile }, {});
expect(resultByTitle(outcome, 'inside group').status).toBe('passed');
const entries = readFileSync(logPath, 'utf8').trim().split('\n');
expect(entries).toEqual([
@@ -65,7 +72,7 @@ test.afterEach(() => log('afterEach:file-late'));
'beforeEach:file-late',
'beforeEach:group',
'beforeEach:group-late',
- 'body:web',
+ 'body:fake',
'afterEach:group-late',
'afterEach:group',
'afterEach:file-late',
@@ -73,6 +80,7 @@ test.afterEach(() => log('afterEach:file-late'));
'afterAll:group',
'afterAll:file',
]);
+ expect(existsSync(path.join(project.dir, '.e2e', 'junit.xml'))).toBe(false);
project.cleanup();
},
120_000,
@@ -105,7 +113,7 @@ test.setup('sign in', { sessions: ['admin'] }, async ({ session }) => {
test('consumer', { session: 'admin' }, async () => {});
`;
- const { outcome, project } = await runProject({ 'tests/runtime-skip.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/runtime-skip.e2e.ts': file }, {});
const conditional = resultByTitle(outcome, 'single organization');
expect(conditional.status).toBe('skipped');
expect(conditional.skip).toEqual({ cause: 'explicit', reason: 'the demo tenant has a single organization' });
@@ -155,7 +163,7 @@ test.describe('wizard', { serial: true }, () => {
`;
const logPath = path.join('/tmp', `e2e-skiprealm-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/skip-realm.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/skip-realm.e2e.ts': file }, {});
expect(resultByTitle(outcome, 'skips first').status).toBe('skipped');
expect(resultByTitle(outcome, 'runs second').status).toBe('passed');
const step1 = resultByTitle(outcome, 'step 1 skips');
@@ -185,7 +193,7 @@ test.describe('wizard', { serial: true }, () => {
test.skip(true, 'not here');
test('never registered', async () => {});
`;
- const { outcome, project } = await runProject({ 'tests/skip-outside.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/skip-outside.e2e.ts': file }, {});
expect(outcome.exitCode).toBe(2);
expect(outcome.report.run.errors.map((error) => error.code)).toEqual(['COLLECTION_ERROR']);
expect(outcome.report.run.errors[0]?.message).toMatch(/must be called inside a test body/);
@@ -205,7 +213,6 @@ test('never registered', async () => {});
};
const notices: string[] = [];
const narrowed = await runProject(files, {
- appUrl: app.url,
runOptions: { files: ['tests/good.e2e.ts'], onEvent: (event) => { if (event.type === 'notice' && event.target === 'collect') notices.push(event.message); } },
});
expect(narrowed.outcome.exitCode).toBe(0);
@@ -217,13 +224,13 @@ test('never registered', async () => {});
]);
narrowed.project.cleanup();
- const named = await runProject(files, { appUrl: app.url, runOptions: { files: ['tests/broken.e2e.ts'] } });
+ const named = await runProject(files, { runOptions: { files: ['tests/broken.e2e.ts'] } });
expect(named.outcome.exitCode).toBe(2);
expect(named.outcome.report.run.errors.map((error) => error.code)).toEqual(['COLLECTION_ERROR']);
expect(named.outcome.report.run.errors[0]?.message).toMatch(/^failed to collect tests\/broken\.e2e\.ts: /);
named.project.cleanup();
- const whole = await runProject(files, { appUrl: app.url });
+ const whole = await runProject(files, {});
expect(whole.outcome.exitCode).toBe(2);
expect(whole.outcome.report.run.errors.map((error) => error.code)).toEqual(['COLLECTION_ERROR']);
expect(whole.outcome.report.run.errors[0]?.message).toMatch(/failed to collect tests\/broken\.e2e\.ts/);
@@ -248,7 +255,7 @@ test('independent survives', async ({ app }) => {
await app.open();
});
`;
- const { outcome, project } = await runProject({ 'tests/hookfail.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/hookfail.e2e.ts': file }, {});
const skipped = resultByTitle(outcome, 'unreachable');
expect(skipped.status).toBe('skipped');
expect(skipped.skip?.cause).toBe('hook-failed');
@@ -282,7 +289,7 @@ test('second passes', async ({ app }) => {
`;
const logPath = path.join('/tmp', `e2e-discard-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/discard.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/discard.e2e.ts': file }, {});
expect(resultByTitle(outcome, 'first fails').status).toBe('failed');
expect(resultByTitle(outcome, 'second passes').status).toBe('passed');
// The failed realm is discarded but its afterAll still runs; the fresh
@@ -338,7 +345,7 @@ test('consumer', { session: 'seeded' }, async ({ app }) => {
`;
const logPath = path.join('/tmp', `e2e-suitehooks-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/hooks.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/hooks.e2e.ts': file }, {});
expect(resultByTitle(outcome, 'seed').status).toBe('passed');
expect(resultByTitle(outcome, 'step 1').status).toBe('passed');
expect(resultByTitle(outcome, 'step 2').status).toBe('passed');
@@ -379,7 +386,7 @@ test.describe('wizard', { serial: true }, () => {
test('step 2', async () => {});
});
`;
- const { outcome, project } = await runProject({ 'tests/serialhook.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/serialhook.e2e.ts': file }, {});
for (const title of ['step 1', 'step 2']) {
const result = resultByTitle(outcome, title);
expect(result.status).toBe('skipped');
@@ -413,7 +420,7 @@ test('fails then cannot retry', { retries: 1 }, async ({ app }) => {
`;
const marker = path.join('/tmp', `e2e-retryhook-${Date.now()}`);
process.env['RETRY_MARKER'] = marker;
- const { outcome, project } = await runProject({ 'tests/retryhook.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/retryhook.e2e.ts': file }, {});
const result = resultByTitle(outcome, 'fails then cannot retry');
expect(result.status).toBe('failed');
expect(result.attempts).toHaveLength(1);
@@ -441,7 +448,7 @@ test('flaky test', { retries: 2 }, async ({ app }) => {
`;
const marker = path.join('/tmp', `e2e-flaky-${Date.now()}`);
process.env['FLAKY_MARKER'] = marker;
- const { outcome, project } = await runProject({ 'tests/flaky.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/flaky.e2e.ts': file }, {});
const result = resultByTitle(outcome, 'flaky test');
expect(result.status).toBe('flaky');
expect(result.attempts).toHaveLength(2);
@@ -463,7 +470,7 @@ test('sleeps forever', { timeout: 1500 }, async ({ app }) => {
await new Promise((resolve) => setTimeout(resolve, 60_000));
});
`;
- const { outcome, project } = await runProject({ 'tests/slow.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/slow.e2e.ts': file }, {});
const result = resultByTitle(outcome, 'sleeps forever');
expect(result.status).toBe('timed-out');
expect(outcome.exitCode).toBe(1);
@@ -472,81 +479,6 @@ test('sleeps forever', { timeout: 1500 }, async ({ app }) => {
120_000,
);
- it(
- 'produces sessions in setup tests and restores them for consumers',
- async () => {
- const setupFile = `import { test, expect } from 'e2e';
-
-test.setup('seed storage', { sessions: ['seeded'] }, async ({ app, screen, session }) => {
- await app.open('/storage');
- await screen.getByRole('button', { name: 'Save marker' }).tap();
- await expect(screen.getByRole('status', { name: 'Marker' })).toHaveText('saved');
- await session.save('seeded');
-});
-`;
- const consumerFile = `import { test } from '@e2e-dev/web';
-import { expect } from 'e2e';
-
-test('starts with the seeded state', { session: 'seeded' }, async ({ app, screen, browser }) => {
- await app.open('/storage');
- await expect(screen.getByRole('status', { name: 'Marker' })).toHaveText('saved');
- const cookies = await browser.cookies();
- if (!cookies.some((cookie) => cookie.name === 'fixture')) {
- throw new Error('expected the fixture cookie from the session');
- }
-});
-
-test('without a session starts clean', async ({ app, screen }) => {
- await app.open('/storage');
- await expect(screen.getByRole('status', { name: 'Marker' })).toHaveText('empty');
-});
-`;
- const { outcome, project } = await runProject(
- { 'tests/auth.setup.e2e.ts': setupFile, 'tests/consumer.e2e.ts': consumerFile },
- { appUrl: app.url },
- );
- expect(resultByTitle(outcome, 'seed storage').status).toBe('passed');
- expect(resultByTitle(outcome, 'starts with the seeded state').status).toBe('passed');
- expect(resultByTitle(outcome, 'without a session starts clean').status).toBe('passed');
- expect(outcome.exitCode).toBe(0);
- const sessionsRoot = path.join(project.dir, '.e2e', 'sessions');
- if (existsSync(sessionsRoot)) {
- expect(readdirSync(sessionsRoot)).toEqual([]);
- }
- project.cleanup();
- },
- 120_000,
- );
-
- it(
- 'skips session consumers when their setup fails',
- async () => {
- const file = `import { test } from 'e2e';
-
-test.setup('failing setup', { sessions: ['broken'] }, async ({ app }) => {
- await app.open();
- throw new Error('cannot authenticate');
-});
-
-test('depends on broken', { session: 'broken' }, async ({ app }) => {
- await app.open();
-});
-
-test('unrelated still runs', async ({ app }) => {
- await app.open();
-});
-`;
- const { outcome, project } = await runProject({ 'tests/dep.e2e.ts': file }, { appUrl: app.url });
- const dependent = resultByTitle(outcome, 'depends on broken');
- expect(dependent.status).toBe('skipped');
- expect(dependent.skip?.cause).toBe('setup-failed');
- expect(resultByTitle(outcome, 'unrelated still runs').status).toBe('passed');
- expect(outcome.exitCode).toBe(1);
- project.cleanup();
- },
- 120_000,
- );
-
it(
'runs serial groups as one unit with shared state and predecessor skips',
async () => {
@@ -606,36 +538,18 @@ test.describe('wizard', { serial: true }, () => {
120_000,
);
- it(
- 'reports config errors as exit 2 without executing tests',
- async () => {
- const { outcome, project } = await runProject(
- { 'tests/none.e2e.ts': `import { test } from 'e2e';\ntest('x', async () => {});\n` },
- {
- appUrl: app.url,
- config: { reporters: ['json', 'list'] as never },
- runOptions: { quiet: true },
- },
- );
- expect(outcome.exitCode).toBe(2);
- expect(outcome.status).toBe('error');
- project.cleanup();
- },
- 120_000,
- );
-
it(
'writes junit.xml beside the report from the same document when the junit reporter is selected',
async () => {
const file = `import { test } from 'e2e';
test('passes', { tags: ['smoke'] }, async () => {});
test('fails', async () => {
- throw new Error('junit & "reports" this');
+ throw new Error('junit reports this');
});
`;
const { outcome, project } = await runProject(
{ 'tests/junit.e2e.ts': file },
- { appUrl: app.url, config: { reporters: ['junit'] } },
+ { config: { reporters: ['junit'] } },
);
expect(outcome.exitCode).toBe(1);
expect(outcome.report.run.results.map((result) => result.tags)).toEqual([['smoke'], []]);
@@ -645,29 +559,12 @@ test('fails', async () => {
expect(xml).toContain(
' name.endsWith('.tmp'))).toEqual([]);
project.cleanup();
},
120_000,
);
- it(
- 'writes no junit.xml unless the reporter is selected',
- async () => {
- const { outcome, project } = await runProject(
- { 'tests/no-junit.e2e.ts': `import { test } from 'e2e';\ntest('x', async () => {});\n` },
- { appUrl: app.url },
- );
- expect(outcome.exitCode).toBe(0);
- expect(existsSync(path.join(project.dir, '.e2e', 'junit.xml'))).toBe(false);
- project.cleanup();
- },
- 120_000,
- );
-
it(
'lists the selected pairs in report order without running anything',
async () => {
@@ -683,12 +580,12 @@ test.describe('group', () => {
test('other', { tags: ['smoke'] }, async () => {});
`,
};
- const { pairs, project } = await listProject(files, { appUrl: 'http://127.0.0.1:9' });
+ const { pairs, project } = await listProject(files, {});
expect(pairs.map((pair) => [pair.file, pair.titlePath.join(' > '), pair.target, pair.disposition])).toEqual([
- ['tests/list.e2e.ts', 'plain', 'web', 'run'],
- ['tests/list.e2e.ts', 'group > nested', 'web', 'run'],
- ['tests/list.e2e.ts', 'group > left out', 'web', 'skip'],
- ['tests/other.e2e.ts', 'other', 'web', 'run'],
+ ['tests/list.e2e.ts', 'plain', 'fake', 'run'],
+ ['tests/list.e2e.ts', 'group > nested', 'fake', 'run'],
+ ['tests/list.e2e.ts', 'group > left out', 'fake', 'skip'],
+ ['tests/other.e2e.ts', 'other', 'fake', 'run'],
]);
expect(pairs.map((pair) => pair.tags)).toEqual([[], ['smoke'], [], ['smoke']]);
expect(pairs[2]?.skipReason).toBe('not today');
@@ -697,30 +594,10 @@ test('other', { tags: ['smoke'] }, async () => {});
// A config glob spelled with a leading `./` selects the same files.
const dotted = await listProject(files, {
- appUrl: 'http://127.0.0.1:9',
config: { tests: './tests/**/*.e2e.ts' },
});
expect(dotted.pairs.map((pair) => pair.title)).toEqual(['plain', 'nested', 'left out', 'other']);
dotted.project.cleanup();
-
- const tagged = await listProject(files, {
- appUrl: 'http://127.0.0.1:9',
- listOptions: { tags: ['smoke'], files: ['tests/other.e2e.ts'] },
- });
- expect(tagged.pairs.map((pair) => pair.title)).toEqual(['other']);
- tagged.project.cleanup();
-
- const grepped = await listProject(files, {
- appUrl: 'http://127.0.0.1:9',
- listOptions: { grepInvert: [/^group/], excludeTags: ['smoke'] },
- });
- expect(grepped.pairs.map((pair) => pair.title)).toEqual(['plain']);
- grepped.project.cleanup();
-
-
- await expect(listProject({ 'tests/empty.txt': '' }, { appUrl: 'http://127.0.0.1:9' })).rejects.toMatchObject({
- code: 'NO_TESTS',
- });
},
120_000,
);
@@ -739,7 +616,7 @@ test('breaks the second time', async () => {
});
`,
};
- const { outcome, project } = await runProject(files, { appUrl: app.url, runOptions: { repeatEach: 3, retries: 0 } });
+ const { outcome, project } = await runProject(files, { runOptions: { repeatEach: 3, retries: 0 } });
expect(outcome.exitCode).toBe(1);
const results = outcome.report.run.results.toSorted((a, b) => a.declarationIndex - b.declarationIndex || a.repeat - b.repeat);
expect(results.map((result) => [result.titlePath[0], result.repeat, result.status])).toEqual([
@@ -751,7 +628,7 @@ test('breaks the second time', async () => {
['breaks the second time', 2, 'failed'],
]);
expect(new Set(results.map((result) => result.id)).size).toBe(6);
- expect(results[0]!.id).toBe(resultId(results[0]!.testId, 'web', 'default'));
+ expect(results[0]!.id).toBe(resultId(results[0]!.testId, 'fake', 'default'));
const paths = results[4]!.attempts[0]!.artifacts.flatMap((artifact) => (artifact.path === undefined ? [] : [artifact.path]));
expect(paths.length).toBeGreaterThan(0);
expect(paths.every((artifactPath) => artifactPath.includes('/repeat-1/'))).toBe(true);
@@ -761,7 +638,6 @@ test('breaks the second time', async () => {
// and hands its reporters the report it selected from.
let handed: FinishedRun | undefined;
const rerun = await runExisting(project, {
- appUrl: app.url,
runOptions: { lastFailed: true },
config: {
reporters: [
@@ -777,7 +653,7 @@ test('breaks the second time', async () => {
expect(rerun.results.filter((result) => result.selected).map((result) => result.test.title)).toEqual(['breaks the second time']);
expect(handed?.lastRun?.run.id).toBe(outcome.report.run.id);
- const invalid = await runExisting(project, { appUrl: app.url, runOptions: { repeatEach: 0 } });
+ const invalid = await runExisting(project, { runOptions: { repeatEach: 0 } });
expect(invalid.exitCode).toBe(2);
expect(invalid.report.run.errors.map((error) => error.message)).toEqual(['repeatEach must be a positive safe integer, got 0']);
project.cleanup();
@@ -798,7 +674,6 @@ test('never runs either', async () => { throw new Error('three'); });
};
const events: string[] = [];
const { outcome, project } = await runProject(files, {
- appUrl: app.url,
runOptions: { maxFailures: 2, onEvent: (event) => { if (event.type === 'run-stopped') events.push(`${event.failures}/${event.limit}`); } },
});
expect(outcome.exitCode).toBe(1);
@@ -814,7 +689,7 @@ test('never runs either', async () => { throw new Error('three'); });
expect(byDeclaration[2]!.skip?.reason).toBe('run stopped after 2 failures (--max-failures 2)');
// The SDK path is bounded like the flag: a count below one is a configuration error, not a run that stops at once.
- const invalid = await runExisting(project, { appUrl: app.url, runOptions: { maxFailures: 0 } });
+ const invalid = await runExisting(project, { runOptions: { maxFailures: 0 } });
expect(invalid.exitCode).toBe(2);
expect(invalid.report.run.errors.map((error) => [error.code, error.message])).toEqual([
['INVALID_CONFIG', 'maxFailures must be a positive safe integer, got 0'],
@@ -840,7 +715,6 @@ test('after the group too', async () => {});
};
let planned = 0;
const { outcome, project } = await runProject(files, {
- appUrl: app.url,
runOptions: { maxFailures: 1, onEvent: (event) => { if (event.type === 'plan') planned = event.total; } },
});
expect(outcome.exitCode).toBe(1);
@@ -875,11 +749,11 @@ test('sleeps until interrupted', async () => {
test('never started', async () => {});
test('never started either', async () => {});
`,
+ 'e2e.config.ts': workerFakeConfigSource(1),
});
const controller = new AbortController();
let planned = 0;
- const interrupted = await runExisting(project, {
- appUrl: app.url,
+ const interrupted = await runExistingWithConfigFile(project, {
runOptions: {
interruptSignal: controller.signal,
onEvent: (event) => {
@@ -903,7 +777,7 @@ test('never started either', async () => {});
// The rerun reads the report the interrupt wrote; the body that slept passes at once now.
writeFileSync(path.join(project.dir, 'tests', 'first.e2e.ts'), sleeping.replace('setTimeout(resolve, 60_000)', 'setTimeout(resolve, 0)'));
- const rerun = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
+ const rerun = await runExistingWithConfigFile(project, { runOptions: { lastFailed: true } });
expect(rerun.exitCode).toBe(0);
const byFile = rerun.results.toSorted((a, b) => (a.test.file === b.test.file ? a.test.declarationIndex - b.test.declarationIndex : a.test.file < b.test.file ? -1 : 1));
expect(byFile.map((result) => [result.test.title, result.selected, result.status])).toEqual([
@@ -919,17 +793,18 @@ test('never started either', async () => {});
it(
'an interrupt during a serial group retry keeps each member\'s verdict from the attempt before it',
async () => {
- const project = createProject({});
- const marker = path.join(project.dir, 'failed-once');
- mkdirSync(path.join(project.dir, 'tests'), { recursive: true });
- writeFileSync(
- path.join(project.dir, 'tests', 'flow.e2e.ts'),
- `import { existsSync, writeFileSync } from 'node:fs';
+ const controller = new AbortController();
+ let starts = 0;
+ const startedAt = Date.now();
+ const { outcome, project } = await runProjectWithConfigFile(
+ {
+ 'tests/flow.e2e.ts': `import { existsSync, writeFileSync } from 'node:fs';
import { test } from 'e2e';
+const marker = new URL('./failed-once', import.meta.url);
test.describe('flow', { serial: true }, () => {
test('first step', async () => {
- if (!existsSync(${JSON.stringify(marker)})) {
- writeFileSync(${JSON.stringify(marker)}, '');
+ if (!existsSync(marker)) {
+ writeFileSync(marker, '');
throw new Error('first attempt fails');
}
await new Promise((resolve) => setTimeout(resolve, 60_000));
@@ -937,21 +812,20 @@ test.describe('flow', { serial: true }, () => {
test('second step', async () => {});
});
`,
- );
- const controller = new AbortController();
- let starts = 0;
- const outcome = await runExisting(project, {
- appUrl: app.url,
- config: { tests: 'tests/**/*.e2e.ts', retries: 1 },
- runOptions: {
- interruptSignal: controller.signal,
- onEvent: (event) => {
- // A serial member starts once per group attempt: the second start is the retry.
- if (event.type === 'test-started' && event.title === 'flow > first step' && (starts += 1) === 2) controller.abort();
+ },
+ {
+ configSource: workerFakeConfigSource(1, `\n tests: 'tests/**/*.e2e.ts',\n retries: 1,`),
+ runOptions: {
+ interruptSignal: controller.signal,
+ onEvent: (event) => {
+ // A serial member starts once per group attempt: the second start is the retry.
+ if (event.type === 'test-started' && event.title === 'flow > first step' && (starts += 1) === 2) controller.abort();
+ },
},
},
- });
+ );
expect(outcome.exitCode).toBe(130);
+ expect(Date.now() - startedAt).toBeLessThan(30_000);
expect(outcome.report.run.serialGroups[0]?.attempts.map((attempt) => attempt.status)).toEqual(['failed', 'interrupted']);
expect(resultByTitle(outcome, 'first step').status).toBe('failed');
expect(outcome.report.run.summary).toMatchObject({ failed: 1, interrupted: 0 });
@@ -960,7 +834,6 @@ test.describe('flow', { serial: true }, () => {
},
120_000,
);
-
it(
'a forced interrupt still writes the junit and markdown files, so none of the previous run is left beside the report',
async () => {
@@ -1013,7 +886,6 @@ test('queued behind it too', async () => {});
`,
});
const aborted = await runExisting(project, {
- appUrl: app.url,
config: { tests: 'tests/**/*.e2e.ts', workers: 1 },
});
expect(aborted.exitCode).toBe(2);
@@ -1038,7 +910,6 @@ test('needs the model', async ({ app }) => {
`,
);
const rerun = await runExisting(project, {
- appUrl: app.url,
config: { tests: 'tests/**/*.e2e.ts', workers: 1 },
runOptions: { lastFailed: true },
});
@@ -1066,14 +937,14 @@ test('fails', async () => {
test('also passes', async () => {});
`,
};
- const fresh = await runProject(files, { appUrl: app.url, runOptions: { lastFailed: true } });
+ const fresh = await runProject(files, { runOptions: { lastFailed: true } });
expect(fresh.outcome.exitCode).toBe(2);
expect(fresh.outcome.report.run.errors.map((error) => [error.code, error.phase])).toEqual([['NO_LAST_RUN', 'collection']]);
expect(fresh.outcome.report.run.errors[0]!.message).toContain(path.join(fresh.project.dir, '.e2e', 'report.json'));
- const first = await runExisting(fresh.project, { appUrl: app.url });
+ const first = await runExisting(fresh.project, {});
expect(first.exitCode).toBe(1);
- const second = await runExisting(fresh.project, { appUrl: app.url, runOptions: { lastFailed: true } });
+ const second = await runExisting(fresh.project, { runOptions: { lastFailed: true } });
expect(second.exitCode).toBe(1);
const byTitle = second.results.toSorted((a, b) => a.test.declarationIndex - b.test.declarationIndex);
expect(byTitle.map((result) => [result.test.title, result.selected, result.status])).toEqual([
@@ -1115,25 +986,25 @@ test('fails', async () => {
const hookErrors = (outcome: RunOutcome) =>
outcome.report.run.errors.map((error) => [error.code, error.phase, error.scopeId, error.scope]);
- const first = await runExisting(project, { appUrl: app.url });
+ const first = await runExisting(project, {});
expect(first.exitCode).toBe(1);
- expect(hookErrors(first)).toEqual([['HOOK_FAILED', 'afterAll', 'teardown', { file: 'tests/hooks.e2e.ts', targetId: 'web', titlePath: ['teardown'] }]]);
+ expect(hookErrors(first)).toEqual([['HOOK_FAILED', 'afterAll', 'teardown', { file: 'tests/hooks.e2e.ts', targetId: 'fake', titlePath: ['teardown'] }]]);
assertValidReport(first.report);
// The body is fixed, the hook is not: the scope runs again and still fails the run.
writeFileSync(path.join(project.dir, 'tests', 'body.e2e.ts'), brokenBody.replace(`throw new Error('still broken');`, ''));
- const second = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
+ const second = await runExisting(project, { runOptions: { lastFailed: true } });
expect(second.exitCode).toBe(1);
expect(selection(second)).toEqual([
['fails', true, 'passed'],
['passes in the scope', true, 'passed'],
['passes outside it', false, 'skipped'],
]);
- expect(hookErrors(second)).toEqual([['HOOK_FAILED', 'afterAll', 'teardown', { file: 'tests/hooks.e2e.ts', targetId: 'web', titlePath: ['teardown'] }]]);
+ expect(hookErrors(second)).toEqual([['HOOK_FAILED', 'afterAll', 'teardown', { file: 'tests/hooks.e2e.ts', targetId: 'fake', titlePath: ['teardown'] }]]);
// The hook is fixed: its scope runs once more and the error is gone, not carried.
writeFileSync(path.join(project.dir, 'tests', 'hooks.e2e.ts'), hooks(''));
- const third = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
+ const third = await runExisting(project, { runOptions: { lastFailed: true } });
expect(third.exitCode).toBe(0);
expect(selection(third)).toEqual([
['fails', false, 'skipped'],
@@ -1141,7 +1012,7 @@ test('fails', async () => {
['passes outside it', false, 'skipped'],
]);
expect(third.report.run.errors).toEqual([]);
- const fourth = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
+ const fourth = await runExisting(project, { runOptions: { lastFailed: true } });
expect(fourth.exitCode).toBe(2);
expect(fourth.report.run.errors.map((error) => error.code)).toEqual(['NO_TESTS']);
project.cleanup();
@@ -1162,7 +1033,7 @@ test('${title}', async () => {
const selection = (outcome: RunOutcome) =>
outcome.results.toSorted((a, b) => (a.test.file < b.test.file ? -1 : 1)).map((result) => [result.test.title, result.selected, result.status, result.skip?.cause]);
- const limited = await runExisting(project, { appUrl: app.url, config, runOptions: { maxFailures: 1 } });
+ const limited = await runExisting(project, { config, runOptions: { maxFailures: 1 } });
expect(limited.exitCode).toBe(1);
expect(selection(limited)).toEqual([
['first', true, 'failed', undefined],
@@ -1171,7 +1042,7 @@ test('${title}', async () => {
// The failure limit stopped `second` before it ran, so it is owed; a rerun narrowed to `first` carries it.
writeFileSync(path.join(project.dir, 'tests', 'first.e2e.ts'), `import { test } from 'e2e';\ntest('first', async () => {});\n`);
- const rerun = await runExisting(project, { appUrl: app.url, config, runOptions: { lastFailed: true, grep: [/first/] } });
+ const rerun = await runExisting(project, { config, runOptions: { lastFailed: true, grep: [/first/] } });
expect(rerun.exitCode).toBe(0);
expect(selection(rerun)).toEqual([
['first', true, 'passed', undefined],
@@ -1181,7 +1052,7 @@ test('${title}', async () => {
assertValidReport(rerun.report);
// Left out by --grep, `second` is still owed: the next --last-failed runs it, and only it.
- const owed = await runExisting(project, { appUrl: app.url, config, runOptions: { lastFailed: true } });
+ const owed = await runExisting(project, { config, runOptions: { lastFailed: true } });
expect(owed.exitCode).toBe(1);
expect(selection(owed)).toEqual([
['first', false, 'skipped', 'filtered'],
@@ -1193,165 +1064,6 @@ test('${title}', async () => {
120_000,
);
- it(
- '--last-failed narrowed past a file that fails to import keeps owing its tests until a run carries them out',
- async () => {
- const failing = (title: string) => `import { test } from 'e2e';
-test('${title}', async () => {
- throw new Error('${title} is broken');
-});
-`;
- const project = createProject({ 'tests/first.e2e.ts': failing('first'), 'tests/second.e2e.ts': failing('second') });
- const first = await runExisting(project, { appUrl: app.url });
- expect(first.exitCode).toBe(1);
-
- // `second` is owed and its file no longer imports; a rerun narrowed to `first` cannot collect it, so it carries it.
- writeFileSync(path.join(project.dir, 'tests', 'first.e2e.ts'), `import { test } from 'e2e';\ntest('first', async () => {});\n`);
- writeFileSync(path.join(project.dir, 'tests', 'second.e2e.ts'), `import './not-written-yet.ts';\n${failing('second')}`);
- const narrowed = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true, files: ['tests/first.e2e.ts'] } });
- expect(narrowed.exitCode).toBe(0);
- expect(narrowed.results.map((result) => [result.test.title, result.status])).toEqual([['first', 'passed']]);
- expect(narrowed.report.run.carried?.results.map((result) => [result.titlePath.at(-1), result.status])).toEqual([['second', 'failed']]);
- assertValidReport(narrowed.report);
-
- // Once the file imports again, the next --last-failed runs `second`, and only it.
- writeFileSync(path.join(project.dir, 'tests', 'second.e2e.ts'), `import { test } from 'e2e';\ntest('second', async () => {});\n`);
- const owed = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
- expect(owed.exitCode).toBe(0);
- expect(owed.results.filter((result) => result.selected).map((result) => result.test.title)).toEqual(['second']);
- expect(owed.report.run.carried).toBeUndefined();
- project.cleanup();
- },
- 120_000,
- );
-
- it(
- '--last-failed carries a failed hook another filter kept from running again, until its scope runs',
- async () => {
- const hooks = (teardown: string) => `import { test } from 'e2e';
-test.afterAll(() => {
- ${teardown}
-});
-test('in the hook scope', async () => {});
-`;
- const project = createProject({
- 'tests/hooks.e2e.ts': hooks(`throw new Error('teardown broke');`),
- 'tests/body.e2e.ts': `import { test } from 'e2e';\ntest('body', async () => {\n throw new Error('still broken');\n});\n`,
- });
- const ran = (outcome: RunOutcome) => outcome.results.filter((result) => result.selected).map((result) => [result.test.title, result.status]);
- const hookScope = { file: 'tests/hooks.e2e.ts', targetId: 'web', titlePath: [] };
-
- const first = await runExisting(project, { appUrl: app.url });
- expect(first.exitCode).toBe(1);
-
- writeFileSync(path.join(project.dir, 'tests', 'body.e2e.ts'), `import { test } from 'e2e';\ntest('body', async () => {});\n`);
- const narrowed = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true, grep: [/body/] } });
- expect(narrowed.exitCode).toBe(0);
- expect(ran(narrowed)).toEqual([['body', 'passed']]);
- expect(narrowed.report.run.errors).toEqual([]);
- expect(narrowed.report.run.carried?.results.map((result) => result.titlePath.at(-1))).toEqual(['in the hook scope']);
- expect(narrowed.report.run.carried?.errors.map((error) => [error.code, error.phase, error.scope])).toEqual([['HOOK_FAILED', 'afterAll', hookScope]]);
- assertValidReport(narrowed.report);
-
- // The scope runs again and the hook still fails: reported by the run itself, carried no more.
- const again = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
- expect(again.exitCode).toBe(1);
- expect(ran(again)).toEqual([['in the hook scope', 'passed']]);
- expect(again.report.run.errors.map((error) => [error.code, error.scope])).toEqual([['HOOK_FAILED', hookScope]]);
- expect(again.report.run.carried).toBeUndefined();
-
- writeFileSync(path.join(project.dir, 'tests', 'hooks.e2e.ts'), hooks(''));
- const fixed = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
- expect(fixed.exitCode).toBe(0);
- expect(ran(fixed)).toEqual([['in the hook scope', 'passed']]);
- expect(fixed.report.run.carried).toBeUndefined();
- project.cleanup();
- },
- 120_000,
- );
-
- it(
- '--last-failed runs a setup whose afterAll failed again through the tests that consume its session',
- async () => {
- const auth = (teardown: string) => `import { test } from 'e2e';
-test.afterAll(() => {
- ${teardown}
-});
-test.setup('sign in', { sessions: ['member'] }, async ({ app, session }) => {
- await app.open();
- await session.save('member');
-});
-`;
- const project = createProject({
- 'tests/auth.e2e.ts': auth(`throw new Error('sign out broke');`),
- 'tests/member.e2e.ts': `import { test } from 'e2e';
-test('signed in', { session: 'member' }, async () => {});
-`,
- 'tests/other.e2e.ts': `import { test } from 'e2e';
-test('unrelated', async () => {
- throw new Error('still broken');
-});
-`,
- });
- const ran = (outcome: RunOutcome) =>
- outcome.results.filter((result) => result.selected).map((result) => [result.test.title, result.status]).toSorted();
-
- const first = await runExisting(project, { appUrl: app.url });
- expect(first.exitCode).toBe(1);
- expect(first.report.run.errors.map((error) => [error.code, error.phase, error.scope])).toEqual([
- ['HOOK_FAILED', 'afterAll', { file: 'tests/auth.e2e.ts', targetId: 'web', titlePath: [] }],
- ]);
-
- writeFileSync(path.join(project.dir, 'tests', 'other.e2e.ts'), `import { test } from 'e2e';
-test('unrelated', async () => {});
-`);
- // A rerun that needs none of the setup's sessions carries its failed afterAll, with no row for the setup.
- const narrowed = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true, grep: [/unrelated/] } });
- expect(narrowed.exitCode).toBe(0);
- expect(ran(narrowed)).toEqual([['unrelated', 'passed']]);
- expect(narrowed.report.run.carried?.results.map((result) => [result.kind, result.titlePath.at(-1)])).toEqual([['setup', 'sign in']]);
- expect(narrowed.report.run.carried?.errors.map((error) => error.code)).toEqual(['HOOK_FAILED']);
- assertValidReport(narrowed.report);
-
- const second = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
- expect(second.exitCode).toBe(1);
- expect(ran(second)).toEqual([
- ['sign in', 'passed'],
- ['signed in', 'passed'],
- ]);
- expect(second.report.run.errors.map((error) => error.code)).toEqual(['HOOK_FAILED']);
-
- writeFileSync(path.join(project.dir, 'tests', 'auth.e2e.ts'), auth(''));
- const third = await runExisting(project, { appUrl: app.url, runOptions: { lastFailed: true } });
- expect(third.exitCode).toBe(0);
- expect(ran(third)).toEqual([
- ['sign in', 'passed'],
- ['signed in', 'passed'],
- ]);
- expect(third.report.run.errors).toEqual([]);
- project.cleanup();
- },
- 120_000,
- );
-
- it(
- 'fails with NO_TESTS unless --pass-with-no-tests',
- async () => {
- const empty = { 'tests/empty.txt': 'not a test' };
- const first = await runProject(empty, { appUrl: app.url });
- expect(first.outcome.exitCode).toBe(2);
- first.project.cleanup();
-
- const second = await runProject(empty, {
- appUrl: app.url,
- runOptions: { passWithNoTests: true },
- });
- expect(second.outcome.exitCode).toBe(0);
- second.project.cleanup();
- },
- 120_000,
- );
-
it(
'selects positional directories and globs, and names the ones that matched nothing',
async () => {
@@ -1369,14 +1081,13 @@ test('unrelated', async () => {});
const unselected = (outcome: RunOutcome) =>
outcome.results.filter((result) => !result.selected).map((result) => result.test.title).toSorted();
- const directory = await runProject(files, { appUrl: app.url, runOptions: { files: ['tests/agent'] } });
+ const directory = await runProject(files, { runOptions: { files: ['tests/agent'] } });
expect(directory.outcome.exitCode).toBe(0);
expect(titles(directory.outcome)).toEqual(['agent one', 'agent two']);
expect(unselected(directory.outcome)).toEqual(['other three', 'top']);
directory.project.cleanup();
const glob = await runProject(files, {
- appUrl: app.url,
runOptions: { files: ['tests/*/*.e2e.ts', 'tests/top.e2e.ts'] },
});
expect(glob.outcome.exitCode).toBe(0);
@@ -1384,7 +1095,6 @@ test('unrelated', async () => {});
glob.project.cleanup();
const missing = await runProject(files, {
- appUrl: app.url,
runOptions: { files: ['tests/agnet', 'tests/*.spec.ts'] },
});
expect(missing.outcome.exitCode).toBe(2);
@@ -1400,70 +1110,6 @@ test('unrelated', async () => {});
},
120_000,
);
- it(
- 'closes a scope when its last test finishes and keeps same-titled siblings apart',
- async () => {
- const file = `import { appendFileSync } from 'node:fs';
-import { test } from 'e2e';
-
-const log = (entry: string) => appendFileSync(process.env.HOOK_LOG!, entry + '\\n');
-
-test.beforeAll(() => log('beforeAll:file'));
-test.afterAll(() => log('afterAll:file'));
-
-test.describe('A', () => {
- test.beforeAll(() => log('beforeAll:A'));
- test.afterAll(() => log('afterAll:A'));
- test.beforeEach(() => log('beforeEach:A'));
- test('a1', async () => { log('body:a1'); });
- test('a2', async () => { log('body:a2'); });
-});
-
-test.describe('B', () => {
- test.beforeAll(() => log('beforeAll:B'));
- test.afterAll(() => log('afterAll:B'));
- test('b1', async () => { log('body:b1'); });
-});
-
-// A second group titled "A" is a scope of its own, not a re-entry of the first.
-test.describe('A', () => {
- test.beforeAll(() => log('beforeAll:A2'));
- test.afterAll(() => log('afterAll:A2'));
- test.beforeEach(() => log('beforeEach:A2'));
- test('a3', async () => { log('body:a3'); });
-});
-
-test('top', async () => { log('body:top'); });
-`;
- const logPath = path.join('/tmp', `e2e-scopes-${Date.now()}.log`);
- process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/scopes.e2e.ts': file }, { appUrl: app.url });
- expect(outcome.exitCode).toBe(0);
- // A's teardown runs once a2 is done, before B enters; the file scope
- // stays open across all of them and closes with the realm.
- expect(readFileSync(logPath, 'utf8').trim().split('\n')).toEqual([
- 'beforeAll:file',
- 'beforeAll:A',
- 'beforeEach:A',
- 'body:a1',
- 'beforeEach:A',
- 'body:a2',
- 'afterAll:A',
- 'beforeAll:B',
- 'body:b1',
- 'afterAll:B',
- 'beforeAll:A2',
- 'beforeEach:A2',
- 'body:a3',
- 'afterAll:A2',
- 'body:top',
- 'afterAll:file',
- ]);
- project.cleanup();
- },
- 120_000,
- );
-
it(
'discards the realm after an afterAll failure so later tests start fresh',
async () => {
@@ -1487,7 +1133,7 @@ test('later', async () => { log('body:later'); });
`;
const logPath = path.join('/tmp', `e2e-afterall-fail-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/afterallfail.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/afterallfail.e2e.ts': file }, {});
expect(resultByTitle(outcome, 'first').status).toBe('passed');
expect(resultByTitle(outcome, 'later').status).toBe('passed');
// The failed teardown ends that suite instance: the file scope closes
@@ -1525,7 +1171,7 @@ test.describe('wizard', { serial: true, retries: 1 }, () => {
test('step 2', async () => {});
});
`;
- const { outcome, project } = await runProject({ 'tests/serialafterall.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/serialafterall.e2e.ts': file }, {});
const skipped = resultByTitle(outcome, 'step 2');
expect(skipped.status).toBe('skipped');
expect(skipped.skip?.cause).toBe('hook-failed');
@@ -1566,7 +1212,7 @@ test.describe('wizard', { serial: true }, () => {
`;
const logPath = path.join('/tmp', `e2e-serialhooks-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/serialhooks.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/serialhooks.e2e.ts': file }, {});
expect(outcome.exitCode).toBe(0);
expect(readFileSync(logPath, 'utf8').trim().split('\n')).toEqual([
'beforeAll:inner',
@@ -1605,7 +1251,7 @@ test('never runs', async () => { log('body'); });
`;
const logPath = path.join('/tmp', `e2e-beforeeach-${Date.now()}.log`);
process.env['HOOK_LOG'] = logPath;
- const { outcome, project } = await runProject({ 'tests/beforeeach.e2e.ts': file }, { appUrl: app.url });
+ const { outcome, project } = await runProject({ 'tests/beforeeach.e2e.ts': file }, {});
const result = resultByTitle(outcome, 'never runs');
expect(result.status).toBe('failed');
expect(result.attempts[0]!.error?.phase).toBe('beforeEach');
@@ -1701,7 +1347,7 @@ test('passes', async ({ app }) => { await app.open(); });
process.env['HOOK_LOG'] = logPath;
const { outcome, project } = await runProject(
{ 'tests/overrun.e2e.ts': file },
- { appUrl: app.url, config: { cleanupTimeout: 1000 } },
+ { config: { cleanupTimeout: 1000 } },
);
const result = resultByTitle(outcome, 'passes');
expect(result.status).toBe('timed-out');
@@ -1713,35 +1359,4 @@ test('passes', async ({ app }) => { await app.open(); });
120_000,
);
- it(
- 'times out suite hooks against their budgets and names the scope in run errors',
- async () => {
- const file = `import { test } from 'e2e';
-
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-test.afterAll(() => sleep(5000));
-
-test.describe('slow scope', () => {
- test.beforeAll(() => sleep(5000));
- test('unreachable', async () => {});
-});
-`;
- const { outcome, project } = await runProject(
- { 'tests/hooktimeouts.e2e.ts': file },
- { appUrl: app.url, config: { timeout: 1000, cleanupTimeout: 1000 } },
- );
- const skipped = resultByTitle(outcome, 'unreachable');
- expect(skipped.status).toBe('skipped');
- expect(skipped.skip?.cause).toBe('hook-failed');
- expect(outcome.report.run.errors.map((error) => [error.phase, error.scopeId, error.code])).toEqual([
- ['beforeAll', 'slow scope', 'HOOK_FAILED'],
- ['afterAll', 'file', 'HOOK_FAILED'],
- ]);
- assertValidReport(outcome.report);
- expect(outcome.exitCode).toBe(1);
- project.cleanup();
- },
- 120_000,
- );
});
diff --git a/packages/e2e/tests/integration/scripted-actions.test.ts b/packages/e2e/tests/integration/scripted-actions.test.ts
index b822a29c7..5b9fa0ea2 100644
--- a/packages/e2e/tests/integration/scripted-actions.test.ts
+++ b/packages/e2e/tests/integration/scripted-actions.test.ts
@@ -1,9 +1,7 @@
/**
- * Node actions, pointer actions, the keyboard, and the app hooks through the
- * real runner against the scripted engine: every action kind reaches the
- * engine with the ref and payload the test named, every pointer action lands
- * at its point, a malformed key is refused before resolving, an engine
- * refusal maps to ACTION_FAILED, and `app.open`, `back`, `restart`, and
+ * Node actions, the keyboard, and the app hooks through the real runner
+ * against the scripted engine: every action kind reaches the engine with the
+ * ref and payload the test named, and `app.open`, `back`, `restart`, and
* `clearState` reach the session hooks.
*/
@@ -11,8 +9,7 @@ import path from 'node:path';
import { describe, expect, it } from 'vitest';
import { FAKE_APP_URL, type RecordedSessionCall } from '../helpers/fake-engine.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
-import { SCENE_ROOT_ID } from '../helpers/scripted-scene.ts';
-import { failed, passed, scriptedSuite, step } from '../helpers/scripted-suite.ts';
+import { passed, scriptedSuite, step } from '../helpers/scripted-suite.ts';
const ACTIONS = `import { test, expect } from 'e2e';
@@ -68,26 +65,6 @@ test('every node action reaches the engine', async ({ app, screen }) => {
await expect(screen.getByRole('button', { name: 'Bold', pressed: true })).toBeVisible();
});
-test('every pointer action reaches the engine', async ({ app, screen }) => {
- await app.open('/');
- await screen.tapAt({ x: 5, y: 6 });
- await screen.getByRole('image', { name: 'Map' }).tap({ position: { x: 10, y: 20 } });
- await screen.getByRole('image', { name: 'Map' }).click({ position: { x: 0, y: 0 } });
- await screen.swipe({ from: { x: 1, y: 2 }, to: { x: 3, y: 4 } });
- await screen.swipe({ direction: 'up', momentum: 'fast' });
- await screen.swipe({ direction: 'down' });
-});
-
-test('a malformed key is INVALID_ARGUMENT', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByLabel('City').press('Bogus');
-});
-
-test('an unknown option is ACTION_FAILED', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByLabel('Plan').selectOption('Enterprise');
-});
-
test('app hooks reach the session', async ({ app, screen }) => {
await app.open('/');
await app.open('/settings');
@@ -98,7 +75,7 @@ test('app hooks reach the session', async ({ app, screen }) => {
});
`;
-describe('scripted engine: actions, pointer actions, and app hooks', () => {
+describe('scripted engine: actions and app hooks', () => {
const run = scriptedSuite('actions.e2e.ts', ACTIONS);
it('dispatches every action kind with the expected ref and payload', () => {
@@ -157,33 +134,6 @@ describe('scripted engine: actions, pointer actions, and app hooks', () => {
expect(step(attempt, 'locator.pressSequentially').label).toBe('getByLabel("City")');
});
- it('dispatches every pointer action at the point the test named', () => {
- const attempt = passed(run.outcome, 'every pointer action reaches the engine');
- expect(run.fake.callsOf(attempt.id, 'performAt').map((entry) => [entry.point, entry.action])).toEqual([
- [{ x: 5, y: 6 }, { kind: 'tap' }],
- [{ x: 110, y: 320 }, { kind: 'tap' }],
- [{ x: 100, y: 300 }, { kind: 'tap' }],
- [{ x: 1, y: 2 }, { kind: 'swipeTo', target: { x: 3, y: 4 } }],
- ]);
- // A positioned tap scrolls the node into view first; a direction swipe is the root swipe.
- expect(run.fake.callsOf(attempt.id, 'perform').map((entry) => [entry.id, entry.action])).toEqual([
- ['map', { kind: 'scrollIntoView' }],
- ['map', { kind: 'scrollIntoView' }],
- [SCENE_ROOT_ID, { kind: 'swipe', direction: 'up', momentum: 'fast' }],
- [SCENE_ROOT_ID, { kind: 'swipe', direction: 'down' }],
- ]);
- expect(step(attempt, 'screen.tapAt').label).toBe('(5, 6)');
- expect(step(attempt, 'screen.swipe').label).toBe('(1, 2) → (3, 4)');
- expect(step(attempt, 'locator.tap').label).toBe('getByRole("image", name: "Map") at (10, 20)');
- });
-
- it('refuses a malformed key before resolving and maps an engine refusal to ACTION_FAILED', () => {
- const malformed = failed(run.outcome, 'a malformed key is INVALID_ARGUMENT', 'INVALID_ARGUMENT', 'press key "Bogus" is not a key');
- expect(run.fake.callsOf(malformed.id, 'perform')).toEqual([]);
- const refused = failed(run.outcome, 'an unknown option is ACTION_FAILED', 'ACTION_FAILED', 'no option matches "Enterprise"');
- expect(run.fake.callsOf(refused.id, 'perform')).toHaveLength(1);
- });
-
it('routes app.open, back, restart, and clearState to the session hooks', () => {
const attempt = passed(run.outcome, 'app hooks reach the session');
const session = (method: RecordedSessionCall['method'], url?: string): RecordedSessionCall => ({
diff --git a/packages/e2e/tests/integration/scripted-capabilities.test.ts b/packages/e2e/tests/integration/scripted-capabilities.test.ts
deleted file mode 100644
index 0a6018daa..000000000
--- a/packages/e2e/tests/integration/scripted-capabilities.test.ts
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Undeclared capabilities through the real runner against a scripted engine
- * that declares only `tap` and `focus`, the `tap` pointer kind, and no
- * keyboard: a declared action still works, a positioned tap skips the
- * `scrollIntoView` the engine lacks, and every undeclared kind fails with
- * UNSUPPORTED_CAPABILITY before reaching the engine.
- */
-
-import { describe, expect, it } from 'vitest';
-import { assertValidReport } from '../helpers/report-schema.ts';
-import { failed, passed, scriptedSuite } from '../helpers/scripted-suite.ts';
-
-const LIMITED = `import { test } from 'e2e';
-
-test('a declared action still works', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('button', { name: 'Submit' }).tap();
-});
-
-test('a positioned tap skips scrollIntoView when the engine lacks it', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('image', { name: 'Map' }).tap({ position: { x: 1, y: 2 } });
-});
-
-test('an undeclared node action is UNSUPPORTED_CAPABILITY', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('button', { name: 'Submit' }).hover();
-});
-
-test('an undeclared pointer action is UNSUPPORTED_CAPABILITY', async ({ app, screen }) => {
- await app.open('/');
- await screen.swipe({ from: { x: 1, y: 2 }, to: { x: 3, y: 4 } });
-});
-
-test('a direction swipe needs the swipe action', async ({ app, screen }) => {
- await app.open('/');
- await screen.swipe({ direction: 'down' });
-});
-
-test('pressSequentially needs a keyboard', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByLabel('City').pressSequentially('Wa');
-});
-`;
-
-describe('scripted engine: undeclared capabilities', () => {
- const run = scriptedSuite('limited.e2e.ts', LIMITED, { actions: ['tap', 'focus'], pointerActions: ['tap'], keyboard: false });
-
- it('honors what the engine declares', () => {
- assertValidReport(run.outcome.report);
- const tapped = passed(run.outcome, 'a declared action still works');
- expect(run.fake.callsOf(tapped.id, 'perform').map((entry) => entry.action.kind)).toEqual(['tap']);
- const positioned = passed(run.outcome, 'a positioned tap skips scrollIntoView when the engine lacks it');
- expect(run.fake.callsOf(positioned.id, 'perform')).toEqual([]);
- expect(run.fake.callsOf(positioned.id, 'performAt').map((entry) => entry.point)).toEqual([{ x: 101, y: 302 }]);
- });
-
- it('fails an undeclared kind with UNSUPPORTED_CAPABILITY before reaching the engine', () => {
- const cases: [string, string][] = [
- ['an undeclared node action is UNSUPPORTED_CAPABILITY', 'the "hover" action is not available on this target: its engine declares tap, focus'],
- ['an undeclared pointer action is UNSUPPORTED_CAPABILITY', 'the "swipeTo" action at a point is not available on this target: its engine declares tap'],
- ['a direction swipe needs the swipe action', 'target "fake" has no engine capability for swipe gestures: engine fake does not implement it'],
- ['pressSequentially needs a keyboard', 'pressSequentially is not available on this target: its engine declares no keyboard'],
- ];
- for (const [title, message] of cases) {
- const attempt = failed(run.outcome, title, 'UNSUPPORTED_CAPABILITY', message);
- expect(attempt.error?.category, title).toBe('configuration');
- expect(run.fake.callsOf(attempt.id, 'perform'), title).toEqual([]);
- expect(run.fake.callsOf(attempt.id, 'performAt'), title).toEqual([]);
- }
- });
-});
diff --git a/packages/e2e/tests/integration/scripted-matchers.test.ts b/packages/e2e/tests/integration/scripted-matchers.test.ts
index 89f03a8ec..3b3592cd6 100644
--- a/packages/e2e/tests/integration/scripted-matchers.test.ts
+++ b/packages/e2e/tests/integration/scripted-matchers.test.ts
@@ -1,6 +1,6 @@
/**
* `expect(locator)` through the real runner against the scripted engine:
- * every matcher and its negation pass on the reference screen, every failure
+ * every matcher and a few negations pass on the reference screen, every failure
* names the observed state, a matcher honors its timeout, and an ambiguous
* locator fails at once.
*/
@@ -44,26 +44,13 @@ test('every matcher passes on the scripted screen', async ({ app, screen }) => {
await expect(screen.getByTestId('missing')).toHaveCount(0);
});
-test('every negated matcher passes on the scripted screen', async ({ app, screen }) => {
+test('negated matchers pass on the scripted screen', async ({ app, screen }) => {
await app.open('/');
// Half a second: a negation holds for the smaller of the grace window and the
// budget, and the first observation must land inside the budget on a loaded CI box.
const soon = { timeout: 500 };
await expect(screen.getByText('Hidden content')).not.toBeVisible(soon);
- await expect(screen.getByRole('heading', { name: 'Dashboard' })).not.toBeHidden(soon);
- await expect(screen.getByText('Never rendered')).not.toBeAttached(soon);
- await expect(screen.getByRole('button', { name: 'Disabled action' })).not.toBeEnabled(soon);
- await expect(screen.getByRole('button', { name: 'Submit' })).not.toBeDisabled(soon);
- await expect(screen.getByRole('checkbox', { name: 'Notifications' })).not.toBeChecked(soon);
- await expect(screen.getByRole('tab', { name: 'Open' })).not.toBeSelected(soon);
- await expect(screen.getByRole('button', { name: 'Menu' })).not.toBeExpanded(soon);
- await expect(screen.getByLabel('Email')).not.toBeFocused(soon);
- await expect(screen.getByRole('heading', { level: 1 })).not.toHaveText('Recent', soon);
- await expect(screen.getByTestId('card')).not.toContainText('footer', soon);
await expect(screen.getByLabel('Notes')).not.toHaveValue('line1 line2', soon);
- await expect(screen.getByTestId('card')).not.toHaveAttribute('hidden', soon);
- await expect(screen.getByTestId('card')).not.toHaveAttribute('class', 'card inactive', soon);
- await expect(screen.getByRole('list', { name: 'Todos' })).not.toHaveAccessibleName('Done', soon);
// The third todo arrives on the scene's clock; wait for it so the negation is about the settled count.
await expect(screen.getByTestId('todo')).toHaveCount(3);
await expect(screen.getByTestId('todo')).not.toHaveCount(2, soon);
@@ -225,7 +212,7 @@ const SECURE_DENIALS = [
const TITLES = [
'every matcher passes on the scripted screen',
- 'every negated matcher passes on the scripted screen',
+ 'negated matchers pass on the scripted screen',
...FAILURES.map(([title]) => title),
'a matcher honors its timeout',
'an ambiguous locator fails an assertion at once',
@@ -235,12 +222,9 @@ const TITLES = [
describe('scripted engine: expect(locator) matchers', () => {
const run = scriptedSuite('matchers.e2e.ts', MATCHERS);
- it('produces a valid report with one result per test in the file', () => {
+ it('passes every matcher and its negation, in a valid report with one result per test', () => {
assertValidReport(run.outcome.report);
expect(run.outcome.results.map((result) => result.test.title).toSorted()).toEqual(TITLES.toSorted());
- });
-
- it('passes every matcher and its negation', () => {
const attempt = passed(run.outcome, 'every matcher passes on the scripted screen');
const apis = new Set(attempt.steps.map((entry) => entry.api));
for (const matcher of [
@@ -251,8 +235,8 @@ describe('scripted engine: expect(locator) matchers', () => {
expect(apis, matcher).toContain(`expect.${matcher}`);
}
expect(attempt.steps.every((entry) => entry.status === 'passed')).toBe(true);
- const negated = passed(run.outcome, 'every negated matcher passes on the scripted screen');
- expect(negated.steps.filter((entry) => entry.api.startsWith('expect.not.'))).toHaveLength(17);
+ const negated = passed(run.outcome, 'negated matchers pass on the scripted screen');
+ expect(negated.steps.filter((entry) => entry.api.startsWith('expect.not.'))).toHaveLength(4);
expect(step(negated, 'expect.not.toBeVisible')).toMatchObject({ kind: 'assertion', label: 'getByText("Hidden content")' });
});
diff --git a/packages/e2e/tests/integration/scripted-polling.test.ts b/packages/e2e/tests/integration/scripted-polling.test.ts
index 62bab05e7..63761dcf8 100644
--- a/packages/e2e/tests/integration/scripted-polling.test.ts
+++ b/packages/e2e/tests/integration/scripted-polling.test.ts
@@ -1,16 +1,11 @@
/**
- * Polling, staleness, failure mapping, and scrolling through the real runner
- * against the scripted engine: matchers poll until the tree catches up, a
- * negated matcher holds for its grace window, a missing node is reported
- * after the deadline and an ambiguous one at once, a stale node is
- * re-resolved once, engine refusals map to ACTION_FAILED without a repeat,
- * and `scrollUntilVisible` swipes the viewport until the target shows.
+ * Polling through the real runner against the scripted engine: matchers poll
+ * until the tree catches up, and a negated matcher holds for its grace window.
*/
import { describe, expect, it } from 'vitest';
import { assertValidReport } from '../helpers/report-schema.ts';
-import { SCENE_ROOT_ID } from '../helpers/scripted-scene.ts';
-import { failed, passed, scriptedSuite, step } from '../helpers/scripted-suite.ts';
+import { passed, scriptedSuite, step } from '../helpers/scripted-suite.ts';
const POLLING = `import { test, expect } from 'e2e';
@@ -48,52 +43,9 @@ test('a negated matcher waits for the flip, then the grace window', async ({ app
await expect(screen.getByRole('status', { name: 'Loading' })).not.toBeVisible({ timeout: 2500 });
await screen.getByRole('status', { name: 'Loading' }).waitFor({ state: 'hidden' });
});
-
-test('waitFor attached and detached read presence, not visibility', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByText('Hidden content').waitFor({ state: 'attached' });
- await screen.getByText('Never rendered').waitFor({ state: 'detached' });
-});
-
-test('a missing node is LOCATOR_NOT_FOUND after the deadline', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('button', { name: 'Never exists' }).tap({ timeout: 250 });
-});
-
-test('an ambiguous locator fails an action at once', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByText('Duplicated').tap({ timeout: 3000 });
-});
-
-test('a stale node is re-resolved and the action retried once', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('button', { name: 'Stale' }).tap();
-});
-
-test('a possibly committed action is ACTION_FAILED and never repeated', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('button', { name: 'Commit' }).tap();
-});
-
-test('a non-actionable node is ACTION_FAILED', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('button', { name: 'Flaky' }).tap();
-});
-
-test('scrollUntilVisible reveals a node after two root swipes', async ({ app, screen }) => {
- await app.open('/');
- await expect(screen.getByRole('button', { name: 'Below the fold' })).toHaveCount(0);
- await screen.scrollUntilVisible(screen.getByRole('button', { name: 'Below the fold' }), { timeout: 3000 });
- await expect(screen.getByRole('button', { name: 'Below the fold' })).toBeVisible();
-});
-
-test('scrollUntilVisible times out with LOCATOR_NOT_FOUND', async ({ app, screen }) => {
- await app.open('/');
- await screen.scrollUntilVisible(screen.getByRole('button', { name: 'Never exists' }), { timeout: 350, direction: 'up' });
-});
`;
-describe('scripted engine: polling, staleness, failure mapping, and scrolling', () => {
+describe('scripted engine: polling', () => {
const run = scriptedSuite('polling.e2e.ts', POLLING);
it('polls until the tree catches up', () => {
@@ -112,49 +64,4 @@ describe('scripted engine: polling, staleness, failure mapping, and scrolling',
const flipped = passed(run.outcome, 'a negated matcher waits for the flip, then the grace window');
expect(step(flipped, 'expect.not.toBeVisible').durationMs).toBeGreaterThanOrEqual(1_150);
});
-
- it('waits for presence with the attached and detached states', () => {
- const attempt = passed(run.outcome, 'waitFor attached and detached read presence, not visibility');
- expect(attempt.steps.filter((entry) => entry.api === 'locator.waitFor').map((entry) => entry.label)).toEqual([
- 'getByText("Hidden content") → attached',
- 'getByText("Never rendered") → detached',
- ]);
- });
-
- it('reports a missing node after the deadline and an ambiguous one at once', () => {
- const missing = failed(run.outcome, 'a missing node is LOCATOR_NOT_FOUND after the deadline', 'LOCATOR_NOT_FOUND', 'matched no nodes within');
- expect(missing.error?.details).toMatchObject({ role: 'button', name: 'Never exists', waitedMs: expect.any(Number) });
- expect(missing.error?.details?.waitedMs).toBeGreaterThanOrEqual(250);
- expect(run.fake.callsOf(missing.id, 'perform')).toEqual([]);
- const ambiguous = failed(run.outcome, 'an ambiguous locator fails an action at once', 'LOCATOR_AMBIGUOUS', 'matched 2 nodes');
- // At once: well inside the 3000 ms the test allowed the action.
- expect(step(ambiguous, 'locator.tap').durationMs).toBeLessThan(3_000);
- expect(run.fake.callsOf(ambiguous.id, 'perform')).toEqual([]);
- });
-
- it('re-resolves a stale node and retries the action once', () => {
- const attempt = passed(run.outcome, 'a stale node is re-resolved and the action retried once');
- expect(run.fake.callsOf(attempt.id, 'perform').map((entry) => entry.id)).toEqual(['stale', 'stale~relocated']);
- });
-
- it('maps engine refusals to ACTION_FAILED without repeating them', () => {
- const committed = failed(run.outcome, 'a possibly committed action is ACTION_FAILED and never repeated', 'ACTION_FAILED', 'the tap may have landed');
- expect(run.fake.callsOf(committed.id, 'perform')).toHaveLength(1);
- expect(committed.error?.category).toBe('test');
- const blocked = failed(run.outcome, 'a non-actionable node is ACTION_FAILED', 'ACTION_FAILED', 'covered by an overlay');
- expect(run.fake.callsOf(blocked.id, 'perform')).toHaveLength(1);
- });
-
- it('scrolls the viewport until the target shows, or reports LOCATOR_NOT_FOUND', () => {
- const revealed = passed(run.outcome, 'scrollUntilVisible reveals a node after two root swipes');
- expect(run.fake.callsOf(revealed.id, 'perform').map((entry) => [entry.id, entry.action])).toEqual([
- [SCENE_ROOT_ID, { kind: 'swipe', direction: 'down', momentum: 'slow' }],
- [SCENE_ROOT_ID, { kind: 'swipe', direction: 'down', momentum: 'slow' }],
- ]);
- expect(step(revealed, 'screen.scrollUntilVisible').label).toBe('getByRole("button", name: "Below the fold")');
- const gaveUp = failed(run.outcome, 'scrollUntilVisible times out with LOCATOR_NOT_FOUND', 'LOCATOR_NOT_FOUND', 'did not become visible while scrolling');
- const swipes = run.fake.callsOf(gaveUp.id, 'perform');
- expect(swipes.length).toBeGreaterThanOrEqual(1);
- expect(swipes.every((entry) => entry.id === SCENE_ROOT_ID && entry.action.kind === 'swipe' && entry.action.direction === 'up')).toBe(true);
- });
});
diff --git a/packages/e2e/tests/integration/scripted-reads.test.ts b/packages/e2e/tests/integration/scripted-reads.test.ts
index 9be8f8b43..54bcb0ae7 100644
--- a/packages/e2e/tests/integration/scripted-reads.test.ts
+++ b/packages/e2e/tests/integration/scripted-reads.test.ts
@@ -1,9 +1,7 @@
/**
- * Locator reads and queries through the real runner against the scripted
- * engine: every read answers from the current tree without waiting, secure
- * fields refuse value reads, strictness applies to reads, every query kind
- * and refinement resolves, and malformed refinements are refused before any
- * query runs.
+ * Locator reads through the real runner against the scripted engine: every
+ * read answers from the current tree without waiting, secure fields refuse
+ * value reads, and malformed refinements are refused before any query runs.
*/
import { describe, expect, it } from 'vitest';
@@ -76,75 +74,6 @@ test('allTextContents over a secure field is denied', async ({ app, screen }) =>
await screen.getByRole('textbox').allTextContents();
});
-test('a read of zero matches is LOCATOR_NOT_FOUND', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByText('Never rendered').textContent();
-});
-
-test('a read of several matches is LOCATOR_AMBIGUOUS', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByText('Duplicated').textContent();
-});
-
-test('isVisible on several matches is LOCATOR_AMBIGUOUS', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByText('Duplicated').isVisible();
-});
-
-test('isDisabled on zero matches is LOCATOR_NOT_FOUND', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('button', { name: 'Never exists' }).isDisabled();
-});
-
-test('waitFor times out with LOCATOR_NOT_FOUND', async ({ app, screen }) => {
- await app.open('/');
- await screen.getByRole('button', { name: 'Never exists' }).waitFor({ timeout: 50 });
-});
-
-test('every query kind and refinement resolves', async ({ app, screen }) => {
- await app.open('/');
- await expect(screen.getByRole('button', { name: 'Submit' })).toBeVisible();
- await expect(screen.getByRole('button', { name: 'sub', exact: false })).toBeVisible();
- await expect(screen.getByRole('button', { name: /^Sub/ })).toBeVisible();
- await expect(screen.getByRole('img', { name: 'Map' })).toBeVisible();
- await expect(screen.getByRole('heading', { level: 2 })).toHaveText('Recent');
- await expect(screen.getByRole('heading')).toHaveCount(2);
- await expect(screen.getByRole('tab', { selected: true })).toHaveText('All');
- await expect(screen.getByRole('button', { disabled: true })).toHaveAccessibleName('Disabled action');
- await expect(screen.getByRole('checkbox', { checked: true })).toHaveCount(1);
- await expect(screen.getByRole('button', { name: 'Menu', expanded: false })).toBeVisible();
- await expect(screen.getByRole('button', { name: 'Bold', pressed: false })).toBeVisible();
- await expect(screen.getByText('Ready')).toHaveCount(2);
- await expect(screen.getByRole('status', { name: 'Ready state' })).toHaveCount(1);
- await expect(screen.getByText('Ready', { visible: true })).toHaveCount(1);
- await expect(screen.getByText('Ready', { visible: true })).toBeVisible();
- await expect(screen.getByLabel('Email')).toBeVisible();
- await expect(screen.getByLabel('email', { exact: false })).toBeVisible();
- await expect(screen.getByLabel(/^Em/)).toBeVisible();
- await expect(screen.getByPlaceholder('you@example.test')).toHaveAccessibleName('Email');
- await expect(screen.getByPlaceholder('EXAMPLE', { exact: false })).toHaveCount(1);
- await expect(screen.getByText('Card body')).toBeVisible();
- await expect(screen.getByText('card BODY', { exact: false })).toBeVisible();
- await expect(screen.getByText(/^Card/)).toBeVisible();
- await expect(screen.getByText('Echoed')).toHaveCount(1);
- await expect(screen.getByDisplayValue('hello-value')).toHaveAccessibleName('Search');
- await expect(screen.getByDisplayValue(/^hello/)).toHaveCount(1);
- await expect(screen.getByDisplayValue('hunter2')).toHaveCount(0);
- await expect(screen.getByTestId('card')).toBeVisible();
- await expect(screen.getByTestId('todos').getByRole('checkbox')).toHaveCount(3);
- await expect(screen.getByTestId('todos').getByRole('listitem').filter({ hasText: 'ship' })).toHaveText('Ship runner');
- await expect(screen.getByTestId('todo').filter({ hasText: /^Write/ })).toHaveCount(1);
- await expect(screen.getByTestId('todo').filter({ has: screen.getByRole('checkbox', { checked: true }) })).toHaveText('Ship runner');
- await expect(screen.getByTestId('todo').filter({ hasText: 'Release', has: screen.getByRole('checkbox') })).toHaveCount(1);
- await expect(screen.getByTestId('todo').filter({ hasText: 'Release', has: screen.getByRole('checkbox', { checked: true }) })).toHaveCount(0);
- await expect(screen.getByTestId('todo').first()).toHaveText('Write spec');
- await expect(screen.getByTestId('todo').last()).toHaveText('Release');
- await expect(screen.getByTestId('todo').nth(1)).toHaveText('Ship runner');
- await expect(screen.getByTestId('todo').nth(7)).toHaveCount(0);
- await expect(screen.getByTestId('todo').filter({ hasText: 'Ship' }).getByRole('checkbox')).toBeChecked();
- await expect(screen.getByRole('list', { name: 'Todos' }).getByText('Release')).toBeVisible();
-});
-
test('filter without a predicate is INVALID_LOCATOR', async ({ app, screen }) => {
await app.open('/');
screen.getByTestId('todo').filter({});
@@ -183,19 +112,6 @@ describe('scripted engine: locator reads and queries', () => {
}
});
- it('applies strictness to reads', () => {
- failed(run.outcome, 'a read of zero matches is LOCATOR_NOT_FOUND', 'LOCATOR_NOT_FOUND', 'matched no nodes', 'getByText("Never rendered")');
- failed(run.outcome, 'a read of several matches is LOCATOR_AMBIGUOUS', 'LOCATOR_AMBIGUOUS', 'matched 2 nodes, expected exactly one');
- failed(run.outcome, 'isVisible on several matches is LOCATOR_AMBIGUOUS', 'LOCATOR_AMBIGUOUS', 'matched 2 nodes');
- failed(run.outcome, 'isDisabled on zero matches is LOCATOR_NOT_FOUND', 'LOCATOR_NOT_FOUND', 'matched no nodes');
- const waited = failed(run.outcome, 'waitFor times out with LOCATOR_NOT_FOUND', 'LOCATOR_NOT_FOUND', 'did not become visible');
- expect(step(waited, 'locator.waitFor').status).toBe('failed');
- });
-
- it('resolves every query kind and refinement', () => {
- passed(run.outcome, 'every query kind and refinement resolves');
- });
-
it('rejects malformed refinements before any query runs', () => {
failed(run.outcome, 'filter without a predicate is INVALID_LOCATOR', 'INVALID_LOCATOR', 'filter() requires hasText and/or has');
failed(run.outcome, 'a negative index is INVALID_LOCATOR', 'INVALID_LOCATOR', 'nonnegative integer');
diff --git a/packages/e2e/tests/integration/selected-secrets.test.ts b/packages/e2e/tests/integration/selected-secrets.test.ts
index 7defb5cd9..b0b765643 100644
--- a/packages/e2e/tests/integration/selected-secrets.test.ts
+++ b/packages/e2e/tests/integration/selected-secrets.test.ts
@@ -28,7 +28,7 @@ test('selects part of a secret in a plain field', async ({ app, agent, screen })
await agent.assert('part of the plain token is selected');
await screen.getByLabel('Token').fill(credentials.user('member').password);
await agent.assert('part of the secret token is selected');
- await screen.getByRole('button', { name: 'Not on this page' }).tap();
+ await screen.getByRole('button', { name: 'Not on this page' }).tap({ timeout: 500 });
});
`;
diff --git a/packages/e2e/tests/integration/skip-after-failure.test.ts b/packages/e2e/tests/integration/skip-after-failure.test.ts
index b6950b5a1..d551374d2 100644
--- a/packages/e2e/tests/integration/skip-after-failure.test.ts
+++ b/packages/e2e/tests/integration/skip-after-failure.test.ts
@@ -81,9 +81,9 @@ describe('a runtime skip after a failure', () => {
expect(output).toContain('Skipped After Failure 1');
});
- it.each(['', 'failOnSkippedFailure: true,'])('shows the original failure when a retry skips (%s)', (policy) => {
- const { report, output, exitCode } = runSkipped(RETRY_SKIP, policy);
- expect(exitCode).toBe(policy === '' ? 0 : 1);
+ it('shows the original failure when a retry skips', () => {
+ const { report, output, exitCode } = runSkipped(RETRY_SKIP, 'failOnSkippedFailure: true,');
+ expect(exitCode).toBe(1);
expect(report.run.results[0]?.attempts.map((attempt) => attempt.status)).toEqual(['failed', 'skipped']);
expect(output).toContain('Skipped After Failure 1');
expect(output).toContain('original failure: expected 1 to be 2');
@@ -127,15 +127,4 @@ describe('a runtime skip after a failure', () => {
expect(exitCode).toBe(0);
expect(output).not.toContain('Skipped After Failure');
});
-
- it('shows the original failure even when the skipped retry also fails cleanup', () => {
- const { output, exitCode } = runSkipped(
- RETRY_SKIP,
- 'failOnSkippedFailure: true,',
- 0,
- `async endAttempt() { throw new Error('cleanup failed'); },`,
- );
- expect(exitCode).toBe(1);
- expect(output).toContain('original failure: expected 1 to be 2');
- });
});
diff --git a/packages/e2e/tests/integration/trace-cache-replay.test.ts b/packages/e2e/tests/integration/trace-cache-replay.test.ts
index 1054f023a..42e5188c0 100644
--- a/packages/e2e/tests/integration/trace-cache-replay.test.ts
+++ b/packages/e2e/tests/integration/trace-cache-replay.test.ts
@@ -6,7 +6,7 @@
* an entry may hold is checked against the report and the entry files.
*/
-import { writeFileSync } from 'node:fs';
+import { readFileSync, statSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { stripVTControlCharacters } from 'node:util';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
@@ -14,6 +14,7 @@ import type { TraceEntry } from '../../src/cache/trace.ts';
import { main } from '../../src/cli/index.ts';
import type { E2EConfig } from '../../src/index.ts';
import { loopCalls } from '../helpers/fake-loop-model.ts';
+import { assertValidReport } from '../helpers/report-schema.ts';
import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { createProject, runExisting, webTarget, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
import {
@@ -35,6 +36,14 @@ import {
const FLOWS_FILE = 'tests/flows.e2e.ts';
+/** The counter flow's entry file: its bytes and modification time, to prove a replay left it alone. */
+function counterEntryFile(project: FixtureProject): { bytes: string; mtimeMs: number } {
+ const counter = readEntries(project).filter(({ entry }) => entry.payload.recordedFor?.testId.endsWith(`::${encodeURIComponent('increments the counter')}`));
+ expect(counter).toHaveLength(1);
+ const { file } = counter[0]!;
+ return { bytes: readFileSync(file, 'utf8'), mtimeMs: statSync(file).mtimeMs };
+}
+
/** One entry per act step every flow makes. */
const ENTRY_COUNT = FLOWS.reduce((total, flow) => total + flow.actions.length, 0);
@@ -45,6 +54,8 @@ describe('trace cache: every flow records on the first run and replays without a
let recordedModelCalls = 0;
/** The archive entry as the first run wrote it; the second run opens the record under another id. */
let recordedArchive: TraceEntry | undefined;
+ /** The counter entry's bytes and modification time as the first run wrote them. */
+ let recordedCounter: { bytes: string; mtimeMs: number } | undefined;
let replayed: RunOutcome;
beforeAll(async () => {
@@ -53,6 +64,7 @@ describe('trace cache: every flow records on the first run and replays without a
recorded = await runExisting(project, { appUrl: app.url, config: cacheConfig(flowsModel()) });
recordedModelCalls = loopCalls.length;
[recordedArchive] = entriesFor(project, 'archives a record');
+ recordedCounter = counterEntryFile(project);
writeFileSync(path.join(project.dir, FLOWS_FILE), flowsSuite(FLOWS, 'replay'), 'utf8');
// The twins form moves every node, and the offers reverse, under the URL the recording saw.
app.setVariant('b');
@@ -64,9 +76,11 @@ describe('trace cache: every flow records on the first run and replays without a
await app?.close();
});
- it('passes every flow on both runs', () => {
+ it('passes every flow on both runs and reports both in the schema', () => {
expectPassed(recorded);
expectPassed(replayed);
+ assertValidReport(recorded.report);
+ assertValidReport(replayed.report);
});
it('misses every step on the first run and writes one entry per act step', () => {
@@ -81,6 +95,7 @@ describe('trace cache: every flow records on the first run and replays without a
expect(entriesFor(project, flow.title), flow.title).toHaveLength(flow.actions.length);
}
expect(readEntries(project)).toHaveLength(ENTRY_COUNT);
+ expect(readEntries(project).map(({ entry }) => entry.schemaVersion)).toEqual(Array.from({ length: ENTRY_COUNT }, () => 'trace-1'));
});
it('replays every step on the second run with zero model calls across the whole run', () => {
@@ -93,6 +108,10 @@ describe('trace cache: every flow records on the first run and replays without a
expect(readEntries(project)).toHaveLength(ENTRY_COUNT);
});
+ it('never rewrites an entry the cache replayed whole: the file keeps its bytes and its modification time', () => {
+ expect(counterEntryFile(project)).toEqual(recordedCounter);
+ });
+
it('keeps a unique() value as a slot in the typed text, the percent-encoded path, and the anchors', () => {
const [entry] = entriesFor(project, 'creates a company');
const { payload } = entry!;
@@ -168,9 +187,11 @@ const DIVERGENCE: readonly Flow[] = [
{ ...archives, title: 'retires a record', open: () => '/records/2b3c4d5e6f7a' },
];
-describe('trace cache: a changed screen hands the step to the agent, which re-records it', () => {
+describe('trace cache: a changed screen hands the step to the agent, which re-records it, and --strict-cache fails it instead', () => {
let app: FixtureApp;
let project: FixtureProject;
+ let strict: RunOutcome;
+ let strictModelCalls = 0;
let replayed: RunOutcome;
beforeAll(async () => {
@@ -179,6 +200,9 @@ describe('trace cache: a changed screen hands the step to the agent, which re-re
expectPassed(await runExisting(project, { appUrl: app.url, config: cacheConfig(flowsModel()) }));
writeFileSync(path.join(project.dir, FLOWS_FILE), flowsSuite(DIVERGENCE, 'replay'), 'utf8');
app.setVariant('renamed');
+ // A strict run first: it keeps the stale entries it failed on, so the lenient run after it meets the same recordings.
+ strict = await runExisting(project, { appUrl: app.url, config: cacheConfig(flowsModel()), runOptions: { strictCache: true } });
+ strictModelCalls = loopCalls.length;
replayed = await runExisting(project, { appUrl: app.url, config: cacheConfig(flowsModel()) });
}, 240_000);
@@ -187,7 +211,23 @@ describe('trace cache: a changed screen hands the step to the agent, which re-re
await app?.close();
});
- it('passes every flow on the second run too', () => {
+ it('fails every diverged step with REPLAY_STALE, exit 2, without a model call or a retry', () => {
+ expect(strict.exitCode).toBe(2);
+ expect(strictModelCalls).toBe(0);
+ for (const [title, reason] of [
+ ['creates a company', 'target-not-found'],
+ ['archives a record', 'wrong-context'],
+ ['retires a record', 'target-not-found'],
+ ] as const) {
+ const step = onlyActStep(strict, title);
+ expect(step.error?.code, title).toBe('REPLAY_STALE');
+ expect(step.error?.message, title).toContain(reason);
+ expect(step.error?.message, title).toContain('re-record it with a read-write run without --strict-cache and commit the changed entry under .e2e/cache');
+ expect(step.cache?.reason, title).toBe(reason);
+ }
+ });
+
+ it('passes every flow on the lenient run', () => {
expectPassed(replayed);
});
@@ -219,44 +259,6 @@ describe('trace cache: a changed screen hands the step to the agent, which re-re
});
});
-describe('trace cache: --strict-cache fails a stale recording instead of handing it to the agent', () => {
- let app: FixtureApp;
- let project: FixtureProject;
- let strict: RunOutcome;
- let strictModelCalls = 0;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- project = createProject({ [FLOWS_FILE]: flowsSuite(DIVERGENCE, 'record') });
- expectPassed(await runExisting(project, { appUrl: app.url, config: cacheConfig(flowsModel()) }));
- writeFileSync(path.join(project.dir, FLOWS_FILE), flowsSuite(DIVERGENCE, 'replay'), 'utf8');
- app.setVariant('renamed');
- strict = await runExisting(project, { appUrl: app.url, config: cacheConfig(flowsModel()), runOptions: { strictCache: true } });
- strictModelCalls = loopCalls.length;
- }, 240_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it('fails every diverged step with REPLAY_STALE, exit 2, without a model call or a retry', () => {
- expect(strict.exitCode).toBe(2);
- expect(strictModelCalls).toBe(0);
- for (const [title, reason] of [
- ['creates a company', 'target-not-found'],
- ['archives a record', 'wrong-context'],
- ['retires a record', 'target-not-found'],
- ] as const) {
- const step = onlyActStep(strict, title);
- expect(step.error?.code, title).toBe('REPLAY_STALE');
- expect(step.error?.message, title).toContain(reason);
- expect(step.error?.message, title).toContain('re-record it with a read-write run without --strict-cache and commit the changed entry under .e2e/cache');
- expect(step.cache?.reason, title).toBe(reason);
- }
- });
-});
-
/** The one-flow suite the target, repeat, and re-keyed suites run: the counter, incremented twice. */
const COUNTER_SUITE = flowsSuite([flowByTitle('increments the counter')], 'record');
diff --git a/packages/e2e/tests/integration/trace-cache-speed.test.ts b/packages/e2e/tests/integration/trace-cache-speed.test.ts
deleted file mode 100644
index 33792b41d..000000000
--- a/packages/e2e/tests/integration/trace-cache-speed.test.ts
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * How fast a trace replay is against the deterministic API it stands in for.
- * One flow, three ways, on the fixture app: `locator.fill` and `locator.click`
- * as a test would write them, the same flow as an `agent.act` recorded through
- * a scripted model, and that recording replayed with no model call. Both sides
- * are measured alike, as the whole flow after `app.open` with its trailing
- * assertion, from the report's step timings, and the fastest repetition after
- * the first stands for each: the first pays for the warm-up, and a loaded
- * runner slows any single one. The gate is a small factor of the twin plus a
- * ceiling well above what a healthy replay takes; like the other PR gates, it
- * catches a replay that stopped being cheap, not a benchmark regression.
- */
-
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { createProject, resultByTitle, runExisting, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
-import { cacheConfig, flowByTitle, flowsModel, flowsSuite, onlyActStep } from '../helpers/trace-cache.ts';
-
-const REPETITIONS = [1, 2, 3, 4] as const;
-/** A replay may take this many times its deterministic twin. */
-const REPLAY_RATIO = 3;
-/** And never longer than this, whatever the twin took: an order of magnitude above a healthy replay. */
-const REPLAY_CEILING_MS = 2_000;
-
-const company = flowByTitle('creates a company');
-
-/** The company flow as a test writes it by hand. */
-const DETERMINISTIC_BODY = ` const name = 'E2E ' + String(Date.now()) + ' Company';
- await screen.getByLabel('Company name').fill(name);
- await screen.getByRole('button', { name: 'Create' }).click();
- await expect(screen.getByRole('heading', { name })).toBeVisible();`;
-
-const SUITE = flowsSuite(
- REPETITIONS.flatMap((n) => [
- { title: `deterministic ${String(n)}`, open: company.open, body: DETERMINISTIC_BODY },
- { title: `agent ${String(n)}`, open: company.open, body: company.body },
- ]),
- 'record',
-);
-
-/** The flow's own time: every step after `app.open`, the trailing expect included, since a replay checks its end state too. */
-function flowMs(outcome: RunOutcome, title: string): number {
- return resultByTitle(outcome, title)
- .attempts.at(-1)!
- .steps.filter((step) => step.api !== 'app.open')
- .reduce((total, step) => total + step.durationMs, 0);
-}
-
-/** The fastest repetition after the first. */
-function fastest(outcome: RunOutcome, prefix: string): number {
- return Math.min(...REPETITIONS.slice(1).map((n) => flowMs(outcome, `${prefix} ${String(n)}`)));
-}
-
-describe('trace cache: a replay runs at the speed of the deterministic API', () => {
- let app: FixtureApp;
- let project: FixtureProject;
- let recorded: RunOutcome;
- let replayed: RunOutcome;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- project = createProject({ 'tests/speed.e2e.ts': SUITE });
- recorded = await runExisting(project, { appUrl: app.url, config: cacheConfig(flowsModel()) });
- replayed = await runExisting(project, { appUrl: app.url, config: cacheConfig(flowsModel()) });
- }, 240_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it('passes both runs, recording on the first and replaying with no model call on the second', () => {
- expect(recorded.exitCode).toBe(0);
- expect(replayed.exitCode).toBe(0);
- for (const n of REPETITIONS) {
- const step = onlyActStep(replayed, `agent ${String(n)}`);
- expect(step.cache).toEqual({ mode: 'self-finalized', replayedActions: 2, totalActions: 2 });
- expect(step.metrics?.modelCalls).toBe(0);
- }
- });
-
- it('replays within a small factor of the deterministic twin, under the ceiling, and faster than it recorded', () => {
- const deterministic = fastest(replayed, 'deterministic');
- const record = fastest(recorded, 'agent');
- const replay = fastest(replayed, 'agent');
- const numbers =
- `deterministic ${String(deterministic)} ms · record ${String(record)} ms · replay ${String(replay)} ms · ` +
- `ratio ${(replay / deterministic).toFixed(2)} (bound ${String(REPLAY_RATIO)}) · ceiling ${String(REPLAY_CEILING_MS)} ms`;
- console.info(`trace cache speed (fastest of ${String(REPETITIONS.length - 1)} after warm-up): ${numbers}`);
- expect(replay, numbers).toBeLessThanOrEqual(REPLAY_RATIO * deterministic);
- expect(replay, numbers).toBeLessThanOrEqual(REPLAY_CEILING_MS);
- expect(replay, numbers).toBeLessThanOrEqual(record);
- });
-});
diff --git a/packages/e2e/tests/integration/trace-modes.test.ts b/packages/e2e/tests/integration/trace-modes.test.ts
index c5d037d5c..f54d1a0d1 100644
--- a/packages/e2e/tests/integration/trace-modes.test.ts
+++ b/packages/e2e/tests/integration/trace-modes.test.ts
@@ -63,11 +63,11 @@ describe('trace modes', () => {
});
it(
- 'records retries only under on-all-retries, and a test mode wins over the config',
+ 'records retries only under on-all-retries, --trace wins over the config, and a test mode wins over the flag',
async () => {
const { outcome, project } = await runProject(
{ 'tests/suite.e2e.ts': SUITE },
- { appUrl: app.url, config: { retries: 2, cache: 'off', trace: 'on-all-retries' } },
+ { appUrl: app.url, config: { retries: 2, cache: 'off', trace: 'off' }, runOptions: { trace: 'on-all-retries' } },
);
try {
expect(tracesPerAttempt(outcome, 'passes first time')).toEqual([0]);
@@ -88,38 +88,4 @@ describe('trace modes', () => {
},
180_000,
);
-
- it(
- 'records every attempt by default outside CI',
- async () => {
- const { outcome, project } = await runProject(
- { 'tests/suite.e2e.ts': SUITE },
- { appUrl: app.url, config: { retries: 1, cache: 'off' } },
- );
- try {
- expect(tracesPerAttempt(outcome, 'passes first time')).toEqual([1]);
- expect(tracesPerAttempt(outcome, 'always fails')).toEqual([1, 1]);
- } finally {
- project.cleanup();
- }
- },
- 180_000,
- );
-
- it(
- 'turns off with --trace off, and a test mode still wins over the flag',
- async () => {
- const { outcome, project } = await runProject(
- { 'tests/suite.e2e.ts': SUITE },
- { appUrl: app.url, config: { retries: 1, cache: 'off', trace: 'on' }, runOptions: { trace: 'off' } },
- );
- try {
- expect(tracesPerAttempt(outcome, 'always fails')).toEqual([0, 0]);
- expect(tracesPerAttempt(outcome, 'first retry only')).toEqual([0, 1]);
- } finally {
- project.cleanup();
- }
- },
- 180_000,
- );
});
diff --git a/packages/e2e/tests/integration/transformed-secrets.test.ts b/packages/e2e/tests/integration/transformed-secrets.test.ts
index 5f3254654..0404d1c90 100644
--- a/packages/e2e/tests/integration/transformed-secrets.test.ts
+++ b/packages/e2e/tests/integration/transformed-secrets.test.ts
@@ -9,12 +9,11 @@
* still passes.
*/
-import { readdirSync, readFileSync, statSync } from 'node:fs';
import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type { RunOutcome } from '../../src/run/runner.ts';
import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
-import { createProject, resultByTitle, runExisting, type FixtureProject } from '../helpers/run-project.ts';
+import { contentsUnder, createProject, resultByTitle, runExisting, type FixtureProject } from '../helpers/run-project.ts';
const TOKEN = 'Tok-9f3aC0DeB1e7';
const NOTE = 'first line 4417\nsecond line Qx';
@@ -50,14 +49,6 @@ test('matches the transformed text', async ({ app, screen }) => {
});
`;
-/** Every regular file under `dir`, recursively. */
-function filesUnder(dir: string): string[] {
- return readdirSync(dir).flatMap((name) => {
- const file = path.join(dir, name);
- return statSync(file).isDirectory() ? filesUnder(file) : [file];
- });
-}
-
describe('secrets shown transformed', () => {
let app: FixtureApp;
let project: FixtureProject;
@@ -101,12 +92,14 @@ describe('secrets shown transformed', () => {
for (const form of FORMS) expect(text).not.toContain(form);
});
- it.each(['plain', 'upper', 'lower', 'note-echo'])('fails on the %s echo with the received text redacted', (id) => {
- const result = resultByTitle(outcome, `fails on the ${id} echo`);
- expect(result.status).toBe('failed');
- const message = result.attempts[0]!.error!.message;
- expect(message).toContain(id === 'note-echo' ? '' : '');
- for (const form of FORMS) expect(message).not.toContain(form);
+ it('fails on each echo with the received text redacted', () => {
+ for (const id of ['plain', 'upper', 'lower', 'note-echo']) {
+ const result = resultByTitle(outcome, `fails on the ${id} echo`);
+ expect(result.status, id).toBe('failed');
+ const message = result.attempts[0]!.error!.message;
+ expect(message).toContain(id === 'note-echo' ? '' : '');
+ for (const form of FORMS) expect(message).not.toContain(form);
+ }
});
it('matches the transformed text as the page shows it', () => {
@@ -116,10 +109,10 @@ describe('secrets shown transformed', () => {
it('leaves no form of either value in the report or any file the reporters write', () => {
const report = JSON.stringify(outcome.report);
for (const form of FORMS) expect(report).not.toContain(form);
- const files = filesUnder(path.join(project.dir, '.e2e'));
- expect(files.some((file) => file.endsWith('.md'))).toBe(true);
- for (const file of files) {
- const text = readFileSync(file, 'utf8');
+ const contents = contentsUnder(path.join(project.dir, '.e2e'));
+ expect(contents.some(([file]) => file.endsWith('.md'))).toBe(true);
+ expect(contents.some(([file]) => file.includes('.zip!'))).toBe(true);
+ for (const [file, text] of contents) {
for (const form of FORMS) expect(text, file).not.toContain(form);
}
});
diff --git a/packages/e2e/tests/integration/truncated-secrets.test.ts b/packages/e2e/tests/integration/truncated-secrets.test.ts
index 23b75c9a2..94d714d23 100644
--- a/packages/e2e/tests/integration/truncated-secrets.test.ts
+++ b/packages/e2e/tests/integration/truncated-secrets.test.ts
@@ -30,7 +30,7 @@ test('echoes a secret across the observation limits', async ({ app, agent, scree
await app.open('/echo-cut');
await screen.getByLabel('Source').fill(credentials.user('member').password);
await agent.assert('the echo is on screen');
- await screen.getByRole('button', { name: 'Not on this page' }).tap();
+ await screen.getByRole('button', { name: 'Not on this page' }).tap({ timeout: 500 });
});
`;
@@ -85,8 +85,5 @@ describe('secrets cut short by observation limits', () => {
// The trace was scanned inside, and its plain text survived the rewrite.
const trace = contents.filter(([file]) => file.includes('.zip!'));
expect(trace.some(([, text]) => text.includes(CONTROL_KEPT))).toBe(true);
- // The same rewrite dropped the screencast of the tainted viewport.
- expect(trace.filter(([file]) => file.includes('.zip!screencast/'))).toEqual([]);
- expect(trace.some(([, text]) => text.includes('"screencast-frame"'))).toBe(false);
});
});
diff --git a/packages/e2e/tests/integration/web-class-assertions.test.ts b/packages/e2e/tests/integration/web-class-assertions.test.ts
deleted file mode 100644
index 51a68c6b8..000000000
--- a/packages/e2e/tests/integration/web-class-assertions.test.ts
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * `expect(browser).toHaveClass` against the web engine loaded the way a project
- * loads it: from a config file, so the engine's `e2e/engine` and the runner's
- * core are two module copies and `instanceof` cannot tell a runner error apart.
- * A node that is not there yet must keep the matcher polling; an ambiguous
- * locator must still fail at once. `toHaveURL` takes Playwright's
- * `ignoreCase`.
- */
-
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
-import {
- resultByTitle,
- runProjectWithConfigFile,
- workerConfigSource,
- type FixtureProject,
- type RunOutcome,
-} from '../helpers/run-project.ts';
-
-const SUITE = `import { test } from '@e2e-dev/web';
-import { expect } from 'e2e';
-
-test('class assertions poll for a node that arrives late', async ({ app, browser }) => {
- await app.open('/classes');
- await expect(browser).toHaveClass(browser.locator('#late-card'), 'card late', { timeout: 1800 });
-});
-
-test('class assertions on an ambiguous locator fail at once', async ({ app, browser }) => {
- await app.open('/classes');
- await expect(browser).toHaveClass(browser.locator('.dup'), 'dup', { timeout: 1800 });
-});
-
-test('an empty class attribute is an empty class list', async ({ app, browser }) => {
- await app.open('/classes');
- await expect(browser).toHaveClass(browser.locator('#blank-card'), '');
-});
-
-test('a missing class attribute is not an empty class list', async ({ app, browser, screen }) => {
- await app.open('/classes');
- await expect(browser).toHaveClass(screen.getByTestId('items'), '', { timeout: 300 });
-});
-
-test('toHaveURL ignoreCase folds the comparison', async ({ app, browser }) => {
- await app.open('/classes');
- await expect(browser).toHaveURL('/CLASSES', { ignoreCase: true });
- await expect(browser).toHaveURL(/CLASSES$/, { ignoreCase: true });
- await expect(browser).not.toHaveURL('/CLASSES', { timeout: 300 });
-});
-
-test('a negated toHaveURL ignoreCase fails on a case-folded match', async ({ app, browser }) => {
- await app.open('/classes');
- await expect(browser).not.toHaveURL('/CLASSES', { ignoreCase: true, timeout: 300 });
-});
-`;
-
-describe('web class assertions', () => {
- let app: FixtureApp;
- let outcome: RunOutcome;
- let project: FixtureProject;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- ({ outcome, project } = await runProjectWithConfigFile(
- { 'tests/classes.e2e.ts': SUITE },
- { appUrl: app.url, configSource: workerConfigSource(1, "\n cache: 'off',") },
- ));
- }, 240_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it('keeps polling while the node is missing and passes once it arrives', () => {
- const result = resultByTitle(outcome, 'class assertions poll for a node that arrives late');
- expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe('passed');
- const step = result.attempts[0]!.steps.find((candidate) => candidate.api === 'expect.toHaveClass');
- expect(step?.durationMs).toBeGreaterThan(500);
- });
-
- it('fails an ambiguous locator with LOCATOR_AMBIGUOUS before the deadline', () => {
- const result = resultByTitle(outcome, 'class assertions on an ambiguous locator fail at once');
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error?.code).toBe('LOCATOR_AMBIGUOUS');
- const step = result.attempts[0]!.steps.find((candidate) => candidate.api === 'expect.toHaveClass');
- expect(step?.durationMs).toBeLessThan(1800);
- });
-
- it('matches an empty string against class="" and not against a missing attribute', () => {
- expect(resultByTitle(outcome, 'an empty class attribute is an empty class list').status).toBe('passed');
- const missing = resultByTitle(outcome, 'a missing class attribute is not an empty class list');
- expect(missing.status).toBe('failed');
- expect(missing.attempts[0]!.error?.code).toBe('ASSERTION_FAILED');
- expect(missing.attempts[0]!.error?.message).toContain('observed: no class attribute');
- });
-
- it('honors ignoreCase on toHaveURL, negated too', () => {
- const folded = resultByTitle(outcome, 'toHaveURL ignoreCase folds the comparison');
- expect(folded.status, JSON.stringify(folded.attempts[0]?.error)).toBe('passed');
- const negated = resultByTitle(outcome, 'a negated toHaveURL ignoreCase fails on a case-folded match');
- expect(negated.status).toBe('failed');
- expect(negated.attempts[0]!.error?.code).toBe('ASSERTION_FAILED');
- expect(negated.attempts[0]!.error?.message).toContain('expected: not URL /CLASSES (ignoring case)');
- });
-
-});
diff --git a/packages/e2e/tests/integration/web-platform.test.ts b/packages/e2e/tests/integration/web-platform.test.ts
index e998dfd30..e7680fe8f 100644
--- a/packages/e2e/tests/integration/web-platform.test.ts
+++ b/packages/e2e/tests/integration/web-platform.test.ts
@@ -3,8 +3,13 @@ import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
import { assertValidReport } from '../helpers/report-schema.ts';
-import { resultByTitle, runProject, type FixtureProject } from '../helpers/run-project.ts';
-import type { RunOutcome } from '../helpers/run-project.ts';
+import {
+ resultByTitle,
+ runProjectWithConfigFile,
+ workerConfigSource,
+ type FixtureProject,
+ type RunOutcome,
+} from '../helpers/run-project.ts';
const KITCHEN_SINK = `import { test } from '@e2e-dev/web';
import { expect } from 'e2e';
@@ -20,9 +25,6 @@ test('deterministic queries and reads', async ({ app, screen, browser }) => {
await expect(screen.getByTestId('items')).toBeVisible();
await expect(screen.getByTestId('item')).toHaveCount(3);
- await expect(screen.getByTestId('item').first()).toHaveText('Item Alpha');
- await expect(screen.getByTestId('item').last()).toHaveText('Item Gamma');
- await expect(screen.getByTestId('item').nth(1)).toHaveText('Item Beta');
await expect(
screen.getByTestId('item').filter({ hasText: 'Beta' }),
).toHaveCount(1);
@@ -39,33 +41,8 @@ test('deterministic queries and reads', async ({ app, screen, browser }) => {
await expect(screen.getByTestId('item')).toHaveText(['Item Alpha', /Beta/, 'Item Gamma']);
await expect(screen.getByTestId('item')).toContainText(['Alpha', 'Beta', 'Gamma']);
await expect(screen.getByTestId('item')).toContainText(['Alpha', 'Gamma']);
- await expect(screen.getByTestId('item')).not.toContainText(['Gamma', 'Alpha']);
- await expect(screen.getByTestId('item')).not.toHaveText(['Item Alpha', 'Item Beta']);
-
- await expect(screen.getByText('Hidden content')).toBeAttached();
- await expect(screen.getByText('Hidden content')).toBeHidden();
- await expect(screen.getByText('Never rendered')).not.toBeAttached();
- expect(await screen.getByText('Hidden content').isHidden()).toBe(true);
- expect(await screen.getByText('Never rendered').isHidden()).toBe(true);
- await expect(screen.getByTestId('hidden-svg')).toBeHidden();
- await expect(screen.getByTestId('zero-box')).toBeHidden();
- await expect(screen.getByRole('button', { name: 'Inside folded details' })).toBeHidden();
- await expect(screen.getByTestId('empty-contents')).toBeHidden();
- await expect(screen.getByTestId('painted-contents')).toBeVisible();
- expect(await screen.getByTestId('hidden-svg').isVisible()).toBe(false);
- expect(await screen.getByTestId('zero-box').isVisible()).toBe(false);
- expect(await screen.getByRole('button', { name: 'Inside folded details' }).isVisible()).toBe(false);
- expect(await screen.getByTestId('empty-contents').isVisible()).toBe(false);
- expect(await screen.getByTestId('painted-contents').isVisible()).toBe(true);
- expect(await screen.getByRole('button', { name: 'Disabled action' }).isDisabled()).toBe(true);
- expect(await screen.getByRole('button', { name: 'Increment' }).isDisabled()).toBe(false);
-
- await expect(screen.getByRole('button', { name: 'Disabled action' })).toBeDisabled();
- await expect(screen.getByRole('button', { name: 'Increment' })).toBeEnabled();
- expect(await screen.getByRole('button', { name: 'Fenced action' }).isDisabled()).toBe(true);
- expect(await screen.getByRole('button', { name: 'Legend action' }).isDisabled()).toBe(false);
- await expect(screen.getByRole('button', { name: 'Fenced action' })).toBeDisabled();
- await expect(screen.getByRole('button', { name: 'Legend action' })).toBeEnabled();
+ await expect(screen.getByTestId('item')).not.toContainText(['Gamma', 'Alpha'], { timeout: 200 });
+ await expect(screen.getByTestId('item')).not.toHaveText(['Item Alpha', 'Item Beta'], { timeout: 200 });
expect(await screen.getByTestId('items').getAttribute('class')).toBeNull();
expect(await browser.locator('#class-card').getAttribute('class')).toBe('card active');
@@ -77,161 +54,6 @@ test('deterministic queries and reads', async ({ app, screen, browser }) => {
expect(await screen.getByTestId('items').getAttribute('__proto__')).toBeNull();
});
-test('role vocabulary: tabs, menus, progress, toolbars, images', async ({ app, screen }) => {
- await app.open('/roles');
-
- await expect(screen.getByRole('image', { name: 'Fixture logo' })).toBeVisible();
- await expect(screen.getByRole('img', { name: 'Fixture logo' })).toBeVisible();
- await expect(screen.getByRole('image', { name: 'Sales chart' })).toBeVisible();
- await expect(screen.getByRole('img')).toHaveCount(2);
- await expect(screen.getByRole('image')).toHaveCount(2);
-
- const tabs = screen.getByRole('tablist', { name: 'Filter' });
- await expect(tabs.getByRole('tab')).toHaveCount(2);
- await expect(tabs.getByRole('tab', { selected: true })).toHaveText('All');
- await expect(screen.getByRole('tabpanel', { name: 'All' })).toHaveText('Everything');
-
- const toolbar = screen.getByRole('toolbar', { name: 'Formatting' });
- await toolbar.getByRole('button', { name: 'Bold' }).tap();
- await expect(toolbar.getByRole('button', { name: 'Bold' })).toHaveAttribute('aria-pressed', 'true');
- await expect(toolbar.getByRole('spinbutton', { name: 'Font size' })).toHaveValue('12');
-
- await screen.getByRole('button', { name: 'View' }).tap();
- const menu = screen.getByRole('menu', { name: 'View' });
- await expect(menu.getByRole('menuitem', { name: 'Zoom in' })).toBeVisible();
- await expect(menu.getByRole('menuitemcheckbox', { name: 'Show grid' })).toBeChecked();
-
- await expect(screen.getByRole('progressbar', { name: 'Upload' })).toBeVisible();
- await expect(screen.getByRole('group', { name: 'Notifications' }).getByRole('checkbox')).toHaveCount(1);
- await expect(screen.getByRole('group', { name: 'Notifications' })).toHaveAccessibleName('Notifications');
- await expect(screen.getByRole('form', { name: 'Sign in' }).getByRole('textbox')).toHaveCount(1);
- await expect(screen.getByRole('separator')).toHaveCount(1);
- await expect(screen.getByRole('article', { name: 'First post' })).toHaveText('Body');
-
- // Name from content for the roles accname allows, the same name on both
- // sides: the role query resolves it and the reader reports it.
- const named: readonly [role: string, name: string][] = [
- ['switch', 'Dark mode'],
- ['columnheader', 'Plan'],
- ['cell', 'Monthly'],
- ['button', 'Shadow action'],
- ['button', 'Clear form'],
- ['button', 'Reset'],
- ];
- for (const [role, name] of named) {
- const control = screen.getByRole(role as 'button', { name, exact: true });
- await expect(control).toHaveCount(1);
- await expect(control).toHaveAccessibleName(name);
- }
-});
-
-test('actions and state', async ({ app, screen, browser }) => {
- await app.open();
-
- await screen.getByRole('button', { name: 'Increment' }).tap();
- await screen.getByRole('button', { name: 'Increment' }).click();
- await expect(screen.getByRole('status')).toHaveText('2');
-
- await screen.getByLabel('Email').fill('user@example.test');
- await expect(screen.getByLabel('Email')).toHaveValue('user@example.test');
- await screen.getByLabel('Email').clear();
- await expect(screen.getByLabel('Email')).toHaveValue('');
-
- await screen.getByLabel('Notifications').check();
- await expect(screen.getByLabel('Notifications')).toBeChecked();
- await screen.getByLabel('Notifications').uncheck();
- await expect(screen.getByLabel('Notifications')).not.toBeChecked();
-
- await screen.getByLabel('Plan').selectOption('Pro');
- await expect(screen.getByLabel('Plan')).toHaveValue('pro');
- await screen.getByLabel('Plan').selectOption({ index: 2 });
- await expect(screen.getByLabel('Plan')).toHaveValue('team');
- await screen.getByLabel('Plan').selectOption({ value: 'free' });
- await expect(screen.getByLabel('Plan')).toHaveValue('free');
-
- await screen.getByRole('button', { name: 'Menu' }).tap();
- await expect(screen.getByRole('button', { name: 'Menu' })).toBeExpanded();
-
- await screen.getByLabel('Email').fill('draft');
- await screen.getByLabel('Email').press('Backspace');
- await expect(screen.getByLabel('Email')).toHaveValue('draf');
-
- await expect(screen.getByLabel('Readonly')).toHaveAttribute('readonly');
- await expect(browser.locator('#class-card')).toHaveAttribute('class', /active/);
- await expect(screen.getByLabel('Readonly')).not.toHaveAttribute('hidden');
- await expect(browser.locator('#class-card')).toHaveAttribute('constructor', 'own');
- await expect(screen.getByLabel('Readonly')).not.toHaveAttribute('constructor');
- await expect(screen.getByLabel('Readonly')).not.toHaveAttribute('toString');
- await expect(screen.getByLabel('Readonly')).not.toHaveAttribute('__proto__');
-
- await screen.getByLabel('Focus target').focus();
- await expect(screen.getByLabel('Focus target')).toBeFocused();
- await expect(screen.getByLabel('Email')).not.toBeFocused();
-
- await expect(browser).toHaveClass(browser.locator('#class-card'), 'card active');
- await expect(browser).toHaveClass(browser.locator('#class-card'), /active/);
- await expect(browser).not.toHaveClass(browser.locator('#class-card'), 'card inactive');
-});
-
-test('check and uncheck refuse an unknown option and leave the box untouched', async ({ app, screen, browser }) => {
- await app.open();
- await browser.evaluate(() => {
- const box = document.querySelector('#notifications')!;
- box.dataset.events = '0';
- for (const type of ['input', 'change', 'click']) {
- box.addEventListener(type, () => { box.dataset.events = String(Number(box.dataset.events) + 1); });
- }
- });
- const box = screen.getByLabel('Notifications');
- const code = async (call: () => Promise) => {
- try {
- await call();
- } catch (error) {
- return (error as { code?: string }).code;
- }
- return 'no error';
- };
-
- expect(await code(() => box.check({ trial: true } as never))).toBe('INVALID_ARGUMENT');
- await expect(box).not.toBeChecked();
- await expect(box).toHaveAttribute('data-events', '0');
- await box.check();
- await expect(box).toBeChecked();
- await expect(box).toHaveAttribute('data-events', '3');
-
- expect(await code(() => box.uncheck({ trial: true } as never))).toBe('INVALID_ARGUMENT');
- await expect(box).toBeChecked();
- await expect(box).toHaveAttribute('data-events', '3');
- await box.uncheck();
- await expect(box).not.toBeChecked();
- await expect(box).toHaveAttribute('data-events', '6');
-});
-
-test('unsupported filter and selectOption shapes fail before acting', async ({ app, screen, browser }) => {
- await app.open();
- const code = async (call: () => unknown) => {
- try {
- await call();
- } catch (error) {
- return (error as { code?: string }).code;
- }
- return 'no error';
- };
- const digest = () =>
- browser.evaluate(() => [...document.querySelector('#digest')!.selectedOptions].map((option) => option.value));
-
- const mixed = { hasText: 'Item', hasNotText: 'Alpha' } as never;
- expect(await code(() => screen.getByTestId('item').filter(mixed).first().textContent())).toBe('INVALID_LOCATOR');
- await expect(screen.getByTestId('item').filter({ hasText: 'Item' })).toHaveCount(3);
- await expect(screen.getByTestId('item').filter({ hasText: 'Item' }).first()).toHaveText('Item Alpha');
-
- const several = ['daily', 'weekly'] as never;
- expect(await code(() => screen.getByLabel('Digest').selectOption(several))).toBe('INVALID_ARGUMENT');
- expect(await digest()).toEqual([]);
- await screen.getByLabel('Digest').selectOption({ value: 'weekly' });
- expect(await digest()).toEqual(['weekly']);
-});
-
test('class assertions report the observed class on failure', async ({ app, browser }) => {
await app.open();
try {
@@ -242,40 +64,22 @@ test('class assertions report the observed class on failure', async ({ app, brow
}
});
-test('assertions poll until the app settles', async ({ app, screen }) => {
- await app.open();
- await expect(screen.getByRole('button', { name: 'Late arrival' })).toBeVisible({
- timeout: 3000,
- });
- await screen.getByRole('button', { name: 'Late arrival' }).waitFor({ state: 'visible' });
-});
-
-test('ambiguous locators fail immediately', async ({ app, screen }) => {
- await app.open();
- await screen.getByText('Duplicated').tap({ timeout: 2000 });
-});
-
-test('missing locators poll until the deadline', async ({ app, screen }) => {
- await app.open();
- await screen.getByRole('button', { name: 'Never exists' }).tap({ timeout: 800 });
-});
-
test('a textarea value compares raw', async ({ app, screen }) => {
await app.open();
await expect(screen.getByLabel('Notes')).toHaveValue('line1\\n\\nline2 ');
await expect(screen.getByLabel('Notes')).toHaveValue(/^line1\\n\\nline2 {2}$/);
- await expect(screen.getByLabel('Notes')).not.toHaveValue('line1 line2');
+ await expect(screen.getByLabel('Notes')).not.toHaveValue('line1 line2', { timeout: 200 });
});
test('a textarea value is never normalized to match', async ({ app, screen }) => {
await app.open();
- await expect(screen.getByLabel('Notes')).toHaveValue('line1 line2');
+ await expect(screen.getByLabel('Notes')).toHaveValue('line1 line2', { timeout: 500 });
});
test('toHaveAttribute reads the value attribute of plain fields', async ({ app, screen }) => {
await app.open('/value-attributes');
await expect(screen.getByLabel('Plain')).toHaveAttribute('value', 'marker-5e0c');
- await expect(screen.getByLabel('Blank')).not.toHaveAttribute('value');
+ await expect(screen.getByLabel('Blank')).not.toHaveAttribute('value', { timeout: 200 });
});
test('toHaveAttribute never judges a secure field', async ({ app, screen }) => {
@@ -311,33 +115,6 @@ test('toHaveValue reads a plain field that holds the same marker', async ({ app,
await expect(screen.getByLabel('Email')).toHaveValue('', { timeout: 500 });
});
-test('acting before open fails with APP_NOT_OPEN', async ({ screen }) => {
- await screen.getByRole('button', { name: 'Increment' }).tap({ timeout: 500 });
-});
-
-test('web navigation, urls, and titles', async ({ app, browser }) => {
- await app.open();
- await browser.goto('/about');
- await expect(browser).toHaveURL('/about');
- await expect(browser).toHaveTitle('About page');
- await browser.back();
- await expect(browser).toHaveURL('/');
- await browser.forward();
- await expect(browser).toHaveURL(/about/);
- const title = await browser.title();
- expect(title).toBe('About page');
-});
-
-test('routes intercept and fulfill', async ({ app, browser, screen }) => {
- await app.open();
- await browser.route('**/api/flags', (route) => route.fulfill({ json: { betaBoard: true } }));
- await browser.goto('/flags');
- await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta on');
- await browser.unroute('**/api/flags');
- await browser.reload();
- await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta off');
-});
-
test('routes are attempt-scoped: registered before the first page, kept across restart and clearState', async ({
app,
browser,
@@ -354,26 +131,8 @@ test('routes are attempt-scoped: registered before the first page, kept across r
await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta on');
});
-test('waitForResponse observes network traffic', async ({ app, browser }) => {
+test('evaluate keeps a nested named binding the loader compiled', async ({ app, browser }) => {
await app.open();
- const [response] = await Promise.all([
- browser.waitForResponse('**/api/flags'),
- browser.goto('/flags'),
- ]);
- expect(response.status).toBe(200);
- const body = await response.json();
- expect(body).toEqual({ betaBoard: false });
-});
-
-test('evaluate runs trusted code with JSON arguments', async ({ app, browser }) => {
- await app.open();
- const result = await browser.evaluate((input: { a: number; b: number }) => input.a + input.b, {
- a: 2,
- b: 40,
- });
- expect(result).toBe(42);
- const href = await browser.evaluate(() => document.location.pathname);
- expect(href).toBe('/');
// A nested named binding survives serialization whatever the loader
// compiled it to (esbuild keepNames once wrapped it in a \`__name\` helper).
const total = await browser.evaluate((input: { a: number; b: number }) => {
@@ -383,55 +142,29 @@ test('evaluate runs trusted code with JSON arguments', async ({ app, browser })
expect(total).toBe(42);
});
-test('cookies round-trip through policy checks', async ({ app, browser }) => {
- await app.open();
- await browser.setCookies([{ name: 'flavor', value: 'oatmeal', url: (await browser.url()) }]);
- const cookies = await browser.cookies();
- const flavor = cookies.find((cookie) => cookie.name === 'flavor');
- expect(flavor?.value).toBe('oatmeal');
-});
-
-test('dialogs are handled by registered handlers', async ({ app, browser, screen }) => {
- await app.open('/dialog');
- const dispose = await browser.onDialog('accept');
- await screen.getByRole('button', { name: 'Ask' }).tap();
- await expect(screen.getByRole('status', { name: 'Answer' })).toHaveText('accepted');
- await dispose();
-});
-
-test('frame locators scope queries into iframes', async ({ app, browser }) => {
+test('reads under an absent frame answer with no match', async ({ app, browser }) => {
await app.open('/frame');
- const frame = browser.frameLocator('#child');
- await frame.getByRole('button', { name: 'Frame button' }).tap();
- await new Promise((resolve) => setTimeout(resolve, 100));
-
const absent = browser.frameLocator('#absent').getByRole('button');
- const started = Date.now();
expect(await absent.count()).toBe(0);
expect(await absent.all()).toEqual([]);
expect(await absent.allTextContents()).toEqual([]);
expect(await absent.isVisible()).toBe(false);
expect(await absent.isHidden()).toBe(true);
- expect(Date.now() - started).toBeLessThan(2_000);
});
test('a nested absent frame counts as zero matches at once', async ({ app, browser }) => {
await app.open('/frame-nested');
- const started = Date.now();
expect(await browser.frameLocator('#outer').frameLocator('#absent').getByRole('button').count()).toBe(0);
- expect(Date.now() - started).toBeLessThan(2_000);
});
test('absence matchers and waits under an absent frame pass at once', async ({ app, browser }) => {
await app.open('/frame');
const absent = browser.frameLocator('#absent').getByRole('button');
- const started = Date.now();
await expect(absent).toBeAttached({ attached: false, timeout: 3000 });
await expect(absent).toBeVisible({ visible: false, timeout: 3000 });
await expect(absent).toBeHidden({ timeout: 3000 });
await absent.waitFor({ state: 'detached', timeout: 3000 });
await absent.waitFor({ state: 'hidden', timeout: 3000 });
- expect(Date.now() - started).toBeLessThan(2_000);
});
test('an action under an absent frame polls for the frame until its timeout', async ({ app, browser }) => {
@@ -439,41 +172,6 @@ test('an action under an absent frame polls for the frame until its timeout', as
await browser.frameLocator('#absent').getByRole('button').tap({ timeout: 800 });
});
-test('contenteditable hosts are textboxes: reached by label, filled, and read as a value', async ({ app, screen }) => {
- await app.open('/editor');
- const notes = screen.getByLabel('Notes');
- await expect(notes).toBeVisible();
- await notes.fill('Hello');
- await expect(notes).toHaveValue('Hello');
- expect(await notes.inputValue()).toBe('Hello');
- await expect(screen.getByTestId('notes')).toHaveValue('Hello');
- // getByRole resolves through Playwright's role selector, which knows a
- // textbox only by an explicit role: a bare host is not found, one with
- // role="textbox" is.
- await expect(screen.getByRole('textbox', { name: 'Notes' })).toHaveCount(0);
- await expect(screen.getByRole('textbox', { name: 'Message' })).toHaveCount(1);
- await screen.getByRole('textbox', { name: 'Message' }).fill('Hi');
- await expect(screen.getByLabel('Message')).toHaveValue('Hi');
-});
-
-test('an editor value keeps the whitespace it renders, and an empty editor reads as empty', async ({ app, screen }) => {
- await app.open('/editor');
- const code = screen.getByLabel('Code');
- await expect(code).toHaveValue(' keep spaces ');
- await expect(code).not.toHaveValue('keep spaces');
- expect(await code.inputValue()).toBe(' keep spaces ');
- const notes = screen.getByLabel('Notes');
- await expect(notes).toHaveValue('');
- expect(await notes.inputValue()).toBe('');
-});
-
-test('nested frame locators resolve each frame inside the one before it', async ({ app, browser }) => {
- await app.open('/frame-nested');
- const inner = browser.frameLocator('#outer').frameLocator('#child');
- await inner.getByRole('button', { name: 'Frame button' }).tap();
- await expect(inner.getByRole('button', { name: 'Frame clicked' })).toBeVisible();
-});
-
test('downloads are captured as artifacts', async ({ app, browser }) => {
await app.open('/downloads');
const download = await browser.waitForDownload(() => browser.locator('a[download]').tap());
@@ -534,37 +232,16 @@ test('a viewport set before the first navigation opens no page', async ({ screen
await screen.getByTestId('items').tap({ timeout: 200 });
});
-test('css selectors via browser.locator', async ({ app, browser }) => {
+test('a *css= capture is refused before the last selector part', async ({ app, browser }) => {
await app.open();
- await expect(browser.locator('ul[data-testid="items"] li').first()).toHaveText('Item Alpha');
await expect(browser.locator('css=ul[data-testid="items"] >> *css=li')).toHaveCount(3);
let capture = 'no error';
try {
await browser.locator('*css=ul[data-testid="items"] >> li').count();
} catch (error) {
- capture = (error as { code?: string }).code ?? 'no code';
+ capture = (error as Error).message;
}
- expect(capture).toBe('INVALID_LOCATOR');
-});
-
-test('app lifecycle: restart preserves storage, clearState clears it', async ({ app, screen }) => {
- await app.open('/storage');
- await screen.getByRole('button', { name: 'Save marker' }).tap();
- await expect(screen.getByRole('status', { name: 'Marker' })).toHaveText('saved');
-
- await app.restart();
- await app.open('/storage');
- await expect(screen.getByRole('status', { name: 'Marker' })).toHaveText('saved');
-
- await app.clearState();
- await app.open('/storage');
- await expect(screen.getByRole('status', { name: 'Marker' })).toHaveText('empty');
-});
-
-test('screenshots land in the artifact directory', async ({ app }) => {
- await app.open();
- const shot = await app.screenshot('home');
- if (!shot.includes('screenshots/')) throw new Error('unexpected screenshot path: ' + shot);
+ expect(capture).toContain('a selector that captures with * (*css=article >> text=Hello) is not supported');
});
test('forbidden URL schemes are refused', async ({ app }) => {
@@ -589,14 +266,6 @@ test('a dialog handler assertion that no step follows', async ({ app, browser, s
await screen.getByRole('button', { name: 'Ask' }).tap();
});
-test('a step call without await', async ({ app }) => {
- app.open();
-});
-
-test('a step call without await before the body throws', async ({ app }) => {
- app.open();
- throw new Error('the body gave up');
-});
`;
const LATE_HANDLER_THEN_TEARDOWN = `import { test } from '@e2e-dev/web';
@@ -647,6 +316,100 @@ test('browser.goto refuses view-source', async ({ app, browser }) => {
});
`;
+// Loaded from a config file, as a project loads it: the engine's `e2e/engine`
+// and the runner's core are two module copies, so `instanceof` cannot tell a
+// runner error apart and a missing node must still keep the matcher polling.
+const CLASSES = `import { test } from '@e2e-dev/web';
+import { expect } from 'e2e';
+
+test('class assertions poll for a node that arrives late', async ({ app, browser }) => {
+ await app.open('/classes');
+ await expect(browser).toHaveClass(browser.locator('#late-card'), 'card late', { timeout: 1800 });
+});
+
+test('class assertions on an ambiguous locator fail at once', async ({ app, browser }) => {
+ await app.open('/classes');
+ await expect(browser).toHaveClass(browser.locator('.dup'), 'dup', { timeout: 1800 });
+});
+
+test('an empty class attribute is an empty class list', async ({ app, browser }) => {
+ await app.open('/classes');
+ await expect(browser).toHaveClass(browser.locator('#blank-card'), '');
+});
+
+test('a missing class attribute is not an empty class list', async ({ app, browser, screen }) => {
+ await app.open('/classes');
+ await expect(browser).toHaveClass(screen.getByTestId('items'), '', { timeout: 300 });
+});
+
+test('toHaveURL ignoreCase folds the comparison', async ({ app, browser }) => {
+ await app.open('/classes');
+ await expect(browser).toHaveURL('/CLASSES', { ignoreCase: true });
+ await expect(browser).toHaveURL(/CLASSES$/, { ignoreCase: true });
+ await expect(browser).not.toHaveURL('/CLASSES', { timeout: 300 });
+});
+
+test('a negated toHaveURL ignoreCase fails on a case-folded match', async ({ app, browser }) => {
+ await app.open('/classes');
+ await expect(browser).not.toHaveURL('/CLASSES', { ignoreCase: true, timeout: 300 });
+});
+`;
+
+// A predicate, which Playwright accepts and a migrated test may still pass, is
+// refused instead of reading as a pattern that matches everything. An option
+// the call does not take is refused instead of leaving it on its default.
+const URL_PATTERNS = `import { test } from '@e2e-dev/web';
+import { expect } from 'e2e';
+
+const toOther: any = (url: URL) => url.pathname === '/other';
+const flag = (name, value) => JSON.parse(JSON.stringify({ [name]: value, timeout: 300 }));
+
+test('waitForResponse with a predicate', async ({ app, browser }) => {
+ await app.open();
+ await Promise.all([browser.waitForResponse(toOther, { timeout: 2_000 }), browser.goto('/flags')]);
+});
+
+test('route with a predicate', async ({ app, browser, screen }) => {
+ await browser.route(toOther, (route) => route.abort());
+ await app.open('/flags');
+ await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta off');
+});
+
+test('unroute with a predicate', async ({ app, browser }) => {
+ await app.open();
+ await browser.unroute(toOther);
+});
+
+test('string and RegExp URL patterns match what they name', async ({ app, browser, screen }) => {
+ await app.open('/about');
+ await expect(browser).toHaveURL('/about');
+ await expect(browser).toHaveURL(/\\/about$/);
+ await expect(browser).not.toHaveURL('/other', { timeout: 200 });
+ await expect(browser).not.toHaveURL(/\\/other$/, { timeout: 200 });
+ await browser.waitForURL(/\\/about$/);
+ await browser.route(/\\/other$/, (route) => route.abort());
+ await browser.route(/\\/api\\/flags$/, (route) => route.fulfill({ json: { betaBoard: true } }));
+ const [response] = await Promise.all([browser.waitForResponse(/\\/api\\/flags$/), browser.goto('/flags')]);
+ expect(response.url).toMatch(/\\/api\\/flags$/);
+ expect(await response.json()).toEqual({ betaBoard: true });
+ await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta on');
+ await browser.unroute(/\\/api\\/flags$/);
+ const [real] = await Promise.all([browser.waitForResponse('**/api/flags'), browser.reload()]);
+ expect(await real.json()).toEqual({ betaBoard: false });
+ await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta off');
+});
+
+test('an option toHaveTitle does not take is refused', async ({ app, browser }) => {
+ await app.open();
+ await expect(browser).toHaveTitle('', flag('ignoreCase', true));
+});
+
+test('an option waitForURL does not take is refused', async ({ app, browser }) => {
+ await app.open();
+ await browser.waitForURL('/', flag('waitUntil', 'load'));
+});
+`;
+
describe('web platform integration', () => {
let app: FixtureApp;
let outcome: RunOutcome;
@@ -654,16 +417,18 @@ describe('web platform integration', () => {
beforeAll(async () => {
app = await startFixtureApp();
- ({ outcome, project } = await runProject(
+ ({ outcome, project } = await runProjectWithConfigFile(
{
'tests/kitchen.e2e.ts': KITCHEN_SINK,
'tests/late-handler.e2e.ts': LATE_HANDLER_THEN_TEARDOWN,
'tests/schemes.e2e.ts': WRAPPED_SCHEMES,
+ 'tests/classes.e2e.ts': CLASSES,
+ 'tests/url-patterns.e2e.ts': URL_PATTERNS,
'marker.txt': 'marker-local-file\n',
},
{
appUrl: app.url,
- config: { actionTimeout: 5_000, assertionTimeout: 4_000, timeout: 30_000 },
+ configSource: workerConfigSource(1, "\n cache: 'off',\n actionTimeout: 5_000,\n assertionTimeout: 4_000,\n timeout: 30_000,"),
},
));
}, 240_000);
@@ -673,37 +438,32 @@ describe('web platform integration', () => {
await app?.close();
});
- it('passes every deterministic behavior test', () => {
- const shouldPass = [
- 'deterministic queries and reads',
- 'a textarea value compares raw',
- 'toHaveAttribute reads the value attribute of plain fields',
- 'role vocabulary: tabs, menus, progress, toolbars, images',
- 'actions and state',
- 'check and uncheck refuse an unknown option and leave the box untouched',
- 'unsupported filter and selectOption shapes fail before acting',
- 'assertions poll until the app settles',
- 'web navigation, urls, and titles',
- 'routes intercept and fulfill',
- 'routes are attempt-scoped: registered before the first page, kept across restart and clearState',
- 'waitForResponse observes network traffic',
- 'evaluate runs trusted code with JSON arguments',
- 'cookies round-trip through policy checks',
- 'dialogs are handled by registered handlers',
- 'frame locators scope queries into iframes',
- 'a nested absent frame counts as zero matches at once',
- 'absence matchers and waits under an absent frame pass at once',
- 'nested frame locators resolve each frame inside the one before it',
- 'css selectors via browser.locator',
- 'contenteditable hosts are textboxes: reached by label, filled, and read as a value',
- 'downloads are captured as artifacts',
- 'app lifecycle: restart preserves storage, clearState clears it',
- 'screenshots land in the artifact directory',
- ];
- for (const title of shouldPass) {
- const result = resultByTitle(outcome, title);
- expect(result.status, `${title}: ${JSON.stringify(result.attempts[0]?.error)}`).toBe('passed');
- }
+ it.each([
+ 'deterministic queries and reads',
+ 'class assertions report the observed class on failure',
+ 'a textarea value compares raw',
+ 'toHaveAttribute reads the value attribute of plain fields',
+ 'routes are attempt-scoped: registered before the first page, kept across restart and clearState',
+ 'reads under an absent frame answer with no match',
+ 'a nested absent frame counts as zero matches at once',
+ 'absence matchers and waits under an absent frame pass at once',
+ 'a *css= capture is refused before the last selector part',
+ 'downloads are captured as artifacts',
+ 'clicks hold the modifiers they name',
+ 'a viewport set before the first navigation holds through app.open',
+ 'an empty class attribute is an empty class list',
+ 'toHaveURL ignoreCase folds the comparison',
+ 'string and RegExp URL patterns match what they name',
+ ])('passes %s', (title) => {
+ const result = resultByTitle(outcome, title);
+ expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe('passed');
+ });
+
+ it('decides absence under an absent frame without waiting out the deadline', () => {
+ const result = resultByTitle(outcome, 'absence matchers and waits under an absent frame pass at once');
+ const steps = result.attempts[0]!.steps.filter((step) => step.api.startsWith('expect.') || step.api === 'locator.waitFor');
+ expect(steps).toHaveLength(5);
+ for (const step of steps) expect(step.durationMs, step.api).toBeLessThan(3000);
});
it('fails an action under an absent frame with LOCATOR_NOT_FOUND once its timeout has run', () => {
@@ -715,10 +475,6 @@ describe('web platform integration', () => {
expect(tap?.durationMs).toBeGreaterThanOrEqual(800);
});
- it('holds click modifiers on the web engine', () => {
- expect(resultByTitle(outcome, 'clicks hold the modifiers they name').status).toBe('passed');
- });
-
it('fails a download that never started with the wait and the trigger time, not a bare timeout', () => {
const result = resultByTitle(outcome, 'a trigger that starts no download times out saying so');
expect(result.status).toBe('failed');
@@ -730,7 +486,6 @@ describe('web platform integration', () => {
});
it('sets a viewport before the first navigation without opening a page', () => {
- expect(resultByTitle(outcome, 'a viewport set before the first navigation holds through app.open').status).toBe('passed');
const result = resultByTitle(outcome, 'a viewport set before the first navigation opens no page');
expect(result.attempts[0]!.steps.map((step) => [step.api, step.status])).toEqual([
['browser.setViewport', 'passed'],
@@ -739,18 +494,6 @@ describe('web platform integration', () => {
expect(result.attempts[0]!.error?.code).toBe('APP_NOT_OPEN');
});
- it('fails ambiguous locators immediately with LOCATOR_AMBIGUOUS', () => {
- const result = resultByTitle(outcome, 'ambiguous locators fail immediately');
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error?.code).toBe('LOCATOR_AMBIGUOUS');
- });
-
- it('fails missing locators with LOCATOR_NOT_FOUND after polling', () => {
- const result = resultByTitle(outcome, 'missing locators poll until the deadline');
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error?.code).toBe('LOCATOR_NOT_FOUND');
- });
-
it('fails toHaveValue on a textarea against the raw value, and prints what it compared', () => {
// The web engine reports the value raw, and the matcher neither collapses
// the newlines to match nor prints a string it never compared.
@@ -796,29 +539,6 @@ describe('web platform integration', () => {
expect(control.attempts[0]!.error?.message).toContain('observed: value "marker-9b1f"');
});
- it('fails a body that returns before a step it started finished with STEP_NOT_AWAITED', () => {
- const result = resultByTitle(outcome, 'a step call without await');
- expect(result.status).toBe('failed');
- const attempt = result.attempts[0]!;
- expect(attempt.error).toMatchObject({ code: 'STEP_NOT_AWAITED', phase: 'body' });
- expect(attempt.error?.message).toContain('app.open');
- // A vitest fork reports raw transformed positions; test-source.test.ts covers the mapped line.
- expect(attempt.error?.source?.file).toBe('tests/kitchen.e2e.ts');
- expect(attempt.steps).toHaveLength(1);
- expect(attempt.steps[0]).toMatchObject({ api: 'app.open', status: 'failed', error: { code: 'STEP_NOT_AWAITED' } });
- expect(outcome.report.run.errors.map((error) => error.code)).toEqual([]);
- });
-
- it('keeps the body error primary and notes the un-awaited step beside it', () => {
- const result = resultByTitle(outcome, 'a step call without await before the body throws');
- expect(result.status).toBe('failed');
- const attempt = result.attempts[0]!;
- expect(attempt.error).toMatchObject({ code: 'ERROR', message: 'the body gave up' });
- expect(attempt.secondaryErrors.map((error) => error.code)).toEqual(['STEP_NOT_AWAITED']);
- expect(attempt.steps[0]).toMatchObject({ api: 'app.open', status: 'failed', error: { code: 'STEP_NOT_AWAITED' } });
- expect(outcome.report.run.errors.map((error) => error.code)).toEqual([]);
- });
-
it.each([
['route', 'a route handler assertion that no step follows'],
['dialog', 'a dialog handler assertion that no step follows'],
@@ -845,12 +565,6 @@ describe('web platform integration', () => {
expect(text).not.toContain('heading "Home"');
});
- it('fails UI operations before open with APP_NOT_OPEN', () => {
- const result = resultByTitle(outcome, 'acting before open fails with APP_NOT_OPEN');
- expect(result.status).toBe('failed');
- expect(result.attempts[0]!.error?.code).toBe('APP_NOT_OPEN');
- });
-
it('denies navigation to a forbidden scheme', () => {
const result = resultByTitle(outcome, 'forbidden URL schemes are refused');
expect(result.status).toBe('failed');
@@ -915,15 +629,54 @@ describe('web platform integration', () => {
expect(download).toMatchObject({ redaction: 'incomplete', mediaType: 'text/csv' });
});
- it('records steps and screenshot artifacts on attempts', () => {
- const result = resultByTitle(outcome, 'screenshots land in the artifact directory');
+ it('keeps a class assertion polling while the node is missing and passes once it arrives', () => {
+ const result = resultByTitle(outcome, 'class assertions poll for a node that arrives late');
+ expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe('passed');
+ const step = result.attempts[0]!.steps.find((candidate) => candidate.api === 'expect.toHaveClass');
+ expect(step?.durationMs).toBeGreaterThan(500);
+ });
+
+ it('fails a class assertion on an ambiguous locator with LOCATOR_AMBIGUOUS before the deadline', () => {
+ const result = resultByTitle(outcome, 'class assertions on an ambiguous locator fail at once');
+ expect(result.status).toBe('failed');
+ expect(result.attempts[0]!.error?.code).toBe('LOCATOR_AMBIGUOUS');
+ const step = result.attempts[0]!.steps.find((candidate) => candidate.api === 'expect.toHaveClass');
+ expect(step?.durationMs).toBeLessThan(1800);
+ });
+
+ it('matches an empty class string against no class attribute only when one is present', () => {
+ const missing = resultByTitle(outcome, 'a missing class attribute is not an empty class list');
+ expect(missing.status).toBe('failed');
+ expect(missing.attempts[0]!.error?.code).toBe('ASSERTION_FAILED');
+ expect(missing.attempts[0]!.error?.message).toContain('observed: no class attribute');
+ });
+
+ it('fails a negated toHaveURL ignoreCase on a case-folded match', () => {
+ const negated = resultByTitle(outcome, 'a negated toHaveURL ignoreCase fails on a case-folded match');
+ expect(negated.status).toBe('failed');
+ expect(negated.attempts[0]!.error?.code).toBe('ASSERTION_FAILED');
+ expect(negated.attempts[0]!.error?.message).toContain('expected: not URL /CLASSES (ignoring case)');
+ });
+
+ it.each([
+ ['waitForResponse with a predicate', 'browser.waitForResponse'],
+ ['route with a predicate', 'browser.route'],
+ ['unroute with a predicate', 'browser.unroute'],
+ ])('%s fails with INVALID_ARGUMENT at %s', (title, api) => {
+ const result = resultByTitle(outcome, title);
+ expect(result.status).toBe('failed');
const attempt = result.attempts[0]!;
- expect(attempt.steps.length).toBeGreaterThan(1);
- expect(attempt.steps.map((step) => step.api)).toContain('app.screenshot');
- const screenshot = attempt.artifacts.find((artifact) => artifact.kind === 'screenshot');
- expect(screenshot).toBeDefined();
- expect(screenshot!.path).toBeDefined();
- const artifactsRoot = path.join(project.dir, '.e2e', 'artifacts');
- expect(existsSync(path.join(artifactsRoot, screenshot!.path!))).toBe(true);
+ expect(attempt.error?.code).toBe('INVALID_ARGUMENT');
+ expect(attempt.error?.message).toContain('string or RegExp');
+ expect(attempt.steps.find((step) => step.status === 'failed')?.api).toBe(api);
+ });
+
+ it.each([
+ ['an option toHaveTitle does not take is refused', 'expect.toHaveTitle options has no key "ignoreCase"'],
+ ['an option waitForURL does not take is refused', 'browser.waitForURL options has no key "waitUntil"'],
+ ])('%s', (title, message) => {
+ const result = resultByTitle(outcome, title);
+ expect(result.status).toBe('failed');
+ expect(result.attempts[0]!.error).toMatchObject({ code: 'INVALID_ARGUMENT', message: expect.stringContaining(message) });
});
});
diff --git a/packages/e2e/tests/integration/web-replaced-nodes.test.ts b/packages/e2e/tests/integration/web-replaced-nodes.test.ts
index e009e20fa..b11287d5d 100644
--- a/packages/e2e/tests/integration/web-replaced-nodes.test.ts
+++ b/packages/e2e/tests/integration/web-replaced-nodes.test.ts
@@ -16,7 +16,7 @@ import { resultByTitle, runProject, type FixtureProject, type RunOutcome } from
const SUITE = `import { test } from '@e2e-dev/web';
import { expect } from 'e2e';
-const READS = 30;
+const READS = 10;
/**
* Runs body while this process's event loop is blocked in 20 ms slices, so a
@@ -58,7 +58,7 @@ test('a field replaced every frame never reads as hidden', async ({ app, screen
test('a field replaced every frame never reads as hidden on a starved runner', async ({ app, screen }) => {
await app.open('/replaced');
- expect(await starved(() => hiddenReads(fields(screen), 10))).toEqual({ testId: 0, label: 0, displayValue: 0 });
+ expect(await starved(() => hiddenReads(fields(screen), 3))).toEqual({ testId: 0, label: 0, displayValue: 0 });
});
test('a stable field beside one replaced every frame reads as shown', async ({ app, screen }) => {
@@ -109,7 +109,7 @@ describe('reads of a node replaced every frame', () => {
beforeAll(async () => {
app = await startFixtureApp();
- // Ninety sequential reads against a page that swaps nodes every frame: on a
+ // Thirty sequential reads against a page that swaps nodes every frame: on a
// loaded CI runner each read takes far longer than locally, and the suite
// asserts what the reads return, not how fast they are. The hook's own
// budget covers all eight tests at that deadline.
diff --git a/packages/e2e/tests/integration/web-url-patterns.test.ts b/packages/e2e/tests/integration/web-url-patterns.test.ts
deleted file mode 100644
index 199c6ec3d..000000000
--- a/packages/e2e/tests/integration/web-url-patterns.test.ts
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * URL patterns in a real browser: `toHaveURL`, `waitForURL`, `route`,
- * `unroute`, and `waitForResponse` take a string or a RegExp, as does the
- * text pattern of `toHaveTitle`. A predicate function, which Playwright
- * accepts and a migrated test may still pass, is refused with
- * INVALID_ARGUMENT instead of reading as a pattern that matches everything.
- */
-
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts';
-import { resultByTitle, runProject, type FixtureProject, type RunOutcome } from '../helpers/run-project.ts';
-
-const SUITE = `import { test } from '@e2e-dev/web';
-import { expect } from 'e2e';
-
-const toOther: any = (url: URL) => url.pathname === '/other';
-
-test('toHaveURL with a predicate', async ({ app, browser }) => {
- await app.open('/about');
- await expect(browser).toHaveURL(toOther, { timeout: 500 });
-});
-
-test('waitForURL with a predicate', async ({ app, browser }) => {
- await app.open('/about');
- await browser.waitForURL(toOther, { timeout: 500 });
-});
-
-test('waitForResponse with a predicate', async ({ app, browser }) => {
- await app.open();
- await Promise.all([browser.waitForResponse(toOther, { timeout: 2_000 }), browser.goto('/flags')]);
-});
-
-test('route with a predicate', async ({ app, browser, screen }) => {
- await browser.route(toOther, (route) => route.abort());
- await app.open('/flags');
- await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta off');
-});
-
-test('unroute with a predicate', async ({ app, browser }) => {
- await app.open();
- await browser.unroute(toOther);
-});
-
-test('toHaveTitle with a predicate', async ({ app, browser }) => {
- await app.open();
- await expect(browser).toHaveTitle(toOther, { timeout: 500 });
-});
-
-test('string and RegExp URL patterns match what they name', async ({ app, browser, screen }) => {
- await app.open('/about');
- await expect(browser).toHaveURL('/about');
- await expect(browser).toHaveURL(/\\/about$/);
- await expect(browser).not.toHaveURL('/other');
- await expect(browser).not.toHaveURL(/\\/other$/);
- await browser.waitForURL(/\\/about$/);
- await browser.route(/\\/other$/, (route) => route.abort());
- await browser.route(/\\/api\\/flags$/, (route) => route.fulfill({ json: { betaBoard: true } }));
- const [response] = await Promise.all([browser.waitForResponse(/\\/api\\/flags$/), browser.goto('/flags')]);
- expect(response.url).toMatch(/\\/api\\/flags$/);
- expect(await response.json()).toEqual({ betaBoard: true });
- await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta on');
- await browser.unroute(/\\/api\\/flags$/);
- const [real] = await Promise.all([browser.waitForResponse('**/api/flags'), browser.reload()]);
- expect(await real.json()).toEqual({ betaBoard: false });
- await expect(screen.getByRole('status', { name: 'Flags' })).toHaveText('beta off');
-});
-`;
-
-describe('web URL patterns in a browser', () => {
- let app: FixtureApp;
- let outcome: RunOutcome;
- let project: FixtureProject;
-
- beforeAll(async () => {
- app = await startFixtureApp();
- ({ outcome, project } = await runProject(
- { 'tests/url-patterns.e2e.ts': SUITE },
- { appUrl: app.url, config: { actionTimeout: 5_000, assertionTimeout: 4_000, timeout: 30_000 } },
- ));
- }, 240_000);
-
- afterAll(async () => {
- project?.cleanup();
- await app?.close();
- });
-
- it.each([
- ['toHaveURL with a predicate', 'expect.toHaveURL'],
- ['waitForURL with a predicate', 'browser.waitForURL'],
- ['waitForResponse with a predicate', 'browser.waitForResponse'],
- ['route with a predicate', 'browser.route'],
- ['unroute with a predicate', 'browser.unroute'],
- ['toHaveTitle with a predicate', 'expect.toHaveTitle'],
- ])('%s fails with INVALID_ARGUMENT at %s', (title, api) => {
- const result = resultByTitle(outcome, title);
- expect(result.status).toBe('failed');
- const attempt = result.attempts[0]!;
- expect(attempt.error?.code).toBe('INVALID_ARGUMENT');
- expect(attempt.error?.message).toContain('string or RegExp');
- expect(attempt.steps.find((step) => step.status === 'failed')?.api).toBe(api);
- });
-
- it('keeps string and RegExp patterns matching only what they name', () => {
- const result = resultByTitle(outcome, 'string and RegExp URL patterns match what they name');
- expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe('passed');
- });
-});
diff --git a/packages/e2e/tests/unit/config.test.ts b/packages/e2e/tests/unit/config.test.ts
index 41cc17f14..683f04526 100644
--- a/packages/e2e/tests/unit/config.test.ts
+++ b/packages/e2e/tests/unit/config.test.ts
@@ -988,6 +988,8 @@ describe('resolveConfig', () => {
expect(own.targets[0]![kind]).toEqual({ mode: 'on-all-retries', source: 'target' });
const flagged = resolveConfig({ targets: [web('off')], [kind]: 'off' }, { projectRoot: ROOT, env: BASE_ENV, cli: { [kind]: 'on' } });
expect(flagged.targets[0]![kind]).toEqual({ mode: 'on', source: 'run' });
+ const off = resolveConfig({ targets: TARGETS, [kind]: 'on' }, { projectRoot: ROOT, env: BASE_ENV, cli: { [kind]: 'off' } });
+ expect(off.targets[0]![kind]).toEqual({ mode: 'off', source: 'run' });
});
it('keeps the mode, at the top and on a target, out of the digest', () => {
From e488bdef1bcf1182f62db01089d59a87ca45debe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nguy=E1=BB=85n=20V=C4=83n=20Hi=E1=BA=BFu?=
<132248583+nguyenvanhieu1710@users.noreply.github.com>
Date: Mon, 5 Oct 2026 15:53:19 +0700
Subject: [PATCH 02/46] chore(deps): update fast-uri to address security
advisories (#846)
---
pnpm-lock.yaml | 10 +++++-----
pnpm-workspace.yaml | 2 +-
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 269c2a5f0..f85c43a78 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -106,7 +106,7 @@ settings:
excludeLinksFromLockfile: false
overrides:
- fast-uri@>=3.0.0 <3.1.6: 3.1.6
+ fast-uri@>=3.0.0 <3.1.8: 3.1.8
js-yaml@<3.15.2: 3.15.2
js-yaml@>=4.0.0 <4.3.2: 4.3.2
nanoid@<3.3.18: 3.3.18
@@ -4472,8 +4472,8 @@ packages:
fast-string-width@3.0.2:
resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==}
- fast-uri@3.1.6:
- resolution: {integrity: sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==}
+ fast-uri@3.1.8:
+ resolution: {integrity: sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==}
fast-wrap-ansi@0.2.2:
resolution: {integrity: sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==}
@@ -10711,7 +10711,7 @@ snapshots:
ajv@8.20.0:
dependencies:
fast-deep-equal: 3.1.3
- fast-uri: 3.1.6
+ fast-uri: 3.1.8
json-schema-traverse: 1.0.0
require-from-string: 2.0.2
@@ -12037,7 +12037,7 @@ snapshots:
dependencies:
fast-string-truncated-width: 3.0.3
- fast-uri@3.1.6: {}
+ fast-uri@3.1.8: {}
fast-wrap-ansi@0.2.2:
dependencies:
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index db5c37e8d..c99fdef0e 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -19,7 +19,7 @@ allowBuilds:
sharp: false
overrides:
- 'fast-uri@>=3.0.0 <3.1.6': 3.1.6
+ 'fast-uri@>=3.0.0 <3.1.8': 3.1.8
'js-yaml@<3.15.2': 3.15.2
'js-yaml@>=4.0.0 <4.3.2': 4.3.2
'nanoid@<3.3.18': 3.3.18
From c681005af83962628c82ea8ea7cc419ac587a1eb Mon Sep 17 00:00:00 2001
From: Feco Linhares <963774+fecolinhares@users.noreply.github.com>
Date: Mon, 5 Oct 2026 06:32:01 -0300
Subject: [PATCH 03/46] test(web): wait for a painted frame before closing a
video segment (#804)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* test(web): hold each segment's wait for a frame that segment received
The observer remembers every page it saw paint, so the wait that closed a
segment was satisfied by the first frame that page ever delivered. A segment
that painted nothing was then recorded as though it had, and the video
assertions fail on it later with `inked` below its threshold.
The wait now forgets what earlier segments painted. The added test stops the
segment that did paint, opens another, and asserts that closing it runs its
timeout out; removing the forget makes it fail.
Rebased onto the file as it stands after #798, which moved the observer
inside the test that owns it, so the wait is now the one taking a page rather
than reaching for it. Spies are restored and forgotten together, so a failed
test cannot leave one behind for the next.
* test(web): wait for a painted frame before stopping each mid-attempt video segment
The flaking test is the mid-attempt split one, not the per-page one: its
first segment sometimes stopped before the screencast delivered a frame,
and Playwright writes a white frame then. Share one paint observer between
both video tests and drop the test that only checked its own copy.
---------
Co-authored-by: Feco Linhares
Co-authored-by: Oskar Kwaśniewski
---
.../web/tests/integration/lifecycle.test.ts | 108 +++++++++++-------
1 file changed, 64 insertions(+), 44 deletions(-)
diff --git a/packages/web/tests/integration/lifecycle.test.ts b/packages/web/tests/integration/lifecycle.test.ts
index e452acb74..fc36f5aa3 100644
--- a/packages/web/tests/integration/lifecycle.test.ts
+++ b/packages/web/tests/integration/lifecycle.test.ts
@@ -97,6 +97,59 @@ async function lastFrame(
}, [...webm]);
}
+/**
+ * Watches the frames the engine's screencast delivers, so a test can hold a
+ * video segment open until it received a painted one: Playwright writes a
+ * white frame for a segment stopped before its first frame arrived.
+ */
+function watchPaint() {
+ const painted = new WeakSet();
+ const restores: (() => void)[] = [];
+ const watchPage = (page: Page): void => {
+ const start = page.screencast.start.bind(page.screencast);
+ const spy = vi.spyOn(page.screencast, 'start').mockImplementation((options) => start({
+ ...options,
+ onFrame: async (frame) => {
+ const hasPaint = await page.evaluate(async (bytes) => {
+ const bitmap = await createImageBitmap(new Blob([new Uint8Array(bytes)], { type: 'image/jpeg' }));
+ const canvas = document.createElement('canvas');
+ canvas.width = bitmap.width;
+ canvas.height = bitmap.height;
+ const context = canvas.getContext('2d')!;
+ context.drawImage(bitmap, 0, 0);
+ bitmap.close();
+ const { data } = context.getImageData(0, 0, canvas.width, canvas.height);
+ for (let index = 0; index < data.length; index += 4) {
+ if (data[index]! < 200 || data[index + 1]! < 200 || data[index + 2]! < 200) return true;
+ }
+ return false;
+ }, [...frame.data]).catch(() => false);
+ if (hasPaint) painted.add(page);
+ await options?.onFrame?.(frame);
+ },
+ }));
+ restores.push(() => spy.mockRestore());
+ };
+ return {
+ /** Observes the context's pages, including those it opens later, before their recording starts. */
+ watchContext(context: BrowserContext): void {
+ context.on('page', watchPage);
+ for (const page of context.pages()) watchPage(page);
+ },
+ /** Waits for a painted frame delivered after this call, so a page that painted for an earlier segment cannot pass for this one. */
+ async settle(page: Page): Promise {
+ painted.delete(page);
+ // A static page may not repaint after navigation overtakes its first screencast frame.
+ await page.screenshot();
+ await expect.poll(() => painted.has(page), { timeout: 5_000, message: 'the recorder received a painted frame' }).toBe(true);
+ },
+ /** Removes every observer, so none reaches the next test. */
+ restore(): void {
+ while (restores.length > 0) restores.pop()!();
+ },
+ };
+}
+
async function withAttempt(
engine: EngineHandle,
app: FixtureApp,
@@ -934,55 +987,18 @@ describe('web engine lifecycle', () => {
const engine = web();
const videoDir = mkdtempSync(path.join(tmpdir(), 'e2e-video-'));
const nodesOf = (snapshot: { root: SemanticNode }) => [...walk(snapshot.root)];
- const painted = new WeakSet();
+ const paint = watchPaint();
const restores: (() => void)[] = [];
- /** Observe the same delivered frames the recorder writes, including on replacement pages. */
- const watchPage = (page: Page): void => {
- const start = page.screencast.start.bind(page.screencast);
- const spy = vi.spyOn(page.screencast, 'start').mockImplementation((options) => start({
- ...options,
- onFrame: async (frame) => {
- const hasPaint = await page.evaluate(async (bytes) => {
- const bitmap = await createImageBitmap(new Blob([new Uint8Array(bytes)], { type: 'image/jpeg' }));
- const canvas = document.createElement('canvas');
- canvas.width = bitmap.width;
- canvas.height = bitmap.height;
- const context = canvas.getContext('2d')!;
- context.drawImage(bitmap, 0, 0);
- bitmap.close();
- const { data } = context.getImageData(0, 0, canvas.width, canvas.height);
- for (let index = 0; index < data.length; index += 4) {
- if (data[index]! < 200 || data[index + 1]! < 200 || data[index + 2]! < 200) return true;
- }
- return false;
- }, [...frame.data]).catch(() => false);
- if (hasPaint) painted.add(page);
- await options?.onFrame?.(frame);
- },
- }));
- restores.push(() => spy.mockRestore());
- };
- /** Install the frame observer before a newly created page starts its recording. */
- const watchContext = (context: BrowserContext): void => {
- context.on('page', watchPage);
- for (const page of context.pages()) watchPage(page);
- };
- const settle = async (): Promise => {
- const page = surfaceOf(engine)!.page();
- // A static page may not repaint after navigation overtakes its first screencast frame.
- await page.screenshot();
- await expect.poll(() => painted.has(page), { timeout: 5_000, message: 'the recorder received a painted frame' }).toBe(true);
- };
try {
await withAttempt(engine, app, videoDir, 'v1', async () => {
await engine.session!.open!(`${app.url}/form`, operation('v1'));
const context = surfaceOf(engine)!.context();
- watchContext(context);
+ paint.watchContext(context);
const browser = context.browser()!;
const newContext = browser.newContext.bind(browser);
const contextSpy = vi.spyOn(browser, 'newContext').mockImplementation(async (options) => {
const next = await newContext(options);
- watchContext(next);
+ paint.watchContext(next);
return next;
});
restores.push(() => contextSpy.mockRestore());
@@ -997,19 +1013,19 @@ describe('web engine lifecycle', () => {
const textbox = observed.find((node) => node.role === 'textbox' && node.name === 'First');
await engine.perform!(textbox!.ref, { kind: 'fill', value: 'recorded', sensitive: false }, operation('v1'));
await engine.artifacts!.screenshot('shot', operation('v1'));
- await settle();
+ await paint.settle(surfaceOf(engine)!.page());
// A restart closes the page and opens another, blank until the harness
// reopens the app: the recording continues in a second segment.
await engine.session!.restart!(operation('v1'));
expect(surfaceOf(engine)!.page()).not.toBe(page);
expect(surfaceOf(engine)!.page().url()).toBe('about:blank');
await engine.session!.open!(`${app.url}/form`, operation('v1'));
- await settle();
+ await paint.settle(surfaceOf(engine)!.page());
// A state reset recreates the context; the recording continues in a third.
await engine.session!.reset!(operation('v1'));
expect(surfaceOf(engine)!.page().url()).toBe('about:blank');
await engine.session!.open!(`${app.url}/`, operation('v1'));
- await settle();
+ await paint.settle(surfaceOf(engine)!.page());
const segments = videoFiles(await engine.artifacts!.stopVideo!(operation('v1')));
// The restart and the state reset each closed a trace segment before the final archive.
expect(await engine.artifacts!.stopTrace!(operation('v1'))).toEqual([
@@ -1041,6 +1057,7 @@ describe('web engine lifecycle', () => {
});
} finally {
for (const restore of restores) restore();
+ paint.restore();
rmSync(videoDir, { recursive: true, force: true });
}
});
@@ -1048,18 +1065,20 @@ describe('web engine lifecycle', () => {
it('splits a running trace around a video started mid-attempt, so every recording fills the viewport', async () => {
const engine = web();
const videoDir = mkdtempSync(path.join(tmpdir(), 'e2e-video-'));
+ const paint = watchPaint();
try {
await withAttempt(engine, app, videoDir, 'v3', async () => {
// A host that records on demand: the trace already sized the screencast when the video starts.
await engine.artifacts!.startTrace!(operation('v3'));
await engine.session!.open!(`${app.url}/`, operation('v3'));
+ paint.watchContext(surfaceOf(engine)!.context());
const viewer = surfaceOf(engine)!.page();
const viewport = viewer.viewportSize()!;
const segments: VideoFile[] = [];
for (const url of [`${app.url}/form`, `${app.url}/`]) {
await engine.artifacts!.startVideo!(operation('v3'));
await engine.session!.open!(url, operation('v3'));
- await viewer.screenshot();
+ await paint.settle(viewer);
segments.push(...videoFiles(await engine.artifacts!.stopVideo!(operation('v3'))));
}
expect(segments.map((segment) => segment.path)).toEqual(['video/video.webm', 'video/video-part2.webm']);
@@ -1077,6 +1096,7 @@ describe('web engine lifecycle', () => {
]);
});
} finally {
+ paint.restore();
rmSync(videoDir, { recursive: true, force: true });
}
});
From 99959329cd00cc905f4fc2838d7ace3f698ad8bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oskar=20Kwa=C5=9Bniewski?=
Date: Mon, 5 Oct 2026 11:44:13 +0200
Subject: [PATCH 04/46] fix(run): charge stray rejections in in-process runs
like a worker process (#809)
* fix(run): catch stray rejections and exceptions in in-process runs
An in-process run (rawConfig, e2e explore) installed no unhandledRejection or
uncaughtException handling, so a test that left a rejection unhandled crashed
the host instead of failing. The in-process runner now routes both the way the
worker entry does, for as long as it lives.
* fix(run): end an in-process worker on a fatal stray as a process would
* fix(run): keep engine disposal errors from a crashed in-process worker
* docs(changeset): narrow the in-process fatal rejection case
---
.changeset/in-process-stray-rejections.md | 5 +
packages/e2e/src/run/in-process.ts | 59 ++++++-
.../integration/in-process-strays.test.ts | 166 ++++++++++++++++++
3 files changed, 221 insertions(+), 9 deletions(-)
create mode 100644 .changeset/in-process-stray-rejections.md
create mode 100644 packages/e2e/tests/integration/in-process-strays.test.ts
diff --git a/.changeset/in-process-stray-rejections.md b/.changeset/in-process-stray-rejections.md
new file mode 100644
index 000000000..db9aca57f
--- /dev/null
+++ b/.changeset/in-process-stray-rejections.md
@@ -0,0 +1,5 @@
+---
+"e2e": patch
+---
+
+A run that executes in the host process, such as `e2e explore`, no longer crashes that process when a test leaves a promise rejection or an exception uncaught. It handles them as a worker process does: a rejection fails the test running when it surfaces, or is recorded as `UNHANDLED_REJECTION` between tests. An uncaught exception, or a rejection that surfaces with no test running before any test has finished, is a run error that fails the test on its way with `WORKER_CRASH` and skips the rest of its file. The handlers are removed when the run ends.
diff --git a/packages/e2e/src/run/in-process.ts b/packages/e2e/src/run/in-process.ts
index bc06ae54a..00a9f52ed 100644
--- a/packages/e2e/src/run/in-process.ts
+++ b/packages/e2e/src/run/in-process.ts
@@ -9,6 +9,7 @@
import type { ModuleRegistration } from '../collect/registry.ts';
import type { Selection, TestTargetPair } from '../collect/select.ts';
import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts';
+import { isAbandonedRejection } from '../internal/abandoned.ts';
import { classifyError, ConfigurationError, serializeError } from '../internal/errors.ts';
import type { DebugTrace } from '../internal/debug.ts';
import type { InMemoryAttempts } from './execute.ts';
@@ -47,6 +48,20 @@ class InProcessRunner implements UnitRunner {
private readonly finish: () => void;
private exited = false;
private closing = false;
+ /**
+ * Set by a fatal error: the worker is dead to the scheduler from then on,
+ * as a crashed process would be, so nothing it reports about its unit
+ * while it winds down counts, and the unit's results are synthesized when
+ * it exits. Unlike a dead process it still disposes its engine, so the
+ * failures of that disposal (`shutdown-done`) are still reported.
+ */
+ private crashed = false;
+ /** Takes what this process failed to catch while the runner lives; see `catchStrays`. */
+ private readonly onRejection = (cause: unknown): void => {
+ if (isAbandonedRejection(cause)) return;
+ if (!this.worker.strayRejection(cause)) this.fail(cause);
+ };
+ private readonly onException = (cause: unknown): void => this.fail(cause);
constructor(
private readonly targetName: string,
@@ -62,20 +77,15 @@ class InProcessRunner implements UnitRunner {
this.worker = new TargetWorker(
{
emit: (message) => {
- if (!this.exited) this.events.onMessage(message);
- },
- fatal: (cause) => {
- if (this.exited) return;
- this.events.onMessage({
- type: 'fatal',
- error: serializeError(classifyError(cause)),
- });
- this.close();
+ if (this.exited || (this.crashed && message.type !== 'shutdown-done')) return;
+ this.events.onMessage(message);
},
+ fatal: (cause) => this.fail(cause),
finished: () => this.end('shut down'),
},
() => this.bootstrap(),
);
+ this.catchStrays();
this.worker.start();
}
@@ -109,6 +119,35 @@ class InProcessRunner implements UnitRunner {
this.close();
}
+ /**
+ * Catches what a test leaves uncaught the way a worker process does
+ * (`worker/entry.ts`): a rejection nobody handled is charged to the attempt
+ * in flight, or recorded against the last test that finished, and is
+ * otherwise fatal to this worker, as an uncaught exception is (see
+ * `fail`). The
+ * listeners live exactly as long as the runner, so a host process keeps its
+ * own handling before and after the run. The scheduler keeps at most one
+ * in-process runner alive at a time, so nothing is charged twice.
+ */
+ private catchStrays(): void {
+ process.on('unhandledRejection', this.onRejection);
+ process.on('uncaughtException', this.onException);
+ }
+
+ /**
+ * Reports an unrecoverable failure and ends the worker the way a process
+ * dies: its unit is interrupted and nothing more it reports is passed on,
+ * so on exit the scheduler fails the test in flight with `WORKER_CRASH`
+ * and skips the unit's others, as it does for a worker process.
+ */
+ private fail(cause: unknown): void {
+ if (this.exited || this.crashed) return;
+ this.events.onMessage({ type: 'fatal', error: serializeError(classifyError(cause)) });
+ this.crashed = true;
+ this.worker.handle({ type: 'interrupt' });
+ this.close();
+ }
+
/** Stops accepting work and asks the queue to wind down. */
private close(): void {
if (this.exited || this.closing) return;
@@ -119,6 +158,8 @@ class InProcessRunner implements UnitRunner {
private end(detail: string): void {
if (this.exited) return;
this.exited = true;
+ process.off('unhandledRejection', this.onRejection);
+ process.off('uncaughtException', this.onException);
this.events.onExit(detail);
this.finish();
}
diff --git a/packages/e2e/tests/integration/in-process-strays.test.ts b/packages/e2e/tests/integration/in-process-strays.test.ts
new file mode 100644
index 000000000..afc6237e8
--- /dev/null
+++ b/packages/e2e/tests/integration/in-process-strays.test.ts
@@ -0,0 +1,166 @@
+import { spawnSync } from 'node:child_process';
+import { afterEach, describe, expect, it } from 'vitest';
+import { createProject, type FixtureProject } from '../helpers/run-project.ts';
+
+const projects: FixtureProject[] = [];
+
+afterEach(() => {
+ for (const project of projects.splice(0)) project.cleanup();
+});
+
+/**
+ * A host process of its own: vitest's handlers would see the stray errors
+ * first, and the point is what an in-process run does to a host that has
+ * none. It runs the fixture through a `rawConfig` on a noop engine with
+ * `engineMembers`, prints what came back and which process listeners it left
+ * behind.
+ */
+const hostScript = (engineMembers: string) => `import { defineEngine } from 'e2e/engine';
+const { run } = await import(new URL('../../../dist/run/runner.js', import.meta.url).href);
+const listeners = () => [process.listenerCount('unhandledRejection'), process.listenerCount('uncaughtException')];
+const before = listeners();
+const engine = defineEngine({ name: 'noop', version: '1', spiVersion: 1, ${engineMembers} });
+const outcome = await run({
+ cwd: process.cwd(),
+ rawConfig: { targets: [{ name: 'headless', platform: 'test', engine }] },
+ env: { ...process.env, CI: '' },
+ quiet: true,
+});
+console.log(JSON.stringify({
+ results: Object.fromEntries(outcome.results.map((r) => [r.test.title, [r.status, r.attempts[0]?.error?.code ?? r.skip?.cause, r.attempts[0]?.error?.message]])),
+ exitCode: outcome.exitCode,
+ runErrors: outcome.report.run.errors.map((e) => [e.code, e.message]),
+ listeners: { before, after: listeners() },
+}));
+`;
+
+/** Runs `tests` in-process inside a fresh host process. */
+function runInHost(tests: Readonly>, engineMembers = '') {
+ const project = createProject({ ...tests, 'host.mjs': hostScript(engineMembers) });
+ projects.push(project);
+ const host = spawnSync(process.execPath, ['host.mjs'], {
+ cwd: project.dir,
+ env: { ...process.env, E2E_TELEMETRY_DISABLED: '1', NO_COLOR: '1' },
+ encoding: 'utf8',
+ timeout: 60_000,
+ });
+ expect(host.stderr).toBe('');
+ expect(host.status).toBe(0);
+ return JSON.parse(host.stdout.trim().split('\n').at(-1)!) as {
+ results: Record;
+ exitCode: number;
+ runErrors: [string, string][];
+ listeners: { before: number[]; after: number[] };
+ };
+}
+
+describe('an in-process run', () => {
+ it('fails the test that left a rejection unhandled, as a worker process does, and leaves the host its own handlers', () => {
+ const outcome = runInHost({
+ 'tests/stray.e2e.ts': `import { test } from 'e2e';
+
+test('leaves a rejection behind', async () => {
+ void Promise.reject(new Error('boom'));
+ await new Promise((resolve) => setTimeout(resolve, 100));
+});
+
+test('runs after the rejection', async () => {});
+`,
+ });
+ expect(outcome.results).toEqual({
+ 'leaves a rejection behind': ['failed', 'ERROR', 'boom'],
+ 'runs after the rejection': ['passed', null, null],
+ });
+ expect(outcome.runErrors).toEqual([]);
+ expect(outcome.listeners).toEqual({ before: [0, 0], after: [0, 0] });
+ });
+
+ it('records a rejection between tests against the last test that finished', () => {
+ const outcome = runInHost({
+ 'tests/a-leaks.e2e.ts': `import { test } from 'e2e';
+
+test('finishes before the leak', async () => {});
+
+test.afterAll(async () => {
+ void Promise.reject(new Error('late'));
+ await new Promise((resolve) => setTimeout(resolve, 100));
+});
+`,
+ 'tests/b-next.e2e.ts': `import { test } from 'e2e';
+
+test('runs afterwards', async () => {});
+`,
+ });
+ expect(outcome.results['finishes before the leak']?.[0]).toBe('passed');
+ expect(outcome.results['runs afterwards']?.[0]).toBe('passed');
+ expect(outcome.runErrors).toHaveLength(1);
+ expect(outcome.runErrors[0]?.[0]).toBe('UNHANDLED_REJECTION');
+ expect(outcome.runErrors[0]?.[1]).toContain('late');
+ });
+
+ it('ends the worker on an uncaught exception, as a crashed worker process, instead of crashing the host', () => {
+ const outcome = runInHost({
+ 'tests/throws.e2e.ts': `import { test } from 'e2e';
+
+test('throws off the stack', async () => {
+ setTimeout(() => {
+ throw new Error('boom-uncaught');
+ });
+ await new Promise((resolve) => setTimeout(resolve, 2_000));
+});
+
+test('never starts', async () => {});
+`,
+ });
+ expect(outcome.results).toEqual({
+ 'throws off the stack': ['failed', 'WORKER_CRASH', expect.any(String)],
+ 'never starts': ['skipped', 'infrastructure-unavailable', null],
+ });
+ expect(outcome.runErrors).toEqual([['ERROR', 'boom-uncaught']]);
+ expect(outcome.exitCode).toBe(3);
+ expect(outcome.listeners).toEqual({ before: [0, 0], after: [0, 0] });
+ });
+
+ it('still reports what disposing the engine failed with after a fatal', () => {
+ const outcome = runInHost(
+ {
+ 'tests/throws.e2e.ts': `import { test } from 'e2e';
+
+test('throws off the stack', async () => {
+ setTimeout(() => {
+ throw new Error('boom-uncaught');
+ });
+ await new Promise((resolve) => setTimeout(resolve, 2_000));
+});
+`,
+ },
+ `dispose: () => { throw new Error('dispose-failed'); },`,
+ );
+ expect(outcome.runErrors).toEqual([
+ ['ERROR', 'boom-uncaught'],
+ ['ENGINE_FAILURE', expect.stringContaining('dispose-failed')],
+ ]);
+ });
+
+ it('ends the worker on a rejection before any test has finished in it', () => {
+ const outcome = runInHost({
+ 'tests/early.e2e.ts': `import { test } from 'e2e';
+
+test.beforeAll(async () => {
+ void Promise.reject(new Error('early'));
+ await new Promise((resolve) => setTimeout(resolve, 500));
+});
+
+test('is on its way when it surfaces', async () => {});
+
+test('follows in the same file', async () => {});
+`,
+ });
+ expect(outcome.results).toEqual({
+ 'is on its way when it surfaces': ['failed', 'WORKER_CRASH', expect.any(String)],
+ 'follows in the same file': ['skipped', 'infrastructure-unavailable', null],
+ });
+ expect(outcome.runErrors).toEqual([['ERROR', 'early']]);
+ expect(outcome.exitCode).toBe(3);
+ });
+});
From 52228f4dc572c82bf4089c9b5504063c59f369fb Mon Sep 17 00:00:00 2001
From: Szymon Rybczak
Date: Mon, 5 Oct 2026 12:29:49 +0200
Subject: [PATCH 05/46] docs(readme): GitHub Trending repository of the day
badge (#852)
---
README.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/README.md b/README.md
index 93a3778ff..59df5057f 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,10 @@
+
+
+
+
# e2e
[e2e](https://tester.army/e2e?utm_source=e2e&utm_medium=github&utm_campaign=readme_intro) is an end-to-end testing framework for web and mobile apps. Describe a goal in natural language and an agent drives the app to reach it. Check the result with locators and assertions in the same test.
From 2e26ec65510356508f680d70e85624d86a8c4c25 Mon Sep 17 00:00:00 2001
From: ammann <161159040+agammann@users.noreply.github.com>
Date: Mon, 5 Oct 2026 03:57:09 -0700
Subject: [PATCH 06/46] fix(cli): forward flags in npm exec hints (#847)
---
.changeset/npm-exec-cli-hint.md | 5 +++++
docs/reference/cli.mdx | 2 ++
packages/e2e/src/internal/package-manager.ts | 2 +-
packages/e2e/tests/unit/cli-init.test.ts | 6 +++---
packages/e2e/tests/unit/cli.test.ts | 6 +++---
packages/e2e/tests/unit/package-manager.test.ts | 2 +-
skills/e2e/references/running.md | 1 +
7 files changed, 16 insertions(+), 8 deletions(-)
create mode 100644 .changeset/npm-exec-cli-hint.md
diff --git a/.changeset/npm-exec-cli-hint.md b/.changeset/npm-exec-cli-hint.md
new file mode 100644
index 000000000..a98ba22e1
--- /dev/null
+++ b/.changeset/npm-exec-cli-hint.md
@@ -0,0 +1,5 @@
+---
+"e2e": patch
+---
+
+Include npm's argument separator in CLI hints so flags such as `--headed` reach the installed `e2e` binary.
diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx
index 59f08775d..04a7eb04d 100644
--- a/docs/reference/cli.mdx
+++ b/docs/reference/cli.mdx
@@ -346,6 +346,8 @@ A positional starting with `-` that names no file is a usage error, exit 2.
This is what `pnpm test:e2e -- --headed` produces: the `--` reaches e2e and
`--headed` would otherwise be taken for a file. The message names the direct
form, `pnpm exec e2e run --headed`.
+For npm, the direct form is `npm exec -- e2e run --headed`; npm needs the
+separator before the binary to pass its flags through.
When nothing is left to run, `NO_TESTS` names the cause: no file matched the
globs (with look-alike files such as `tests/login.test.ts`), a positional
diff --git a/packages/e2e/src/internal/package-manager.ts b/packages/e2e/src/internal/package-manager.ts
index 3e44e3a59..de1797e46 100644
--- a/packages/e2e/src/internal/package-manager.ts
+++ b/packages/e2e/src/internal/package-manager.ts
@@ -54,7 +54,7 @@ export function runScriptCommand(manager: PackageManager, name: string): string
export function execCommand(manager: PackageManager, command: string): string {
switch (manager) {
case 'npm':
- return `npm exec ${command}`;
+ return `npm exec -- ${command}`;
case 'pnpm':
return `pnpm exec ${command}`;
case 'yarn':
diff --git a/packages/e2e/tests/unit/cli-init.test.ts b/packages/e2e/tests/unit/cli-init.test.ts
index 95a4db774..5830149b6 100644
--- a/packages/e2e/tests/unit/cli-init.test.ts
+++ b/packages/e2e/tests/unit/cli-init.test.ts
@@ -379,8 +379,8 @@ describe('e2e init', () => {
it.each([
['e2e run --workers 1', 'npm run test:e2e'],
- ['e2e runner --ci', 'npm exec e2e run'],
- ['vitest', 'npm exec e2e run'],
+ ['e2e runner --ci', 'npm exec -- e2e run'],
+ ['vitest', 'npm exec -- e2e run'],
])('keeps an existing test:e2e script (%s) and points the run step at %s', async (script, step) => {
writeFileSync(path.join(dir, 'package.json'), `${JSON.stringify({ name: 'existing-app', scripts: { 'test:e2e': script } })}\n`);
vi.mocked(clack.select).mockResolvedValueOnce('web').mockResolvedValueOnce('none');
@@ -667,7 +667,7 @@ describe('e2e init', () => {
expect(existsSync(path.join(dir, '.agents'))).toBe(false);
expect(existsSync(path.join(dir, '.claude'))).toBe(false);
expect(existsSync(path.join(dir, '.mcp.json'))).toBe(false);
- expect(output()).toContain('npm exec e2e guide');
+ expect(output()).toContain('npm exec -- e2e guide');
expect(output()).toContain('claude mcp add e2e -- npx e2e mcp');
});
diff --git a/packages/e2e/tests/unit/cli.test.ts b/packages/e2e/tests/unit/cli.test.ts
index 512f8c5fb..ac6c5b518 100644
--- a/packages/e2e/tests/unit/cli.test.ts
+++ b/packages/e2e/tests/unit/cli.test.ts
@@ -99,7 +99,7 @@ describe('e2e run argument parsing', () => {
expect(runMock).not.toHaveBeenCalled();
expect(process.exitCode).toBe(2);
expect(written(stderrSpy)).toMatch(
- /^error: "--headed" is a flag, not a test file\. It arrived as a file because a "--" came before it; "(?:pnpm|npm run|yarn|bun run) test:e2e -- --headed" reaches e2e as "run -- --headed" when the package manager forwards the separator\. Run the CLI directly instead: (?:pnpm exec|npm exec|yarn|bun run) e2e run --headed\n\(add --help for usage\)\n$/u,
+ /^error: "--headed" is a flag, not a test file\. It arrived as a file because a "--" came before it; "(?:pnpm|npm run|yarn|bun run) test:e2e -- --headed" reaches e2e as "run -- --headed" when the package manager forwards the separator\. Run the CLI directly instead: (?:pnpm exec|npm exec --|yarn|bun run) e2e run --headed\n\(add --help for usage\)\n$/u,
);
});
@@ -108,7 +108,7 @@ describe('e2e run argument parsing', () => {
expect(runMock).not.toHaveBeenCalled();
expect(process.exitCode).toBe(2);
expect(written(stderrSpy)).toMatch(
- /Run the CLI directly instead: (?:pnpm exec|npm exec|yarn|bun run) e2e run --tag smoke tests\/b\.e2e\.ts\n/u,
+ /Run the CLI directly instead: (?:pnpm exec|npm exec --|yarn|bun run) e2e run --tag smoke tests\/b\.e2e\.ts\n/u,
);
});
@@ -453,7 +453,7 @@ describe('e2e list', () => {
expect(process.exitCode).toBe(2);
const stderr = written(stderrSpy);
expect(stderr.startsWith('error: "--tag" is a flag, not a test file.')).toBe(true);
- expect(stderr).toMatch(/Run the CLI directly instead: (?:pnpm exec|npm exec|yarn|bun run) e2e list --tag smoke\n/u);
+ expect(stderr).toMatch(/Run the CLI directly instead: (?:pnpm exec|npm exec --|yarn|bun run) e2e list --tag smoke\n/u);
expect(stderr.endsWith('(add --help for usage)\n')).toBe(true);
});
diff --git a/packages/e2e/tests/unit/package-manager.test.ts b/packages/e2e/tests/unit/package-manager.test.ts
index 4f847f413..c4dd44f73 100644
--- a/packages/e2e/tests/unit/package-manager.test.ts
+++ b/packages/e2e/tests/unit/package-manager.test.ts
@@ -38,7 +38,7 @@ describe('runScriptCommand', () => {
describe('execCommand', () => {
it('runs the installed binary with each manager', () => {
- expect(execCommand('npm', 'e2e guide')).toBe('npm exec e2e guide');
+ expect(execCommand('npm', 'e2e guide')).toBe('npm exec -- e2e guide');
expect(execCommand('pnpm', 'e2e guide')).toBe('pnpm exec e2e guide');
expect(execCommand('yarn', 'e2e guide')).toBe('yarn e2e guide');
expect(execCommand('bun', 'e2e guide')).toBe('bun run e2e guide');
diff --git a/skills/e2e/references/running.md b/skills/e2e/references/running.md
index e3b24732f..39b6674df 100644
--- a/skills/e2e/references/running.md
+++ b/skills/e2e/references/running.md
@@ -67,6 +67,7 @@ npx e2e list tests/signup.e2e.ts --tag smoke --reporter json
With a `package.json` script `"test:e2e": "e2e run"`, pnpm forwards `--`
literally: `pnpm test:e2e -- --headed` reaches e2e as `run -- --headed` and
exits 2. Write `pnpm test:e2e --headed` or `pnpm exec e2e run --headed`.
+With npm, use `npm exec -- e2e run --headed` to pass flags to e2e.
## The replay cache
From dc2afca0c02a882e2631a18f9d49768c4e4d84f2 Mon Sep 17 00:00:00 2001
From: Krzysztof Moch
Date: Mon, 5 Oct 2026 12:58:11 +0200
Subject: [PATCH 07/46] feat(mcp)!: e2e mcp is headless by default, and
open_session takes headed (#851)
* feat(mcp)!: e2e mcp runs headless by default and takes --headed, like run
Sessions were headed outside CI and took --headless, the opposite of
e2e run and e2e explore. --headless stays accepted and hidden: it now
asks for the default, and a refused flag would reach an MCP host as
nothing but a failed connection.
* docs(skill): --headed shows the UI when the engine supports it
* feat(mcp): open_session takes headed, so the agent picks per session
The server's --headed sets the default for sessions that do not say,
and headed: true or false overrides it, so a user who asks to watch
needs no change to the MCP client config.
---
.changeset/mcp-headless-default.md | 5 ++++
docs/reference/cli.mdx | 6 ++---
docs/reference/mcp.mdx | 10 +++----
packages/e2e/src/cli/index.ts | 9 ++++---
packages/e2e/src/cli/mcp.ts | 5 ++--
packages/e2e/src/mcp/server.ts | 3 ++-
packages/e2e/src/mcp/session.ts | 26 +++++++++++++++----
.../tests/integration/mcp-disconnect.test.ts | 2 +-
.../e2e/tests/integration/mcp-server.test.ts | 25 ++++++++++++++++--
.../integration/mcp-session-host.test.ts | 14 ++++++++++
.../mcp-stdout-backpressure.test.ts | 2 +-
skills/e2e/references/mcp.md | 7 ++---
12 files changed, 87 insertions(+), 27 deletions(-)
create mode 100644 .changeset/mcp-headless-default.md
diff --git a/.changeset/mcp-headless-default.md b/.changeset/mcp-headless-default.md
new file mode 100644
index 000000000..fcb8d361e
--- /dev/null
+++ b/.changeset/mcp-headless-default.md
@@ -0,0 +1,5 @@
+---
+"e2e": minor
+---
+
+`e2e mcp` sessions are now headless by default, the same as `e2e run` and `e2e explore`, and the agent decides per session: `open_session {headed: true}` shows the browser or simulator, `headed: false` hides it. `--headed` makes headed the default for sessions that do not say. Before, sessions were headed outside CI and only the server's `--headless` could hide them; `--headless` is still accepted, since it now asks for the default, but it is no longer documented.
diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx
index 04a7eb04d..86723faa9 100644
--- a/docs/reference/cli.mdx
+++ b/docs/reference/cli.mdx
@@ -281,15 +281,15 @@ See [Coding agents](/coding-agents).
```bash npm
-npx e2e mcp [--config ] [--target ] [--headless] [--max-sessions ]
+npx e2e mcp [--config ] [--target ] [--headed] [--max-sessions ]
```
```bash pnpm
-pnpm exec e2e mcp [--config ] [--target ] [--headless] [--max-sessions ]
+pnpm exec e2e mcp [--config ] [--target ] [--headed] [--max-sessions ]
```
```bash bun
-bunx e2e mcp [--config ] [--target ] [--headless] [--max-sessions ]
+bunx e2e mcp [--config ] [--target ] [--headed] [--max-sessions ]
```
diff --git a/docs/reference/mcp.mdx b/docs/reference/mcp.mdx
index b0334209b..54121ed5d 100644
--- a/docs/reference/mcp.mdx
+++ b/docs/reference/mcp.mdx
@@ -5,15 +5,15 @@ description: Let a coding agent inspect and interact with your app over MCP.
```bash npm
-npx e2e mcp [--config ] [--target ] [--headless] [--max-sessions ]
+npx e2e mcp [--config ] [--target ] [--headed] [--max-sessions ]
```
```bash pnpm
-pnpm exec e2e mcp [--config ] [--target ] [--headless] [--max-sessions ]
+pnpm exec e2e mcp [--config ] [--target ] [--headed] [--max-sessions ]
```
```bash bun
-bunx e2e mcp [--config ] [--target ] [--headless] [--max-sessions ]
+bunx e2e mcp [--config ] [--target ] [--headed] [--max-sessions ]
```
@@ -64,7 +64,7 @@ Start the server in the project directory. Config discovery works like
| --- | --- |
| `--config ` | The config every session loads unless `open_session` names one. Configs are reloaded on every `open_session`, with every file they import by path. |
| `--target ` | The target every session opens on. Otherwise `open_session` names one, or the only target is used; several targets and no name is `TARGET_REQUIRED`. |
-| `--headless` | Hide the browser or simulator. Sessions are headed by default outside CI. |
+| `--headed` | Show the browser or simulator, when the engine supports it, in every session whose `open_session` does not set `headed`. Sessions are headless by default, as in `e2e run`. |
| `--max-sessions ` | How many sessions may be open at once, 1 through 16; default 4. Each session is its own browser or device. |
Operator output goes to stderr, prefixed `e2e mcp:`. Stdout carries only the
@@ -74,7 +74,7 @@ protocol; anything a config module writes to stdout is redirected.
| Tool | Arguments | Result |
| --- | --- | --- |
-| `open_session` | `target` (optional), `config` (optional path) | Loads the config, starts the target's app command, boots the engine, starts one attempt, opens the app URL when the engine can navigate. Returns the session id, the catalog, and the first observation. |
+| `open_session` | `target` (optional), `config` (optional path), `headed` (optional boolean) | Loads the config, starts the target's app command, boots the engine, starts one attempt, opens the app URL when the engine can navigate. Returns the session id, the catalog, and the first observation. `headed: true` shows this session's browser or simulator, `false` hides it; without it the session follows `--headed`. |
| `tools` | `tool` (optional), `session` (required while several are open) | The catalog, one line per tool. With `tool`, its full description and argument schema. |
| `call` | `tool`, `args` (optional), `session` (required while several are open) | Runs one catalog tool. Arguments are validated against the tool's closed schema first: a wrong type or an argument the tool does not declare is `INVALID_ARGUMENT`, and nothing runs. An action that fails is an error result that still carries the screen it re-observed, and leads with the tool, its target, and the code: `navigate file:///etc/passwd failed: POLICY_DENIED: forbidden URL scheme: file:`. |
| `close_session` | `session` (required while several are open) | Saves a recording still running, ends the attempt, disposes the engine, stops the app processes the session started once no other session uses them. |
diff --git a/packages/e2e/src/cli/index.ts b/packages/e2e/src/cli/index.ts
index 36aa1590a..1a9d8f3a1 100644
--- a/packages/e2e/src/cli/index.ts
+++ b/packages/e2e/src/cli/index.ts
@@ -496,7 +496,10 @@ function createProgram(version: string, telemetry: Telemetry): Command {
)
.option('--config ', 'config file (default: the nearest e2e.config.ts)')
.option('--target ', 'target every session opens on (default: the only target, or the one open_session names)')
- .option('--headless', 'hide the UI during live sessions (default: headed outside CI)')
+ .option('--headed', 'show the UI in sessions whose open_session does not set headed, when the engine supports it')
+ // Sessions were headed by default before 0.18 and took --headless; a client config that still passes it
+ // asks for the default now, and a refused flag would surface as nothing but a failed connection.
+ .addOption(new Option('--headless').hideHelp())
.option(
'--max-sessions ',
`sessions open at once, each with its own browser or device, ${SESSION_BOUNDS.min} through ${SESSION_BOUNDS.max} (default: ${SESSION_BOUNDS.default})`,
@@ -506,12 +509,12 @@ function createProgram(version: string, telemetry: Telemetry): Command {
'after',
[
'',
- examples(['e2e mcp', 'e2e mcp --target web --headless', 'e2e mcp --max-sessions 8', 'claude mcp add e2e -- npx e2e mcp']),
+ examples(['e2e mcp', 'e2e mcp --target web --headed', 'e2e mcp --max-sessions 8', 'claude mcp add e2e -- npx e2e mcp']),
'',
docsLine('/reference/mcp'),
].join('\n'),
)
- .action(async (options: { config?: string; target?: string; headless?: boolean; maxSessions?: number }) => {
+ .action(async (options: { config?: string; target?: string; headed?: boolean; maxSessions?: number }) => {
process.exitCode = await mcp(version, options, telemetry);
});
diff --git a/packages/e2e/src/cli/mcp.ts b/packages/e2e/src/cli/mcp.ts
index dc140086f..7b810e430 100644
--- a/packages/e2e/src/cli/mcp.ts
+++ b/packages/e2e/src/cli/mcp.ts
@@ -7,7 +7,6 @@
*/
import { Writable } from 'node:stream';
-import { isCiMode } from '../config/resolve.ts';
import { errorMessage, exitCodeForCategory, classifyError } from '../internal/errors.ts';
import { serveMcp, type ServeOptions } from '../mcp/server.ts';
import { mcpSessionEvent } from '../telemetry/events.ts';
@@ -16,7 +15,7 @@ import type { Telemetry } from '../telemetry/telemetry.ts';
export interface McpCommandOptions {
config?: string | undefined;
target?: string | undefined;
- headless?: boolean | undefined;
+ headed?: boolean | undefined;
maxSessions?: number | undefined;
}
@@ -74,7 +73,7 @@ export async function mcp(version: string, options: McpCommandOptions, telemetry
cwd: process.cwd(),
configPath: options.config,
target: options.target,
- headed: options.headless !== true && !isCiMode(process.env),
+ headed: options.headed === true,
maxSessions: options.maxSessions,
env: process.env,
version,
diff --git a/packages/e2e/src/mcp/server.ts b/packages/e2e/src/mcp/server.ts
index e1c46b790..94fe180c5 100644
--- a/packages/e2e/src/mcp/server.ts
+++ b/packages/e2e/src/mcp/server.ts
@@ -29,6 +29,7 @@ export interface ServeOptions {
readonly configPath?: string | undefined;
/** The target every session opens on; otherwise a call names one, or the only one is used. */
readonly target?: string | undefined;
+ /** Whether a session shows its UI when `open_session` does not set `headed`. */
readonly headed: boolean;
/** How many sessions may be open at once, from `--max-sessions`. */
readonly maxSessions?: number | undefined;
@@ -46,7 +47,7 @@ export interface ServeOptions {
}
const INSTRUCTIONS = `e2e is a local-first end-to-end test runner; this server drives an e2e project's app (its e2e.config.ts) live.
-Call open_session (optionally with a target and a config path) to get a session, its tool catalog, and the first observation. Then call {tool, args} runs any catalog tool: observe, the grammar its engine honors, type_secret, locate, screenshot, start_recording and stop_recording when the engine records video (a video of the app for a pull request), and the project's own tools; tools lists them, tools {tool} shows one tool's arguments. close_session when done.
+Call open_session (optionally with a target, a config path, and headed: true when the user wants to watch) to get a session, its tool catalog, and the first observation. Then call {tool, args} runs any catalog tool: observe, the grammar its engine honors, type_secret, locate, screenshot, start_recording and stop_recording when the engine records video (a video of the app for a pull request), and the project's own tools; tools lists them, tools {tool} shows one tool's arguments. close_session when done.
Several sessions can be open at once, each with its own browser or device, so parallel agents (subagents) each open their own: pass the session id from open_session to every tools, call, and close_session.
Write deterministic tests (tests/*.e2e.ts) from what you saw and run them with the CLI: npx e2e run . Resources e2e://guide and e2e://guide/{topic} hold the writing guide.`;
diff --git a/packages/e2e/src/mcp/session.ts b/packages/e2e/src/mcp/session.ts
index 8f90a66e2..744e4fe8f 100644
--- a/packages/e2e/src/mcp/session.ts
+++ b/packages/e2e/src/mcp/session.ts
@@ -47,6 +47,7 @@ interface LiveSession {
readonly id: string;
readonly target: ResolvedTarget;
readonly configPath: string;
+ readonly headed: boolean;
readonly attempt: StandaloneAttempt;
readonly step: InteractiveStep;
readonly screen: ScreenPresenter;
@@ -65,6 +66,8 @@ export interface OpenSessionOptions {
readonly target?: string | undefined;
/** A config file to load instead of the server's default, relative to the server's directory. */
readonly config?: string | undefined;
+ /** Whether this session shows its UI; otherwise the server's `headed`. */
+ readonly headed?: boolean | undefined;
}
export interface SessionHostOptions {
@@ -73,6 +76,7 @@ export interface SessionHostOptions {
/** Loads the config at an absolute path fresh for each session, so an edited config applies without a restart. */
readonly loadConfig: (configPath: string) => Promise;
readonly env: NodeJS.ProcessEnv;
+ /** Whether a session shows its UI when `open_session` does not say, from `--headed`. */
readonly headed: boolean;
/** The target every session opens on, from `--target`; a call may still name one. */
readonly defaultTarget?: string | undefined;
@@ -116,8 +120,9 @@ export class SessionHost {
* it, or disconnects, aborts the open and what it started.
*/
open(options: OpenSessionOptions, signal?: AbortSignal): Promise {
- const usage = new SessionUsage(this.sessions.liveCount, this.options.headed);
- return this.sessions.admit((id) => this.openSession(id, options, usage, signal)).catch((cause: unknown) => {
+ const headed = options.headed ?? this.options.headed;
+ const usage = new SessionUsage(this.sessions.liveCount, headed);
+ return this.sessions.admit((id) => this.openSession(id, options, headed, usage, signal)).catch((cause: unknown) => {
this.options.onSessionEnd?.(usage.openFailed(classifyError(cause).code));
throw cause;
});
@@ -209,7 +214,13 @@ export class SessionHost {
);
}
- private async openSession(id: string, options: OpenSessionOptions, usage: SessionUsage, request: AbortSignal | undefined): Promise {
+ private async openSession(
+ id: string,
+ options: OpenSessionOptions,
+ headed: boolean,
+ usage: SessionUsage,
+ request: AbortSignal | undefined,
+ ): Promise {
// The catalog renders synchronously, so the optional SDK is loaded once
// here when installed. Without it the catalog reads the tools' Standard
// Schemas, and only a model-backed call needs the package.
@@ -242,7 +253,7 @@ export class SessionHost {
// one attempt that closes as passed, so it traces under `on` only.
config,
target: { ...target, video: { mode: 'off', source: 'default' } },
- headed: this.options.headed,
+ headed,
env: this.options.env,
signal: abort.signal,
timeoutMs: ttlMs + CLOSE_GRACE_MS,
@@ -282,6 +293,7 @@ export class SessionHost {
id,
target,
configPath: loaded.configPath,
+ headed,
attempt,
step,
screen,
@@ -371,7 +383,7 @@ export class SessionHost {
private openingText(live: LiveSession, config: ResolvedConfig, screen: string): string {
const engine = live.target.engine;
const lines = [
- `Session ${live.id} open on target "${live.target.name}" (platform ${live.target.platform}, engine ${engine === undefined ? 'none' : `${engine.name} ${engine.version}`}), ${this.options.headed ? 'headed' : 'headless'}; config ${live.configPath}.`,
+ `Session ${live.id} open on target "${live.target.name}" (platform ${live.target.platform}, engine ${engine === undefined ? 'none' : `${engine.name} ${engine.version}`}), ${live.headed ? 'headed' : 'headless'}; config ${live.configPath}.`,
];
if (live.target.app.base !== undefined) {
lines.push(`App: ${live.target.app.base.href}.`);
@@ -458,6 +470,10 @@ export class SessionHost {
inputSchema: z.object({
target: z.string().min(1).optional().describe('Target name from the config; required when the config declares several'),
config: z.string().min(1).optional().describe("Path to an e2e config file, relative to the server's directory; default: the nearest e2e.config.ts"),
+ headed: z
+ .boolean()
+ .optional()
+ .describe('Show the browser or simulator, when the engine supports it; pass true when the user wants to watch. Default: headless, unless the server was started with --headed'),
}).strict(),
readOnly: false,
call: async (args, extra) => textResult(await this.open(args, extra.signal)),
diff --git a/packages/e2e/tests/integration/mcp-disconnect.test.ts b/packages/e2e/tests/integration/mcp-disconnect.test.ts
index bb4918b52..a18bab05f 100644
--- a/packages/e2e/tests/integration/mcp-disconnect.test.ts
+++ b/packages/e2e/tests/integration/mcp-disconnect.test.ts
@@ -99,7 +99,7 @@ describe('e2e mcp when the client goes away', { timeout: 90_000 }, () => {
project = createProject({ 'e2e.config.ts': config(await freePort()), 'app.cjs': APP });
stderr = '';
responses = new Map();
- server = spawn(process.execPath, [CLI, 'mcp', '--headless'], {
+ server = spawn(process.execPath, [CLI, 'mcp'], {
cwd: project.dir,
env: { ...process.env, CI: '' },
stdio: ['pipe', 'pipe', 'pipe'],
diff --git a/packages/e2e/tests/integration/mcp-server.test.ts b/packages/e2e/tests/integration/mcp-server.test.ts
index c1d09f291..a47becb24 100644
--- a/packages/e2e/tests/integration/mcp-server.test.ts
+++ b/packages/e2e/tests/integration/mcp-server.test.ts
@@ -148,7 +148,7 @@ describe('e2e mcp', { timeout: 120_000 }, () => {
});
transport = new StdioClientTransport({
command: process.execPath,
- args: [CLI, 'mcp', '--headless'],
+ args: [CLI, 'mcp'],
cwd: project.dir,
env: { ...(process.env as Record), APP_URL: app.url, CI: '' },
stderr: 'pipe',
@@ -173,7 +173,7 @@ describe('e2e mcp', { timeout: 120_000 }, () => {
expect(callTool.description).toContain('call {tool: "tap", args: {target: "n42"}, session: ""}');
expect(callTool.inputSchema).toMatchObject({ type: 'object', required: ['tool'] });
expect(tools.find((tool) => tool.name === 'tools')?.annotations).toMatchObject({ readOnlyHint: true });
- expect(tools.find((tool) => tool.name === 'open_session')?.inputSchema).toMatchObject({ properties: { target: {}, config: {} } });
+ expect(tools.find((tool) => tool.name === 'open_session')?.inputSchema).toMatchObject({ properties: { target: {}, config: {}, headed: { type: 'boolean' } } });
expect(client.getInstructions()).toContain('call {tool, args} runs any catalog tool');
expect(client.getInstructions()).toContain('Several sessions can be open at once');
@@ -481,6 +481,27 @@ describe('e2e mcp', { timeout: 120_000 }, () => {
expect(closed.text).not.toContain('Cleanup:');
});
+ it('opens sessions headed with --headed, and headless with the retired --headless', async () => {
+ for (const [flag, mode] of [['--headed', 'headed'], ['--headless', 'headless']] as const) {
+ const other = new Client({ name: 'e2e-mcp-flag-test', version: '0.0.0' });
+ await other.connect(
+ new StdioClientTransport({
+ command: process.execPath,
+ args: [CLI, 'mcp', flag, '--config', 'custom.config.ts', '--target', 'kiosk'],
+ cwd: project.dir,
+ env: { ...(process.env as Record), APP_URL: app.url, CI: '' },
+ stderr: 'ignore',
+ }),
+ );
+ try {
+ const opened = (await other.callTool({ name: 'open_session', arguments: {} })) as { content: { text?: string }[] };
+ expect(opened.content[0]?.text, flag).toContain(`), ${mode};`);
+ } finally {
+ await other.close();
+ }
+ }
+ });
+
it('kept stdout for the protocol: the config\'s console.log landed on stderr', () => {
expect(stderr).toContain('config loaded');
expect(stderr).toContain('e2e mcp: [info] e2e mcp');
diff --git a/packages/e2e/tests/integration/mcp-session-host.test.ts b/packages/e2e/tests/integration/mcp-session-host.test.ts
index 3b721c7cb..362d34503 100644
--- a/packages/e2e/tests/integration/mcp-session-host.test.ts
+++ b/packages/e2e/tests/integration/mcp-session-host.test.ts
@@ -126,6 +126,20 @@ describe('SessionHost', { timeout: 60_000 }, () => {
expect(fake.stats()).toMatchObject({ attemptsStarted: 2, attemptsEnded: 2, disposes: 2 });
});
+ it("lets open_session's headed override the server's default either way", async () => {
+ const fake = createFakeEngine();
+ const summaries: McpSessionSummary[] = [];
+ const onSessionEnd = (summary: McpSessionSummary): void => void summaries.push(summary);
+ const headless = host(() => fake, { headed: false, onSessionEnd });
+ expect(await headless.open({ headed: true })).toContain('), headed;');
+ await headless.close('done');
+ const headed = host(() => fake, { headed: true, onSessionEnd });
+ expect(await headed.open({ headed: false })).toContain('), headless;');
+ await headed.close('done');
+ expect(fake.inits.map((init) => init.headed)).toEqual([true, false]);
+ expect(summaries.map((summary) => summary.headed)).toEqual([true, false]);
+ });
+
it('closes an idle session and disposes the engine', async () => {
const fakes = engines();
const idle = host(fakes.next, { idleMs: 100 });
diff --git a/packages/e2e/tests/integration/mcp-stdout-backpressure.test.ts b/packages/e2e/tests/integration/mcp-stdout-backpressure.test.ts
index a05e70a82..0e44d5586 100644
--- a/packages/e2e/tests/integration/mcp-stdout-backpressure.test.ts
+++ b/packages/e2e/tests/integration/mcp-stdout-backpressure.test.ts
@@ -33,7 +33,7 @@ describe('e2e mcp stdout backpressure', { timeout: 60_000 }, () => {
beforeAll(() => {
project = createProject({ 'e2e.config.ts': CONFIG });
- server = spawn(process.execPath, [CLI, 'mcp', '--headless'], {
+ server = spawn(process.execPath, [CLI, 'mcp'], {
cwd: project.dir,
env: { ...process.env, CI: '' },
stdio: ['pipe', 'pipe', 'pipe'],
diff --git a/skills/e2e/references/mcp.md b/skills/e2e/references/mcp.md
index 9875d27d6..84a9c9bd3 100644
--- a/skills/e2e/references/mcp.md
+++ b/skills/e2e/references/mcp.md
@@ -24,8 +24,9 @@ No model is needed: without the optional `ai` package, sessions open and the
deterministic tools work.
Flags: `--config ` names the default config file, `--target `
-fixes the target every session opens on, `--headless` hides the browser or
-simulator (sessions are headed by default outside CI), `--max-sessions `
+fixes the target every session opens on, `--headed` shows the browser or
+simulator when the engine supports it in sessions that do not set `headed`
+(sessions are headless by default, as in `e2e run`), `--max-sessions `
sets how many sessions may be open at once (default 4, 1 through 16).
## Tools
@@ -34,7 +35,7 @@ Four tools; everything a session can do is a catalog behind `call`.
| Tool | Does |
| --- | --- |
-| `open_session` | Loads the config (`config` names another file; default the nearest `e2e.config.ts`), starts the declared app command if any, boots the engine, opens the app URL, and returns the session id, the catalog, and the first observation. `target` is required when the config declares several. Each session has its own browser or device. |
+| `open_session` | Loads the config (`config` names another file; default the nearest `e2e.config.ts`), starts the declared app command if any, boots the engine, opens the app URL, and returns the session id, the catalog, and the first observation. `target` is required when the config declares several. `headed: true` shows the browser or simulator when the user wants to watch; without it the session follows the server's `--headed`, headless by default. Each session has its own browser or device. |
| `tools` | The catalog: one line per tool with its argument names (`?` marks optional), the first sentence of its description, and `[read-only]` where it changes nothing. `tools {tool}` shows the full description and the JSON Schema of its arguments. |
| `call` | Runs one catalog tool: `call {tool: "tap", args: {target: "n42"}}`. Arguments are checked against the tool's schema first; a wrong one fails with `INVALID_ARGUMENT` naming the field. |
| `close_session` | Saves a recording still running, ends the attempt, disposes the engine, stops the app processes the session started once no other session uses them. |
From 8bc7d850cb0082122100a20bb710f7f0c9691f25 Mon Sep 17 00:00:00 2001
From: Ray Tien
Date: Mon, 5 Oct 2026 19:06:08 +0800
Subject: [PATCH 08/46] fix(cache): replay a recorded step that types an empty
value (#789) (#834)
The recorder writes a `type` or `typeText` action with value "" (clearing a
field), but the reader rejected it as `invalid-entry`, so the step re-ran the
model on every run and failed with REPLAY_STALE under --strict-cache.
Co-authored-by: Claude Sonnet 5.5
---
.changeset/cache-empty-typed-value.md | 5 +++++
packages/e2e/src/cache/trace.ts | 9 +++++++--
.../e2e/tests/helpers/fixture-pages/forms.ts | 11 +++++++++++
packages/e2e/tests/helpers/trace-cache.ts | 16 ++++++++++++++++
packages/e2e/tests/unit/trace-cache.test.ts | 10 ++++++++++
5 files changed, 49 insertions(+), 2 deletions(-)
create mode 100644 .changeset/cache-empty-typed-value.md
diff --git a/.changeset/cache-empty-typed-value.md b/.changeset/cache-empty-typed-value.md
new file mode 100644
index 000000000..a71198858
--- /dev/null
+++ b/.changeset/cache-empty-typed-value.md
@@ -0,0 +1,5 @@
+---
+"e2e": patch
+---
+
+A recorded step that types an empty value, which clears a field, replays instead of failing with `REPLAY_STALE` (`invalid-entry`). The cache wrote the entry but its reader rejected the empty `type` or `typeText` value, so such a step could never replay from a recording.
diff --git a/packages/e2e/src/cache/trace.ts b/packages/e2e/src/cache/trace.ts
index 6583e45a3..eabab07c5 100644
--- a/packages/e2e/src/cache/trace.ts
+++ b/packages/e2e/src/cache/trace.ts
@@ -529,7 +529,7 @@ function readRecordedAction(document: unknown): RecordedAction | undefined {
switch (name) {
case 'type': {
const target = readDescriptor(raw['target']);
- const value = readInputText(raw['value']);
+ const value = readTypedValue(raw['value']);
if (target === undefined || value === undefined) return undefined;
return { name: 'type', summary, target, value };
}
@@ -620,7 +620,7 @@ function readRecordedAction(document: unknown): RecordedAction | undefined {
case 'back':
return { name: 'back', summary };
case 'typeText': {
- const value = readInputText(raw['value']);
+ const value = readTypedValue(raw['value']);
if (value === undefined || typeof raw['replace'] !== 'boolean') return undefined;
return { name: 'typeText', summary, value, replace: raw['replace'] };
}
@@ -909,3 +909,8 @@ function readInputText(value: unknown): string | undefined {
}
return value;
}
+
+/** A typed value: `readInputText`, except that `''` is valid, since typing it clears a field. */
+function readTypedValue(value: unknown): string | undefined {
+ return value === '' ? value : readInputText(value);
+}
diff --git a/packages/e2e/tests/helpers/fixture-pages/forms.ts b/packages/e2e/tests/helpers/fixture-pages/forms.ts
index 12467815e..578906efe 100644
--- a/packages/e2e/tests/helpers/fixture-pages/forms.ts
+++ b/packages/e2e/tests/helpers/fixture-pages/forms.ts
@@ -126,6 +126,17 @@ ${rows}
}
export const FORM_PAGES: Record = {
+ // Clearing a field is a typed value too: the status reads `cleared` only
+ // once the input emptied after holding text.
+ '/clear-field': constant(`
+
+Clear field
+
+
Clear field
+
+
+
+`),
// Repeated cross-sell rows: several buttons share role, name, and test id, so
// every derived query is ambiguous and nothing in the query vocabulary can
// separate them. Only the reference the model selected can.
diff --git a/packages/e2e/tests/helpers/trace-cache.ts b/packages/e2e/tests/helpers/trace-cache.ts
index 03a6f7d7e..d9569e50c 100644
--- a/packages/e2e/tests/helpers/trace-cache.ts
+++ b/packages/e2e/tests/helpers/trace-cache.ts
@@ -203,6 +203,22 @@ export const FLOWS: readonly Flow[] = [
},
actions: [1],
},
+ {
+ title: 'clears a field',
+ open: () => '/clear-field',
+ body: ` await agent.act('type a draft, then clear it');
+ await expect(screen.getByRole('status', { name: 'Draft state' })).toHaveText('cleared');`,
+ script: {
+ 'type a draft, then clear it': {
+ act: (call) => {
+ const draft = nodeIdFor(call.prompt, /textbox "Draft"/u);
+ return [type(draft, 'draft'), type(draft, '')];
+ },
+ done: /"Draft state" text="cleared"/u,
+ },
+ },
+ actions: [2],
+ },
{
title: 'mixes deterministic and agent steps',
open: () => undefined,
diff --git a/packages/e2e/tests/unit/trace-cache.test.ts b/packages/e2e/tests/unit/trace-cache.test.ts
index af4db061e..f22d752be 100644
--- a/packages/e2e/tests/unit/trace-cache.test.ts
+++ b/packages/e2e/tests/unit/trace-cache.test.ts
@@ -69,6 +69,15 @@ describe('trace-1 entry', () => {
expect(entry.payload.startPath).toBe('/settings');
});
+ it('round-trips an empty typed value: clearing a field is a recordable action', () => {
+ const target = { role: 'textbox', name: 'Name' };
+ const actions: RecordedAction[] = [
+ { name: 'type', summary: 'clear textbox', target, value: '' },
+ { name: 'typeText', summary: 'clear the focused field', value: '', replace: true },
+ ];
+ expect(entryOf(trace({ actions })).payload.actions).toEqual(actions);
+ });
+
it('round-trips the postcondition: end path and end anchors', () => {
const endAnchors = [
{ role: 'status', name: 'Marker', text: 'saved' },
@@ -112,6 +121,7 @@ describe('trace-1 entry', () => {
['unknown action name', withPayload({ actions: [{ ...tap, name: 'click' }] })],
['tap without a target', withPayload({ actions: [{ name: 'tap', summary: 'tap' }] })],
['type without a value', withPayload({ actions: [{ ...tap, name: 'type' }] })],
+ ['select with an empty value', withPayload({ actions: [{ name: 'select', summary: 's', target: { role: 'x' }, value: '' }] })],
[
'oversized input value',
withPayload({
From 260ef9507b9eb6626a42faaede064432c5f3208b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kacper=20Kapu=C5=9Bciak?=
<39658211+kacperkapusciak@users.noreply.github.com>
Date: Mon, 5 Oct 2026 13:22:43 +0200
Subject: [PATCH 09/46] docs(examples): add standalone Vite project (#854)
* docs(examples): add standalone Vite project
* docs(examples): clarify Vite setup behavior
---
.fallowrc.json | 1 +
.oxlintrc.json | 4 +-
AGENTS.md | 8 +
README.md | 5 +
examples/.gitignore | 7 +
examples/README.md | 18 ++
examples/with-vite/.gitignore | 14 ++
examples/with-vite/README.md | 33 ++++
examples/with-vite/e2e.config.ts | 25 +++
examples/with-vite/index.html | 13 ++
examples/with-vite/package.json | 31 ++++
.../with-vite/public/e2e-lockup-32-white.svg | 9 +
examples/with-vite/public/favicon.png | Bin 0 -> 1012 bytes
examples/with-vite/screenshot.png | Bin 0 -> 29240 bytes
examples/with-vite/src/App.tsx | 65 +++++++
examples/with-vite/src/main.tsx | 13 ++
examples/with-vite/src/styles.css | 163 ++++++++++++++++++
examples/with-vite/tests/agent.e2e.ts | 25 +++
examples/with-vite/tests/greeting.e2e.ts | 26 +++
examples/with-vite/tsconfig.json | 15 ++
examples/with-vite/vite.config.ts | 8 +
skills/e2e/SKILL.md | 3 +-
22 files changed, 484 insertions(+), 2 deletions(-)
create mode 100644 examples/.gitignore
create mode 100644 examples/README.md
create mode 100644 examples/with-vite/.gitignore
create mode 100644 examples/with-vite/README.md
create mode 100644 examples/with-vite/e2e.config.ts
create mode 100644 examples/with-vite/index.html
create mode 100644 examples/with-vite/package.json
create mode 100644 examples/with-vite/public/e2e-lockup-32-white.svg
create mode 100644 examples/with-vite/public/favicon.png
create mode 100644 examples/with-vite/screenshot.png
create mode 100644 examples/with-vite/src/App.tsx
create mode 100644 examples/with-vite/src/main.tsx
create mode 100644 examples/with-vite/src/styles.css
create mode 100644 examples/with-vite/tests/agent.e2e.ts
create mode 100644 examples/with-vite/tests/greeting.e2e.ts
create mode 100644 examples/with-vite/tsconfig.json
create mode 100644 examples/with-vite/vite.config.ts
diff --git a/.fallowrc.json b/.fallowrc.json
index 7df1d7782..92733e667 100644
--- a/.fallowrc.json
+++ b/.fallowrc.json
@@ -25,6 +25,7 @@
],
"ignorePatterns": [
"docs/**",
+ "examples/**",
"**/next-env.d.ts",
"**/.e2e/**",
"packages/e2e/tests/tmp-projects/**"
diff --git a/.oxlintrc.json b/.oxlintrc.json
index 5369d67fd..82795af8c 100644
--- a/.oxlintrc.json
+++ b/.oxlintrc.json
@@ -16,7 +16,9 @@
"packages/*/tests/fixtures/**",
// Vendored conformance data, kept byte for byte.
"packages/*/tests/conformance/*/**",
- "packages/*/tests/tmp-projects/**"
+ "packages/*/tests/tmp-projects/**",
+ // Standalone user-facing projects with their own installs; nothing here builds them.
+ "examples/**"
],
"rules": {
// Sequential execution is the contract of a test runner, not a perf bug.
diff --git a/AGENTS.md b/AGENTS.md
index 14a0598c1..4a6de356d 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -107,6 +107,14 @@ suites that consume the built packages the way a user would.
"Committed recordings" under Gotchas. Scenario files are copies: keep
diffs against the source minimal, and name no company a scenario was
distilled from.
+- `examples/` — standalone user-facing projects, one per technology
+ (`with-vite`), each the same
+ one-screen greeter demo with deterministic and agent tests. They install
+ the published packages from npm, sit outside the pnpm workspace, commit no
+ lockfile, and run in no CI; oxlint and fallow ignore them. A change runs
+ the example's suite by hand and updates the "Last checked" line in its
+ README. A native example keeps its tests in an `e2e/` folder beside the
+ native project, as a user would.
- `docs/` (the Mintlify docs site; pages are the `.mdx` files under `docs/`,
navigation, theme, and redirects in `docs/docs.json`, extra CSS in
`docs/style.css`; `docs/examples/` is typechecked and shown verbatim on
diff --git a/README.md b/README.md
index 59df5057f..5c50d2631 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,11 @@ npx e2e init
config and an example test. The
[quickstart](https://e2e.tester.army/docs/quickstart) covers the rest.
+To see a finished setup in your stack, open
+[`examples/`](https://github.com/tester-army/e2e/tree/main/examples):
+The Vite example is a standalone project with a passing
+suite.
+
## Packages
| Package | What it does |
diff --git a/examples/.gitignore b/examples/.gitignore
new file mode 100644
index 000000000..c9ec8b777
--- /dev/null
+++ b/examples/.gitignore
@@ -0,0 +1,7 @@
+# The examples pin no lockfile here: each one installs the latest release in
+# its ranges, the way a fresh project would. A copied example can commit its own.
+package-lock.json
+pnpm-lock.yaml
+yarn.lock
+bun.lock
+bun.lockb
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 000000000..244ce8c5b
--- /dev/null
+++ b/examples/README.md
@@ -0,0 +1,18 @@
+# Examples
+
+Standalone apps with three locator tests and two agent tests each.
+Every app uses the same demo: enter a name and press Greet.
+
+| Example | Technology | Runs on |
+| --- | --- | --- |
+| [with-vite](with-vite) | Vite + React | Chromium |
+
+Each folder installs published packages from npm and works outside this
+repository. See its README for run commands. Agent tests need
+`AI_GATEWAY_API_KEY`; without it, they skip.
+
+## For contributors
+
+Run the suite locally after changes. Update the README's "Last checked"
+versions. CI does not run these examples. Oxlint and fallow exclude them.
+Do not commit lockfiles. Keep the demo and tests consistent across examples.
diff --git a/examples/with-vite/.gitignore b/examples/with-vite/.gitignore
new file mode 100644
index 000000000..6f8371de9
--- /dev/null
+++ b/examples/with-vite/.gitignore
@@ -0,0 +1,14 @@
+node_modules/
+dist/
+
+# What `npx e2e init` adds: the run output under .e2e/.
+.e2e/artifacts/
+.e2e/cache/
+.e2e/sessions/
+.e2e/report.json
+.e2e/ai-trace.json
+.e2e/junit.xml
+.e2e/summary.md
+.e2e/failures/
+.e2e/logs/
+.e2e/videos/
diff --git a/examples/with-vite/README.md b/examples/with-vite/README.md
new file mode 100644
index 000000000..cb7a323ea
--- /dev/null
+++ b/examples/with-vite/README.md
@@ -0,0 +1,33 @@
+# e2e with Vite
+
+A Vite + React greeting app with three locator tests and two agent tests.
+
+
+
+## Run
+
+Use Node 22.22.3+, 24.8+, or 26+. From this folder:
+
+```bash
+npm install
+npm run test:e2e
+```
+
+The runner starts the app at `http://localhost:5173`.
+It downloads Chromium on the first run if needed.
+
+## Agent tests
+
+Set a [Vercel AI Gateway](https://vercel.com/docs/ai-gateway) key:
+
+```bash
+AI_GATEWAY_API_KEY="your-key" npm run test:e2e
+```
+
+Without the key, the agent tests skip. Add `-- --no-cache` to use fresh
+model calls. Results are in `.e2e/report.json`.
+
+See [e2e.config.ts](e2e.config.ts) and [tests/](tests/) for the setup.
+For your own app, follow the [Quickstart](https://e2e.tester.army/docs/quickstart).
+
+Last checked with e2e 0.15.2, @e2e-dev/web 0.11.1, Vite 8.3, and Node 22.
diff --git a/examples/with-vite/e2e.config.ts b/examples/with-vite/e2e.config.ts
new file mode 100644
index 000000000..6c476b10d
--- /dev/null
+++ b/examples/with-vite/e2e.config.ts
@@ -0,0 +1,25 @@
+import type { E2EConfig } from 'e2e';
+import { web } from '@e2e-dev/web';
+import { gateway } from 'ai';
+
+export default {
+ // The Vercel AI Gateway serves the model and reads AI_GATEWAY_API_KEY.
+ // Only tests that use `agent` need it; tests/greeting.e2e.ts runs without one.
+ agents: {
+ default: {
+ model: gateway('openai/gpt-6-luna-fast'),
+ system: 'You are a thorough QA agent. Verify every outcome.',
+ },
+ },
+ targets: [
+ {
+ engine: web(),
+ app: {
+ url: 'http://localhost:5173',
+ // The runner starts the Vite dev server and waits for the URL to answer.
+ // Outside CI, a server you already started on that port is reused.
+ command: { executable: 'npm', args: ['run', 'dev'], reuseExisting: true },
+ },
+ },
+ ],
+} satisfies E2EConfig;
diff --git a/examples/with-vite/index.html b/examples/with-vite/index.html
new file mode 100644
index 000000000..497874e71
--- /dev/null
+++ b/examples/with-vite/index.html
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+ e2e - with-vite
+
+
+
+
+
+
diff --git a/examples/with-vite/package.json b/examples/with-vite/package.json
new file mode 100644
index 000000000..8b7dd3167
--- /dev/null
+++ b/examples/with-vite/package.json
@@ -0,0 +1,31 @@
+{
+ "name": "e2e-example-with-vite",
+ "private": true,
+ "type": "module",
+ "scripts": {
+ "dev": "vite",
+ "build": "tsc --noEmit && vite build",
+ "preview": "vite preview",
+ "test:e2e": "e2e run"
+ },
+ "dependencies": {
+ "@fontsource/dm-mono": "^5.3.0",
+ "@fontsource/inter": "^5.3.0",
+ "@fontsource/stack-sans-notch": "^5.3.0",
+ "react": "^19.3.0",
+ "react-dom": "^19.3.0"
+ },
+ "devDependencies": {
+ "@e2e-dev/web": "^0.11.1",
+ "@types/node": "^26.6.4",
+ "@types/react": "^19.1.4",
+ "@types/react-dom": "^19.1.4",
+ "@vitejs/plugin-react": "^6.1.1",
+ "ai": "^7.0.0",
+ "e2e": "^0.15.2",
+ "playwright": "^1.63.0",
+ "typescript": "^7.0.2",
+ "vite": "^8.3.2",
+ "zod": "^4.1.8"
+ }
+}
diff --git a/examples/with-vite/public/e2e-lockup-32-white.svg b/examples/with-vite/public/e2e-lockup-32-white.svg
new file mode 100644
index 000000000..fedbdaa2b
--- /dev/null
+++ b/examples/with-vite/public/e2e-lockup-32-white.svg
@@ -0,0 +1,9 @@
+
diff --git a/examples/with-vite/public/favicon.png b/examples/with-vite/public/favicon.png
new file mode 100644
index 0000000000000000000000000000000000000000..5f7a4e64be8f7800142f4e7090e2eb275ea8428d
GIT binary patch
literal 1012
zcmV?+6401W;L788tRGDx6yq=FevG&dyG1Z*Qlf
zq9W30G@cXC>2y?DTFOqG5dfB#ml=Y9z}VOrN28#ifV5hzX9OxME2RTh1Xfm77_+mp
zG&wn`z{0`;+aMBd6R54Nb?D3yshyo2T3cI_=@8i3+F~7SP)9BaWMpJGeOqTyaYX<~
zOiXl4KbqsQ#A#oG-N|ydwZL0&>eOOXfzrw798N~2ngo_>FMdV1VTeY`A`53
zts!)laAbqloSU0ttgf!o=H@0t*kM@N5Xi~Nkqq5=TSt=+y6*07*^2Y?^D-L34mLqS
zK}rI7d3irdAU;0+`v|nQwo*?|55uw`ySlmRpl9h
z;o)JLn3$mR^K&jRwA$6x6Xe
zLi6|c=brB9=%Bm1JLw$G#N5X(x&n~^spI2g>8GZqhC)I@NQHx7h_Kex)v*G=W&?l#
zrkoJGy{O>&`g%rcYO2&D4bb`d`O)a;C|7)Sb+yztH#hTtf`fyT
zi;GhLcfFIWI0Qrm3kZ4pU4_4CmTXkV|U^
z$;ru5k7Bg9UqC)TKkZ*edc9r}I5;@qDYm}8&Lpe}yuZIwOG^to1+ufV6@~cl@W3e*
zdN@b^A^ZFLj;~|=UlNOpi)@V6!Cq%gz#AzqFL%C>==FNe;l;%T`C6{8ukCKiDtl{E
zW@aXxot??n^M&;G_IkDzx3#tLioLtL>$P)4G&VNY^X~$)+01hQMR{{`qZl~u%H`!H
z&k47;Vzm__BO~eP=!i@v6K!m4@cIoR_xJY-;Mwi%t#k~+2s>tp&%O<_va(z*r5M{7
iYF>YusecH+PT&s*wR?8={y!i90000fa0i
literal 0
HcmV?d00001
diff --git a/examples/with-vite/screenshot.png b/examples/with-vite/screenshot.png
new file mode 100644
index 0000000000000000000000000000000000000000..0f28e4d0ca4905868d44b20bbe70206dca04f76d
GIT binary patch
literal 29240
zcmeFZ^;^{I_bxu_R!mS@a!ZJkf=b8UN(cx@BPk^!F?5H8(ntwPDIv`O(lsa@f^;)<
zGehSvbJo1y*ZB+1_qxvcT-W!f?YM<`y`JY;Yu)R<@AU*dQB}Bdk>Mf&fw-dhNLB-Z
zI17I~^Y{7F@Sp7?y*vcsDne2AUoDTswF$CB>fI9=!oObHG30N~zqxqpqEeA&-ZNZq
zZQKh=WiuS7I$B;OHK=x<+^i}5YZkp+;j3FJa_Ud4PZP^U&}+oHYWEgLCcpWG!~Pbj
z=Q0u!zH!|)lRtjkXArSZgzx)DyIhI%z1ROnoIxNyUO0URK79Tk@Zo;c<)w*gchgx$
zWKASI^T+E0q*u~1NO`=yOJbIU-Zo63ZRc{
zx2%x0_n-Hq%8?#`Xh3th9-bRFgu-;o5SbbVa&=8CW3_uZn06-*7%7GfoqvzFZWelO
zFQ_C4hrM>6^yJFcB4c%Nz}K!~y`yN%i6?s!yZxH_WwuXi9J><5h=lzx|Js@0Uw8^w
z1xxL9iFyOOQf-uRtrKQTCD;i!(-ab&nwJpX_gy8S%x>&^gZj6yYjs4z@gp>{c6Sh7
zKT4^$TrK4TtKU*)ir-3)bM(_6&pBh)w>ktQ4!*us@^%|8H2uKrC;JpdhCsB&=jfaY
zcc|`nlqTZ0OBfP)&JGnARl|?Wd)gf`YsILhc+9lXMEdPdSf$)Mk^kns^v9H4Av{si
z(}|b($-us-PgO#$ALH1&uYl5h_{B{R)BRNcbcLyvg=k5JV`@a5b-dvN!o94Bg
z8OYhi4i`m8Z!MjnkgkADatV_U_wmh7)jpVIRjXXZR)?U4josJp4>i`8SoftKBvxOo
za>p5eV0F_gzaso%BvmfYHgCbM9JATRmBIm!H}b^wv_r=yz7LnLM}^eUT&g$m*=(xa
zpG@E_r&5!g?!4P|F!h$=h=M+zH|^A!bNKZY;#2dm`&*-2{)9tVJC;5}t0Y%r&%-zq
zi*4o&!W|Y;N&S@<1qcJ
zEt*SVz#XEr!C07`QLpS_aZfEvq8!gT#^DDx$uXn0T6F*Ytd;Fbd
z>P3o}^K)ul^S_mTQb->>D`q;f`|*3i9kbh~CgZw1YKI-KocEtT$7s8;vys^wwaXc2
z>@^!EB4;f=x;lhHcS|1|I&I1c;`hdoxQf27LN|L;nad*hgV_iXjL&v{
z`2=kPWERKircSZQMS-e~H!cU2i^&F7N#WKX0o_IMPSZ5}E
zf(6oY{Y+k;9hfa}GL)_uoqDUPgTI=CX5hQZDjW-r4!;v;SelSQ^`*pG;*KN
zFMZimzc%s%j_W(2=Cwk%4>h0sPL7gxsxcW&lN%#*ZO7PZOz8EyRd#>)>_)yClxq?W
zT7D@GpmYV`yISAp{WS%fU1mGtl;k>i+wL8#uiHxha$1D6(x->JD+Bwm-{}v^(MKB%
z6b2=>G-uwDQ5@}~3EIu^t}FdrR}^gP_P(H9SNDi-u&~C?yO@H?-9ZC7enWbS{c&eh
z$w_SyPS|FU^k%JvIaIH#et#)#=gv(RsR*7~#2;>Z@2=1U-Ftw$nP9usC4pn|n0l@9
z<(}B|JF2pAr;elD)fQzIpN*g9;bIG4-;xLSGYm|%)~l0+FUy89J$13$C}t3Kc$OfX
zk3QKOYf157dnyk{d~LAQ=JGwe-g~ue&eqaL2gHn>GZdmz#U1=%Z=5x>L~U5_B0|MOlu5@84*_SVM~w(!_$TvRG+Ymx|Z5}vPyaI{)(
z-FNfX#^GWLakGWhed*7QoV0890u@tltoyzvx%4Vti)W`{^_+|JKOV!BO?s_DN`|NV
z^X=g!w4edY@;ZveYw4S1?TPjah}s4U8R|1BK3i=I2DX*`n-s(TM4o~J8*%)MvH-Jl
zXVG#Mo%@y|dJK-D*)xpqeuyI_0`cl{F59UJe?hhW^pRiJ^$5wWCC3(4
zlNqFRt%RE`!z;4OOi#=Z()f)AMpT*pWkbjLSkiLF>d0YFpTLhP9J&u^>#jqQR99fW
z|E&I%+xjS!G$~)%Yf+z32akT>Za;$#CroP8JZQjCVy-jpR^i<WuAxkIqZ>0}Ekn`!696+Ky
zN6th)KVvZ>H%#dEHcz|g+ga2ieY9k*7$9qXlx_O3PmeJDfi)Sj9c=k>jpw%T^2l~0
zmD*ZgrhADR0ly7__xwP!p*K$Qc)PoG_E+4vQ3NucUr#wx|v
zex*NK-D1);&L~`JFMj8^L4nzKXEs7$JMI(2nhw7<7vh6_7{hvD134L!L;XSml4V->
z5Xbpveh0pg34WF~y|0e{aMJ6KCvqnE0bx8e3tm%(z&y-Hj$3G5&WYgAGq7eUb9s
zgPKA|Go=`PQx0MLedieRqM=KVoT41_y}7Rk-a|(20%}Jp!E9J^2`qJX=|;9Y##bac
zSo;`O*MwC}h`_r4v!Xcj6QObQES<5((Xu~xjlj^GvzJswH)WUUAfp6wYoF>`9?TyL
zpfj%U*qj;X{G;r;nqN+srtp8novxBFhzw_8K9gWuwH$L>GrGNS`aL5#{U5acmhu~;
zC|*zJodi_fo_?OXhPpZn#<^Qcmx#kAi#ac(MM&v*%)TIM#3@cN2nMZ30jtC99H@@)$(;w#*1A^
zeJRr1_D~${;dt?&YMgrH=xPra655;sU)r5`?Mf48w%fUh0NS4{=XfAg@4xK7CG=9<
zFkink>q4f#LKs6SAfCTSk>4tdfNJ=P+m#NU4sk0CaT#Ll=J|T~VDaYv;SUeOEI5Y%
zt&(1X2H75SyRAJ>f3`V{3LZRb#qji%{~53G|MC+5|IhaSb_kI6$J>`2;`YB>!2eO)
zJ_t)!8unVvKi*l^gfwAOP`Loj9wa^QeXInwVit-Ei|?z<@_fUBz$OhYX)=-NV!brWJj|R2PxeB=bN+Fnw--h`?~{H4SLOQ
zR8{h#hi!;uHiAj4_`s!iFa8nlOG$&7$3q6lT2_Dj=61J?8f$a-YJYZLmU^1m{W$CI
zc4K8m&|Y0^f%Y_OG^dsG4%YDwvB8dQ*-k!S>T)Y9Pe;DmgEOFfaErQ+~
zwNVqDewQrey`aLuX7<>0w8XmfNG#x@P2gf}Cim
zxRsU!2B3(@q=)&&)X)1igO9nnA2|JZ{+msM->~A&cjAauiky@u0e`sJZyq5<`}AIW
ztKdgQVi?;s*|BmY5m*Thnl0$56rP1Qg@0~tcvQBWp8BdL?{3l^(+2@`!1z4l)V4&v
z8OTtMTTJ%02f8E@K|+|0v&|VpCtb->67Fq|0dfd?SQ}XkiHsn~+X*YRv|XfCW>Kt$
zO;+sL?Is}_Xn&!gBib|DbzY5?SZ~jS2n=RxTD`p@T(Mp@reAK)%!jNye6Dew4GYKb
z3m|Ww_0pleap&%=bGo7k;0%H)|4BZUc7vS)`eon6V=(Wjn&hUeI8z5}0Or=&{k4(1
z#rx0}MqONX!pK13B8c*_xR*X||TqA{MY-uI;xo)6Z-#cC!L76=&j!ViY^Q!C$?m
z0f7>K42La6ON-Y|7M`a$^g0Qnp3@pZ;Nj$#ATiRXH|`G^QjvHH-KX=I^d-ur^R=Zm
zL*Y-DO@(G&a7?E^FcWh|kHzqu6@dMsdlhmF*s2cNk1T6w2Bvdz3zWG36Hw4cLUDV=
zn_Y}K<+q#>caDOEX+(%S@!DHxkBI;sIAb|(j=^~k{Hdt^Cd}o8$A?UmMvwq~4zOP1
zfOGz{`1!E-Tx!Wai%IU-s?`GE5M2AMiF@Vf3=uWynLu{0y|wF2W3URlt3v`atMC?y
zyw$hx-(}hu?|zZ-Cv<+kYhIz6B<_FMVUnYZs$0p)m7hO^9zN{wW-RyV4e8^x?N;`P
zIeqRseRky&ZXJB(k3Vp!CejP2!QSUoy{u#u$nUqWTK*0=u%hTmU=jev?;fpat;XQG-SINNdE1lp1L5r}w5
zyChJRlf}*Q>UeMY#~8Ny3e%mhav`(<-34W%FOwwPRX-39T4Ww4-dm3R!0I{mlS~G3
z6bYH-WV9QfuGOh_E1z`QhW)-bSARzNF*;z-9wTkGk&CLqXZER~v~zU;s{TWxasSLl
zc4p#Mud;wWw76}>x$0@nz^?7A-Bm-@Y&QRAr{#bgjWcqyrgpL!DoEn}0F9l@=7IT{
zbC*dRVHRbMKBDemCWKt1}7}
z68-=D{J=qal+GMgg!IO%GpWf2!+wJ!i{mbRDwb$BD0ZZ^#MK=qe_-`;fabc}+ij};
zHLP+nBxDVH2+N3HPQWu7l;Y)phptQ7U5A4&O*n`Ko|)gcMl{;@{g7yPc5;!$;wBn&%m9p(f1_wOp3
z9i(H3B;LJ#w!ZJG3BtZd&?~|@n9@~uJ-uP@8>7}
z;Yppi(8B6JgY(l(4%)#^*tEWW=r(HIBJR2pBl-)-B@_RXUH_9+3@U=@>8Na%tCwoA7^ZB>|S=jJF
z#urz+U$?W>p%`%QW&ewfum1wOYh^n&Oc%f|8%bO*$FS`6Yf?zWD<%%=%Benn0
zL-51Zgu~Fg?FANlt3$Xq+ub~dp_11Y%LIg+<`^@R-A1`md^AK=q4mAEjlbJ1c?`9S
zbKY&~+lRG^*>KOW2&p4I4Np>XZIQuG!(ZhCCvH6}OCFj}N5r@tuLEdMr5=lR3W>sb
zE+$uRcfD1XNchOj4ILc?ns@?q*uv2&Qtx}ER+X@wKC5%5pf|2mcWt$}C3Ic5q7}G#
zhOtE;(6z6ggT5#;c>HV?WX}Lo^YIEtb$gXhJ;3rY(=^pIGu(ffS30PlQOj|9csVo4
zZ79i|_x=g(P^KEr^kK^Q^VR%IDGcJadZx2wW8{z9`BInRw|*WvfTlatXErQ7N;Zka
z#7ZouV`3xyLP#P&vd_$B#`9l^z+%>j&IACo$z9tqFg!Wh?LOE}G4VUh09NTHQK4+q
zslzJ8b0M&SLV)1A$lz3(>}DoMcee`Uv1g3sA-%hX?>B2*AvGhMhUuS8LBlrbu65xYugKMA+$z
zJ=mRPB%Yx1%ST^+Hm0=kA{;LAxt54dYz(0Z^Ia15NWc-rYlAO`u;0j7hb^L$+fdkd
zRN-jY0Y;Kw^oV!6uOVd=A{1)Vv$I)jr#u!qQI{Ac+}1v7wZ@lY_-zes^4&PYeV{Je
zRyNTp$^jE~5{xDQ@Ffg|#{=7UNucJV$n#vC{C1OJCOwTxkKw}1D5YYY^zmh;ipw<1
zi6?s#<;317yk(q`IHbww`9dhIliuA@wch(mesccRM=L+NB<~`1`EwV3C&=tfy#*Mg
z1nbqVBUg7u(!4nixP-RcpAZ{j?PV#xdryl)3u^bOtdc#2QALf*Yqk5vkxNR6t^2^|
z!MRjR5_fHmGa1Y`$TV*WUj+yY2oFMJA?&(q4|}*M9xAT;X0xf9)O`;J0$*O)gxeG_
zSt&6OO~u37_c~~5uV^RV@`AV>X);*oPVvV8x3$&Ilj0}txVMG_+&59Fxu};7Sb?Sv
z-+7ln;`-u*H-URSsy{=1YcqSeDaC)YNrCESD67o$teSLzZn<%Su${Pzo_GDLQvq?N
zLnsp>Zpf%y1x=u-xLkLd9Oz8ny^-n?D;WTjqPbC%_)7bvD)jiZDivVHCv$)s3Se)k
zGIX7%!G$q1@+3f}jm?DPl;ThDJla`SXXl&LgFRp@ATHWh<2C`daCVrPMP`J<3HZ^$
zUt3ZrMLG1;D&=I@JVJrD(>rJViwz{Z(poROFGGH#*_m)!T~_+7D?gp@-rTA=R;OUP
zGnJD|;ClMy5g`D5o`xzfkC7K1#jA;R(zja*D%BxCtXLc1_ErFo;3~Ge?skjL!s+Vv
zJ_KTlHAL5}D&&@T*N9b|XdlQeiC&}2z+>?EKob9?!0Or$44-jYGf_O5-747|E9Q@%
zz1?=P3_24Nho6U>(z`&vjeGQ|6y3j=5Z#&FN63FZ(*cX+!#>*UH-DB7y8;A2P6?E_-3Y
zz`o4}2`m%j2;&+{&=S?3{ASFDrf~jHf35!YnTw3dHARC?!BJ}gbVfTLWbj2EM|^}e
zIXV-=8NU(x+S^niL4?XMxYgbilL-b+hzBpx3bazFDUk))LI;iPsk;jE2Dl{iA=aeV~B(~$kEwB94rY`u
zQkkbn`)=t&_R$gTBb*#@D&KYZhp^l2yx7GoXeO&M!e5Qwf<(h=8L=&+OUy4UY7AtV
z|8sxHHhPGj^Un=3W|vjFa+=9665WvJsjgQPo)iMttdlhH+kaO+bjHNe6b)1t_w>&{%CB~
z`34fU`qc0oBo*sEag^iQa1pz)k7G9K1QH^uW-F(rxz*1TbKdvpvlT7h%61*B$bY#2
z{TPu^4gbs8(4+xz+G+z*BwgID_(p~ol+1!^EKZqu{bU~_P?PIc?Z!E-zV;5dUfY`K
z_x`au|MKlXq{`@TJZu?*lM#UIFl2!c&^4WvA$p#K2-L>Jj2$kT+yN=OrrD$*a?&!6
zv=ba8idY7}8FPW^W{i~TT&L9Du+*;&Wr6VcJA?+P%*Xwjs>?vv7e7y@MB;Y3s&O~x
zw-2b)sD<5dleGs8AZwIuSb3KZn}xK>DJ~!Fj(rC%!81(llMC~fllXjp-^Fdupo?g1
zwb#!h_ZR3dvLmzxeZaaJS-e8FPi5vGTiI
zDC7LuS`fw|7ahbDppVfvR&IEIW!nrdY!J5*{fp?lJHY7|TXh*URPWV%LC-J)+q$R4
zX0j#x_};BBVx1#2s%=8SO@6*}tu@VIB9j%6qe+M6dX5%?JVhnBX5DQ}2EV}$KKcDJD3z`U7bUf6z|%#&}TdXa2uT2032U}ETXhRIo@H2E;Q
zvJr9AW~{DB{!Yqi?xHtFUw$E&2KCItW+;HV(Gl&6FaKofy&y?^KtS^5Yid;1X{O(+PYWmPvrd0WmjaKVRgrEmXCXNd_42vGK*x^hP!(Q
z?b%wtFoViU6h#C@t64ZR??y4z<_{{ov2o*tS|&T)oY;0p9eXwl`ek=2;6St^Kw{I(UMUbzq}P8
zY4`HNDbe8iBDcy9Mpbz!BEQBfok6%J(KXSQJHI=TTl%A=b_Vd4mV7tk6cU6F-PeOp
zw0&wnDRL}=G;|BMm}gMVKg0>^U9X}26Orh({kwdI_@i6Q3ZyFrr>Rpk74zEBH&qG_
zL6RHsN&pph7cejHVy^I{$1IkSz}8;5n)gF>dbda{#E($9!M*gLA=!&t0D-D}of#c`7&v%J6X3Icj{#x51&;@M?&sTL~-Bz7|DXQRRe5FZ}?84-#T$U*ZdJ5Za
zO2uWTlY9fhFP;#;!g)Wo@?cwnR-2zI#YH2ty>8TP_>bvgqOY6rEaY^h%k^i-7RaWc
zeO9Y6fcnPJBlas
zvNRV}$82j|H$NS2FVan(TmI1&ZQjOSPejQv+}VIN=wNhSXY&ku_#t+qw0B->d5BUC
z%1F2buG7Z0#%mTH{SqzjV{U^E9ZuQ4M}=)p2SA=|W8Y5zD@4jtm!Dp!Z&LXzPfL7PC)eON8QgEGjSu51dejqf`!X=9vO^oxQ<$X=Yv@yg}&}xIM
zUid;W?(aXcKSI*K;E0hle;i3^#nnY{=;XHw!(y5K#IeM_063`%rk@$U+`M8Xjy*CK
zk3tXg>D_Kz&9kFU2hTV-
z;?P6_PE~vv>x~T$wx6zz*&lX}_=WH}TF#97!BCj}VhSKYm~I|G4nGd@^_(JToGb^seKZ
zT;rZC>z?a2W!6-#Lo-d?;0{1S|H+g1>)iz^Rf&@wIJ8`r`YY}cAt7uVL0kmXy$bj@
z3PwjDuww+cc2~<)Bs;wa{(Orio}HpGUFbx*O}NI@%mcT#R=Jpzf`1R$-3IzgPpf|}
zMl@{YLWlO_`Qh2Qi^MEI{~t+c(ypA@qhDQN{{Eyx!xb9kC*4x}8mJ%kG9%v?${`u{?_n*{_NDP!@cDRiIq+w()8S%36aTl
zmPD@eyoG+P>FL`VoS6IiXsKz#d*_8tMZbpc-SI_k$9qi%Wl$ag)o8Erv>4b|TEf5C
zX2?`YAQ?+yO`)jKixj*|@mTLkQ_xVK<$8R})^;$@bMu2ueyPs1s8yopbI)F6ai|P(
z8;&2XUxEmwhxpZu6K^Z_
zSjoo!BQb?`uN{T{DvVL2%iB#3kLV5F-it54#JE3h9$_(Jai+1
zuizYiE!=!{vakGU0=UqsS#pWF``{aM@0Q$t3}lSfdv3~CT#w_Qc;ws0Tm_R*p2bM1
zbzhaySXsb2*ub
z<;0BDr3>&(tOJpo}reYhGo6F?0BKBy