diff --git a/.changeset/ios-webdriver-element-types.md b/.changeset/ios-webdriver-element-types.md new file mode 100644 index 000000000..18e659f7b --- /dev/null +++ b/.changeset/ios-webdriver-element-types.md @@ -0,0 +1,5 @@ +--- +'@e2e-dev/mobile': patch +--- + +iOS devices driven through agent-device's WebDriver runtime (hosted Appium clouds such as TestMu AI, BrowserStack, and AWS Device Farm) resolve roles like a local simulator does. That runtime reports XCUITest class names (`XCUIElementTypeButton`, `XCUIElementTypeStaticText`) where the native runner reports `Button` and `StaticText`, and the engine read them as unknown types, so `getByRole('button', 'CPU Load')` matched nothing and `observe` listed the node as `xcuielement-type-button`. The `XCUIElementType` prefix is now dropped before the type is mapped, so a button is a `button`, a tab bar's buttons are `tab`s, and the navigation bar titles the screen. diff --git a/.changeset/testmu-devices.md b/.changeset/testmu-devices.md new file mode 100644 index 000000000..f610a0367 --- /dev/null +++ b/.changeset/testmu-devices.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/testmu": minor +--- + +`@e2e-dev/testmu`: `mobile({ device: testmu({ device, osVersion, app }) })` runs a mobile target on TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real devices (`deviceType: 'real'`), one per worker slot, through agent-device's `testmu` cloud provider. Each slot leases a device from an agent-device daemon the provider starts for the run under `stateDir` (default `.e2e/testmu`); the worker's first command starts the TestMu AI session, which installs `app` (an `lt://` id, an `https` URL, or a local path), and releasing the lease ends it. It reads `LT_USERNAME` and `LT_ACCESS_KEY` from the run's environment and fails the lease before starting anything when either is missing, groups sessions on the dashboard under `project` (default `e2e`) and `build` (default the run id), heartbeats each lease while the run holds it so a session slow to start keeps its lease, releases a lease it allocated when the run was interrupted, and releases each lease once. It needs an agent-device that includes the `testmu` provider, which no published release does yet, with `@e2e-dev/mobile`'s agent-device overridden to the same version; `agent-device` and `@e2e-dev/mobile` are its peers. diff --git a/AGENTS.md b/AGENTS.md index 6340a3daf..c12d1fee0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,12 @@ suites that consume the built packages the way a user would. hosted iOS simulators and Android emulators for the mobile engine (`DeviceProvider`). Expo publishes no SDK for the sessions API, so it calls Expo's GraphQL API with `fetch`, and `@e2e-dev/mobile` is its only peer. +- `packages/testmu` - the published `@e2e-dev/testmu` package: TestMu AI + (formerly LambdaTest) hosted Android emulators, iOS simulators, and real + devices for the mobile engine (`DeviceProvider`). agent-device's `testmu` + cloud provider is the vendor client: the package allocates agent-device + leases from a daemon it starts per run, so `agent-device` (which must be + the same copy `@e2e-dev/mobile` uses) and `@e2e-dev/mobile` are its peers. - `apps/testbed` (`@e2e-dev/testbed`, private) — dogfood project that consumes the **built** packages like a real user would: the playground app where every runner feature (sessions, routes, downloads, frames, uploads, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cc2969edf..93827362c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -254,5 +254,5 @@ build against, and what a third-party engine builds against too. A change to that contract bumps all three packages together in one release, with a changeset for each, so an engine and a runner from the same release always match. Engines declare a peer range on `e2e` that points one way only -(engine to runner, `>=x <1`), and each integration (`@e2e-dev/kernel`, `@e2e-dev/eas`) does +(engine to runner, `>=x <1`), and each integration (`@e2e-dev/kernel`, `@e2e-dev/eas`, `@e2e-dev/testmu`) does the same on the engines it plugs into; do not make it mutual or narrow it. diff --git a/README.md b/README.md index 9ebbc17c1..0bcb1f5a2 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ config and an example test. The | [`@e2e-dev/github`](https://www.npmjs.com/package/@e2e-dev/github) | Reporter that posts results as a pull request comment. | | [`@e2e-dev/kernel`](https://www.npmjs.com/package/@e2e-dev/kernel) | Kernel hosted browsers for the web engine. | | [`@e2e-dev/eas`](https://www.npmjs.com/package/@e2e-dev/eas) | EAS Simulators hosted iOS simulators and Android emulators for the mobile engine. | +| [`@e2e-dev/testmu`](https://www.npmjs.com/package/@e2e-dev/testmu) | TestMu AI (formerly LambdaTest) hosted Android emulators, iOS simulators, and real devices for the mobile engine. | ## Documentation diff --git a/docs/docs.json b/docs/docs.json index 19b432cee..52905d389 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -203,7 +203,8 @@ "pages": [ "integrations/index", "integrations/kernel", - "integrations/eas" + "integrations/eas", + "integrations/testmu" ] }, { diff --git a/docs/examples/mobile/testmu.config.ts b/docs/examples/mobile/testmu.config.ts new file mode 100644 index 000000000..eacf33ccf --- /dev/null +++ b/docs/examples/mobile/testmu.config.ts @@ -0,0 +1,35 @@ +import type { E2EConfig } from 'e2e'; +import { mobile } from '@e2e-dev/mobile'; +import { testmu } from '@e2e-dev/testmu'; + +const apk = 'https://prod-mobile-artefacts.lambdatest.com/assets/docs/proverbial_android.apk'; + +export default { + targets: [ + { + name: 'android-emulator', + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Galaxy S22 Ultra 5G', osVersion: '14', app: apk }), + }), + app: { bundleId: 'com.lambdatest.proverbial' }, + }, + { + name: 'ios-simulator', + engine: mobile({ + platform: 'ios', + device: testmu({ device: 'iPhone 16', osVersion: '18.0', app: './build/MyApp.zip' }), + }), + app: { bundleId: 'com.example.app' }, + }, + { + name: 'android-real', + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Pixel 6', osVersion: '14', app: apk, deviceType: 'real' }), + }), + app: { bundleId: 'com.lambdatest.proverbial' }, + }, + ], + workers: 1, +} satisfies E2EConfig; diff --git a/docs/examples/mobile/testmu.e2e.ts b/docs/examples/mobile/testmu.e2e.ts new file mode 100644 index 000000000..5602626dc --- /dev/null +++ b/docs/examples/mobile/testmu.e2e.ts @@ -0,0 +1,8 @@ +import { test } from '@e2e-dev/mobile'; +import { expect } from 'e2e'; + +test('Proverbial home screen', async ({ app, screen }) => { + await app.open(); + await expect(screen.getByText('Proverbial')).toBeVisible(); + await expect(screen.getByRole('button', 'GEOLOCATION')).toBeVisible(); +}); diff --git a/docs/integrations/index.mdx b/docs/integrations/index.mdx index 10f027b96..b6cebe12a 100644 --- a/docs/integrations/index.mdx +++ b/docs/integrations/index.mdx @@ -16,6 +16,9 @@ integration decides where it comes from. `@e2e-dev/eas`: hosted iOS simulators and Android emulators for mobile targets. + + `@e2e-dev/testmu`: hosted Android emulators, iOS simulators, and real devices for mobile targets. + A service without an integration is one small provider in your project: diff --git a/docs/integrations/testmu.mdx b/docs/integrations/testmu.mdx new file mode 100644 index 000000000..ddd216fd0 --- /dev/null +++ b/docs/integrations/testmu.mdx @@ -0,0 +1,240 @@ +--- +title: TestMu AI +sidebarTitle: TestMu AI +description: Run mobile targets on TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real devices with the testmu() device provider from @e2e-dev/testmu. +--- + +`testmu()` from `@e2e-dev/testmu` is a +[device provider](/mobile#hosted-devices) for +[TestMu AI](https://www.lambdatest.com) (formerly LambdaTest). The run gets +hosted Android emulators, iOS simulators, or real devices, leased when it +starts and released when it ends. agent-device reaches them through its +`testmu` cloud provider, which drives each session over TestMu AI's Appium +hub. Tests, config, and CI stay the same. + + +No Mac, Xcode, or Android SDK on the machine that runs e2e. + + +## Setup + +The provider runs on agent-device's `testmu` provider, so install +`agent-device` beside the package, at a release that includes it: + + +```bash npm +npm install --save-dev @e2e-dev/testmu agent-device@ +``` + +```bash pnpm +pnpm add -D @e2e-dev/testmu agent-device@ +``` + +```bash bun +bun add -d @e2e-dev/testmu agent-device@ +``` + + +`` is an agent-device release that includes the `testmu` provider. +No published release does yet: until one does, install agent-device from a +build that includes it. The package's `agent-device` peer range excludes +only the releases known to lack the provider, so it cannot tell you whether +a newer one has it. + +`@e2e-dev/mobile` pins its own agent-device. The provider starts the daemon +from your copy and the engine's workers send commands through theirs, so +both must be the same agent-device. Override the engine's pin: + + +```json npm +{ + "overrides": { + "agent-device": "$agent-device" + } +} +``` + +```yaml pnpm +# pnpm-workspace.yaml +overrides: + agent-device: $agent-device +``` + +```json bun +{ + "overrides": { + "agent-device": "" + } +} +``` + + +For npm and bun the `overrides` field goes in `package.json`. + +```ts title="e2e.config.ts" +import type { E2EConfig } from 'e2e'; +import { mobile } from '@e2e-dev/mobile'; +import { testmu } from '@e2e-dev/testmu'; + +const apk = 'https://prod-mobile-artefacts.lambdatest.com/assets/docs/proverbial_android.apk'; + +export default { + targets: [ + { + name: 'android-emulator', + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Galaxy S22 Ultra 5G', osVersion: '14', app: apk }), + }), + app: { bundleId: 'com.lambdatest.proverbial' }, + }, + { + name: 'ios-simulator', + engine: mobile({ + platform: 'ios', + device: testmu({ device: 'iPhone 16', osVersion: '18.0', app: './build/MyApp.zip' }), + }), + app: { bundleId: 'com.example.app' }, + }, + { + name: 'android-real', + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Pixel 6', osVersion: '14', app: apk, deviceType: 'real' }), + }), + app: { bundleId: 'com.lambdatest.proverbial' }, + }, + ], + workers: 1, +} satisfies E2EConfig; +``` + +`device` and `osVersion` must match TestMu AI's catalog exactly, as the +[capabilities generator](https://www.lambdatest.com/capabilities-generator) +lists them: a virtual iOS device takes a version like `18.0`, a real iOS +device one like `18`, and Android one like `14`. TestMu AI checks them when +the session starts and refuses a name or version it does not list. + +Keep `app.bundleId`: TestMu AI installs the app when the session starts, +and `app.open()` launches the installed bundle id or package, not the upload +name. Leave `app.appPath` out; a target that names one fails its lease. + +## Authenticate + +Set `LT_USERNAME` and `LT_ACCESS_KEY` to your TestMu AI username and access +key in the environment `e2e run` starts in, such as CI secrets. Without +either, the lease fails before anything starts, naming the missing one. + +agent-device's `testmu` runtime reads the credentials from the environment +of the daemon that drives the sessions. The provider starts that daemon from +the runner process, and agent-device starts a daemon with the process's own +environment, so the provider copies both values from the run's environment +into it first. + +## Install the app + +`app` names the build TestMu AI installs on every session: + +- **An app id**: an `lt://` reference to a build already in TestMu AI's app + storage. +- **A URL**: an `https` link to the build. +- **A local path**: a build on the machine that runs e2e, resolved against + the directory of `e2e.config.ts` and uploaded when the session starts. + +The build must suit the device: + +| Device | Build | +| --- | --- | +| Android emulator or real device | `.apk` or `.aab` | +| iOS simulator | a zipped `.app` bundle, not an `.ipa` | +| Real iOS device | a signed `.ipa` | + +## Real devices + +`deviceType: 'real'` leases a real device instead of an emulator or +simulator. Real and virtual devices have separate catalogs, so check +`device` and `osVersion` against the real-device one, and give a real iOS +device a signed `.ipa`. Real devices need an agent-device release whose +`testmu` provider supports them. + +## Options + +| Option | Default | Meaning | +| ------------- | --------------- | ------- | +| `device` | required | Device name exactly as TestMu AI's catalog lists it (`'Galaxy S22 Ultra 5G'`, `'iPhone 16'`). | +| `osVersion` | required | OS version exactly as the catalog lists it for that device: `'14'` on Android, `'18.0'` on a virtual iOS device, `'18'` on a real one. | +| `app` | required | The build TestMu AI installs: an `lt://` app id, an `https` URL, or a local path. See [Install the app](#install-the-app). | +| `deviceType` | `'virtual'` | `'virtual'` for an emulator or simulator, `'real'` for a real device. | +| `project` | `'e2e'` | Dashboard project the sessions are grouped under. | +| `build` | the run id | Dashboard build the sessions are grouped under. | +| `sessionName` | TestMu AI's | Name of every session on the dashboard. | +| `stateDir` | `'.e2e/testmu'` | Directory for the agent-device daemon each run starts, relative to the directory of `e2e.config.ts`. Keep it out of version control. | + +An option not in this table, a missing or empty `device`, `osVersion`, or +`app`, or another `deviceType` fails the config load with `INVALID_CONFIG`. + +## Write a test + +Tests are the same as on a local device: + +```ts title="tests/home.e2e.ts" +import { test } from '@e2e-dev/mobile'; +import { expect } from 'e2e'; + +test('Proverbial home screen', async ({ app, screen }) => { + await app.open(); + await expect(screen.getByText('Proverbial')).toBeVisible(); + await expect(screen.getByRole('button', 'GEOLOCATION')).toBeVisible(); +}); +``` + +## Sessions + +Each target leases up to `workers` devices (`--workers` overrides it), one +per worker slot, before the run's clock starts. A session takes about 30 to +75 seconds to start, allocation and app upload included. TestMu AI +runs as many sessions at once as the plan allows, so keep `workers` times +the number of TestMu AI targets within it. + +The provider logs each lease as it is granted: + +```text +ℹ leasing 1 android device(s) from testmu +ℹ testmu (1 of 1): lease 4f0c…: Galaxy S22 Ultra 5G, android 14 (virtual); the session starts on the first command +ℹ testmu: leased 4f0c… +``` + +The video, device logs, and network logs of every session are on the +TestMu AI dashboard, under the project and build the options name. + +## What the provider does + +- Starts one agent-device daemon per run under `stateDir/` and + allocates a `testmu` lease from it for each worker slot. +- Keeps each lease alive while the run holds it. A lease asks for + agent-device's longest inactivity window, 10 minutes, and the provider + heartbeats it every 2 minutes, so a session that takes long to start keeps + its lease. A failed heartbeat is logged once and does not fail the run. +- Hands the worker the lease scope and the device selectors as agent-device + client configuration. The worker's first command starts the TestMu AI + session from them, which installs `app`. +- Releases each lease when the run ends, which ends its session. A lease + is released once however often the engine asks, and a failed release can + be tried again. +- Releases a lease it was granted after the run was interrupted, or could + not hand to the run, before failing the lease. + +## Limits + +Sessions run over TestMu AI's Appium hub through agent-device's WebDriver +runtime, so what agent-device does not support there is not available to +tests: + +- `device` settings: permissions, location, network, and appearance. +- System alerts. +- Recording through agent-device. Leave [`video`](/reference/config#video) + off for these targets. TestMu AI records every session itself, and the + video and device logs are on its dashboard. +- Device logs through agent-device; read them on the dashboard. +- Port reverse. To reach a server on your machine or network, use TestMu + AI Tunnel. diff --git a/docs/mobile.mdx b/docs/mobile.mdx index f1f94c8d8..27577eaf6 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -146,7 +146,8 @@ ends. The engine drives each lease through the agent-device daemon the lease names instead of the local one, so any service that runs an agent-device daemon next to a simulator, an emulator, or a physical device works. Nothing in the framework knows a vendor: for [EAS Simulators](/integrations/eas), -`easSimulators()` from `@e2e-dev/eas` is the provider, and for +`easSimulators()` from `@e2e-dev/eas` is the provider, for +[TestMu AI](/integrations/testmu) `testmu()` from `@e2e-dev/testmu`, and for any other service a provider is a few dozen lines in your project. ```ts @@ -302,7 +303,8 @@ on `app.bundleId`, so traces recorded locally replay on hosted devices. A device cloud agent-device speaks to itself needs no session API of your own: the provider allocates a lease through a daemon it starts for the run and -hands the worker the lease scope as `client`. +hands the worker the lease scope as `client`. `testmu()` from +`@e2e-dev/testmu` works this way for [TestMu AI](/integrations/testmu). ```ts title="device-cloud-provider.ts" import { createAgentDeviceClient } from 'agent-device'; diff --git a/docs/package.json b/docs/package.json index c55d4f486..8050c5c70 100644 --- a/docs/package.json +++ b/docs/package.json @@ -10,6 +10,7 @@ }, "devDependencies": { "@e2e-dev/mobile": "workspace:*", + "@e2e-dev/testmu": "workspace:*", "e2e": "workspace:*", "@e2e-dev/web": "workspace:*", "@types/node": "26.6.2", diff --git a/docs/reference/environment.mdx b/docs/reference/environment.mdx index 767e5d6d3..217ec2186 100644 --- a/docs/reference/environment.mdx +++ b/docs/reference/environment.mdx @@ -103,6 +103,7 @@ What is sent and how to read it is on the [Telemetry](/telemetry) page. | `KERNEL_API_KEY` | value | The API key `kernel()` from `@e2e-dev/kernel` creates browsers with; unset, the browser lease fails. See [Kernel](/integrations/kernel) | | `EXPO_TOKEN` | value | The Expo access token `easSimulators()` from `@e2e-dev/eas` starts and stops simulator sessions with; unset, it uses the eas-cli login in `.expo/state.json` under `HOME` (`USERPROFILE` on Windows), and with neither the device lease fails. See [EAS Simulators](/integrations/eas) | | `HOME`, `USERPROFILE` | value | The home `easSimulators()` finds the eas-cli login under when `EXPO_TOKEN` is unset: `USERPROFILE` on Windows, `HOME` elsewhere, the OS user's home when unset | +| `LT_USERNAME`, `LT_ACCESS_KEY` | value | The TestMu AI username and access key `testmu()` from `@e2e-dev/testmu` leases devices with; it copies them into the runner process's environment for the agent-device daemon it starts, and with either unset the device lease fails. See [TestMu AI](/integrations/testmu) | | `E2E_AGENT_DEVICE_POOL__` | value | Written by `@e2e-dev/mobile` in `prepare` and read by each worker to find its booted device. Not meant to be set by hand | ## What app processes inherit diff --git a/package.json b/package.json index 4d515218b..3fe17722d 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ ], "type": "module", "scripts": { - "build": "pnpm --filter e2e run build && pnpm --filter @e2e-dev/web run build && pnpm --filter @e2e-dev/mobile run build && pnpm --filter @e2e-dev/github run build && pnpm --filter @e2e-dev/kernel run build && pnpm --filter @e2e-dev/eas run build", + "build": "pnpm --filter e2e run build && pnpm --filter @e2e-dev/web run build && pnpm --filter @e2e-dev/mobile run build && pnpm --filter @e2e-dev/github run build && pnpm --filter @e2e-dev/kernel run build && pnpm --filter @e2e-dev/eas run build && pnpm --filter @e2e-dev/testmu run build", "check": "pnpm run lint && pnpm run check:dead-code && pnpm run typecheck && pnpm run docs:check-errors && pnpm run check:peer-ranges && pnpm run docs:check", "check:dead-code": "fallow dead-code", "check:peer-ranges": "node scripts/check-peer-ranges.ts", @@ -24,7 +24,7 @@ "canary": "node scripts/canary-changeset.ts && changeset version --snapshot canary && pnpm run build", "canary:publish": "changeset publish --tag canary --no-git-tag && node scripts/restore-peer-ranges.ts", "typecheck": "pnpm run build && pnpm -r run typecheck && tsc -p scripts", - "test": "pnpm run build && pnpm --filter e2e run test && pnpm --filter @e2e-dev/web run test && pnpm --filter @e2e-dev/mobile run test && pnpm --filter @e2e-dev/github run test && pnpm --filter @e2e-dev/kernel run test && pnpm --filter @e2e-dev/eas run test && pnpm run test:scripts", + "test": "pnpm run build && pnpm --filter e2e run test && pnpm --filter @e2e-dev/web run test && pnpm --filter @e2e-dev/mobile run test && pnpm --filter @e2e-dev/github run test && pnpm --filter @e2e-dev/kernel run test && pnpm --filter @e2e-dev/eas run test && pnpm --filter @e2e-dev/testmu run test && pnpm run test:scripts", "test:testbed": "pnpm run build && pnpm --filter @e2e-dev/testbed test", "test:web-benchmark": "pnpm run build && pnpm --filter @e2e-dev/web-benchmark test", "test:mobile-benchmark": "pnpm run build && pnpm --filter @e2e-dev/mobile-benchmark test", diff --git a/packages/mobile/src/nodes.ts b/packages/mobile/src/nodes.ts index ea9e6212c..dcb5f350d 100644 --- a/packages/mobile/src/nodes.ts +++ b/packages/mobile/src/nodes.ts @@ -266,9 +266,10 @@ const ANDROID_TITLE_IDS = [':id/collapsing_toolbar', ':id/action_bar', ':id/tool /** * Platform element type of one raw node as a kebab-case token: XCTest sends - * `NavigationBar` and `StaticText`, Android sends `android.widget.TextView`; - * both read as one vocabulary here, the Android package prefix dropped. - * `role` is the fallback some platforms send instead. + * `NavigationBar` and `StaticText` (`XCUIElementTypeNavigationBar` over + * WebDriver), Android sends `android.widget.TextView`; all read as one + * vocabulary here, the class and package prefixes dropped. `role` is the + * fallback some platforms send instead. */ function kindOf(raw: RawNode): string { return normalizeKind(raw.type ?? raw.role ?? ''); @@ -279,9 +280,9 @@ function isAndroidClass(type: string | undefined): boolean { return type !== undefined && type.includes('.'); } -/** One element-type spelling for `NavigationBar`, `navigation-bar`, and `android.widget.NavigationBar` alike. */ +/** One element-type spelling for `NavigationBar`, `XCUIElementTypeNavigationBar`, `navigation-bar`, and `android.widget.NavigationBar` alike. */ export function normalizeKind(type: string): string { - const simple = type.slice(type.lastIndexOf('.') + 1); + const simple = type.slice(type.lastIndexOf('.') + 1).replace(/^XCUIElementType(?=[A-Z])/, ''); return simple .replaceAll(/([a-z0-9])([A-Z])/g, '$1-$2') .replaceAll(/[\s_]+/g, '-') diff --git a/packages/mobile/tests/unit/nodes.test.ts b/packages/mobile/tests/unit/nodes.test.ts index 45089d0a6..24f5f76fb 100644 --- a/packages/mobile/tests/unit/nodes.test.ts +++ b/packages/mobile/tests/unit/nodes.test.ts @@ -214,6 +214,36 @@ describe('snapshot projection', () => { expect(screenTitle(projected)).toBe('Settings'); }); + it('reads the XCUIElementType class names a WebDriver session reports as the native XCTest types', () => { + const native: RawNode[] = [ + { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'Profiler', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { ref: 'e2', index: 1, parentIndex: 0, depth: 1, type: 'NavigationBar', identifier: 'Profiler' }, + { ref: 'e3', index: 2, parentIndex: 1, depth: 2, type: 'StaticText', label: 'Profiler' }, + { ref: 'e4', index: 3, parentIndex: 0, depth: 1, type: 'Button', label: 'CPU Load' }, + { ref: 'e5', index: 4, parentIndex: 0, depth: 1, type: 'TextView', label: 'Notes', value: 'idle' }, + { ref: 'e6', index: 5, parentIndex: 0, depth: 1, type: 'TabBar' }, + { ref: 'e7', index: 6, parentIndex: 5, depth: 2, type: 'Button', label: 'Home', selected: true }, + { ref: 'e8', index: 7, parentIndex: 5, depth: 2, type: 'Button', label: 'Settings' }, + ]; + const webDriver = native.map((node) => ({ ...node, type: `XCUIElementType${node.type}` })); + const kindsAndRoles = (nodes: readonly RawNode[]) => project(nodes).index.map((entry) => [entry.kind, entry.node.role]); + expect(kindsAndRoles(webDriver)).toEqual(kindsAndRoles(native)); + const projected = project(webDriver); + expect(projected.index.map((entry) => entry.kind)).toEqual([ + 'application', + 'navigation-bar', + 'static-text', + 'button', + 'text-view', + 'tab-bar', + 'button', + 'button', + ]); + expect(projected.index.map((entry) => entry.node.role)).toEqual(['application', 'navigation', 'text', 'button', 'textbox', 'tablist', 'tab', 'tab']); + expect(projected.viewport).toEqual({ width: 390, height: 844 }); + expect(screenTitle(projected)).toBe('Profiler'); + }); + it('maps iOS composite widgets onto the vocabulary roles a browser reports for them', () => { const projected = project([ { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'Shop' }, diff --git a/packages/testmu/LICENSE b/packages/testmu/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/packages/testmu/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/testmu/NOTICE b/packages/testmu/NOTICE new file mode 100644 index 000000000..a5f5c8473 --- /dev/null +++ b/packages/testmu/NOTICE @@ -0,0 +1,5 @@ +@e2e-dev/testmu +Copyright 2026 TesterArmy, Inc. + +This product includes software developed at TesterArmy, Inc. +(https://tester.army). diff --git a/packages/testmu/README.md b/packages/testmu/README.md new file mode 100644 index 000000000..09dea7f87 --- /dev/null +++ b/packages/testmu/README.md @@ -0,0 +1,66 @@ +# @e2e-dev/testmu + +[TestMu AI](https://www.lambdatest.com) (formerly LambdaTest) for [`e2e`](https://www.npmjs.com/package/e2e): +`mobile({ device: testmu({ device, osVersion, app }) })` runs a mobile target on +TestMu AI's hosted Android emulators, iOS simulators, and real devices. + +## Install + +```bash +npm install --save-dev @e2e-dev/testmu agent-device@ +``` + +The provider drives TestMu AI through agent-device's `testmu` cloud +provider, so the project needs an agent-device that includes it, and +`@e2e-dev/mobile` must use that same agent-device: override its pin (npm and +bun `overrides`, pnpm `overrides` in `pnpm-workspace.yaml`). No published +agent-device release includes the `testmu` provider yet; `` is the +first one that does. The peer range only excludes the releases known to lack +it. + +## Usage + +```ts title="e2e.config.ts" +import type { E2EConfig } from 'e2e'; +import { mobile } from '@e2e-dev/mobile'; +import { testmu } from '@e2e-dev/testmu'; + +export default { + targets: [ + { + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Galaxy S22 Ultra 5G', osVersion: '14', app: 'https://example.com/app.apk' }), + }), + app: { bundleId: 'com.example.app' }, + }, + ], + workers: 2, +} satisfies E2EConfig; +``` + +It authenticates with `LT_USERNAME` and `LT_ACCESS_KEY` from the run's +environment. Each worker slot leases one device from an agent-device daemon +the provider starts for the run; the worker's first command starts the +TestMu AI session, which installs `app`, and the lease is released when the +run ends, which ends the session. + +- `device` and `osVersion` must match TestMu AI's catalog exactly: `18.0` + for a virtual iOS device, `18` for a real one, `14` on Android. +- `app` is an `lt://` app id, an `https` URL, or a local path resolved + against the project root. Keep the target's `app.bundleId` and leave + `app.appPath` out. +- `deviceType: 'real'` picks a real device (default `'virtual'`). +- `project` (default `e2e`), `build` (default the run id), and `sessionName` + label the sessions on the dashboard. +- `stateDir` (default `.e2e/testmu`) holds each run's daemon. + +Sessions run over TestMu AI's Appium hub, so agent-device's device settings, +system alerts, recording, device logs, and port reverse are not available +there. TestMu AI records every session itself. + +Full documentation lives at [e2e.tester.army/docs/integrations/testmu](https://e2e.tester.army/docs/integrations/testmu). + +## License + +Apache-2.0 diff --git a/packages/testmu/package.json b/packages/testmu/package.json new file mode 100644 index 000000000..f4fc14f91 --- /dev/null +++ b/packages/testmu/package.json @@ -0,0 +1,73 @@ +{ + "name": "@e2e-dev/testmu", + "version": "0.0.0", + "description": "TestMu AI (formerly LambdaTest) for e2e: run mobile targets on hosted Android emulators, iOS simulators, and real devices", + "keywords": [ + "e2e", + "end-to-end", + "testing", + "mobile-testing", + "testmu", + "lambdatest", + "device-cloud", + "real-devices", + "ios-simulator", + "android-emulator", + "agent-device", + "ci" + ], + "license": "Apache-2.0", + "contributors": [ + "Oskar Kwaśniewski ", + "Szymon Rybczak " + ], + "repository": { + "type": "git", + "url": "git+https://github.com/tester-army/e2e.git", + "directory": "packages/testmu" + }, + "bugs": { + "url": "https://github.com/tester-army/e2e/issues" + }, + "homepage": "https://github.com/tester-army/e2e/tree/main/packages/testmu#readme", + "publishConfig": { + "registry": "https://registry.npmjs.org/", + "access": "public", + "tag": "latest" + }, + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "NOTICE" + ], + "scripts": { + "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsc --project tsconfig.build.json && node ../../scripts/stamp-dist.ts", + "prepublishOnly": "node ../../scripts/check-dist.ts", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest", + "test:unit": "vitest run tests/unit" + }, + "peerDependencies": { + "@e2e-dev/mobile": ">=0.9.0 <1", + "agent-device": ">0.21.18 <1", + "e2e": ">=0.15.0 <1" + }, + "devDependencies": { + "@e2e-dev/mobile": "workspace:*", + "@types/node": "26.6.2", + "agent-device": "0.21.18", + "e2e": "workspace:*", + "typescript": "7.0.2", + "vitest": "5.0.1" + }, + "engines": { + "node": ">=22.12.0" + } +} diff --git a/packages/testmu/src/credentials.ts b/packages/testmu/src/credentials.ts new file mode 100644 index 000000000..faa6b987e --- /dev/null +++ b/packages/testmu/src/credentials.ts @@ -0,0 +1,45 @@ +/** + * The TestMu AI credentials a run authenticates with: `LT_USERNAME` and + * `LT_ACCESS_KEY` from the run's environment. agent-device's `testmu` + * runtime reads them from the environment of the daemon that drives the + * sessions, not from a request. + */ + +const LT_USERNAME = 'LT_USERNAME'; +const LT_ACCESS_KEY = 'LT_ACCESS_KEY'; + +interface TestmuCredentials { + readonly username: string; + readonly accessKey: string; +} + +/** `LT_USERNAME` and `LT_ACCESS_KEY` from the run's environment; throws naming each one that is unset or blank. */ +export function testmuCredentials(env: Readonly>): TestmuCredentials { + const username = envValue(env, LT_USERNAME); + const accessKey = envValue(env, LT_ACCESS_KEY); + if (username === undefined || accessKey === undefined) { + const missing = [username === undefined ? LT_USERNAME : undefined, accessKey === undefined ? LT_ACCESS_KEY : undefined].filter((name) => name !== undefined); + throw new Error( + `${missing.join(' and ')} ${missing.length === 1 ? 'is' : 'are'} not set; set ${LT_USERNAME} and ${LT_ACCESS_KEY} to your TestMu AI username and access key in the environment \`e2e run\` starts in`, + ); + } + return { username, accessKey }; +} + +/** + * Puts the run's credentials in this process's environment, which is where + * the daemon gets them: agent-device's client starts its local daemon with + * `process.env` and takes no environment of its own, while a run's + * environment can differ from `process.env` (a host that passes `env`). + * Every worker of the run already starts with these values. + */ +export function shareWithDaemon(credentials: TestmuCredentials): void { + if (process.env[LT_USERNAME] !== credentials.username) process.env[LT_USERNAME] = credentials.username; + if (process.env[LT_ACCESS_KEY] !== credentials.accessKey) process.env[LT_ACCESS_KEY] = credentials.accessKey; +} + +/** A non-empty variable from the run's environment, trimmed, or `undefined`. */ +function envValue(env: Readonly>, name: string): string | undefined { + const value = env[name]?.trim(); + return value === undefined || value === '' ? undefined : value; +} diff --git a/packages/testmu/src/index.ts b/packages/testmu/src/index.ts new file mode 100644 index 000000000..65b060dfa --- /dev/null +++ b/packages/testmu/src/index.ts @@ -0,0 +1,8 @@ +/** + * `@e2e-dev/testmu` public surface: `testmu()`, a device provider that leases + * TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real + * devices for `@e2e-dev/mobile` through agent-device's `testmu` provider. + */ + +export { testmu } from './provider.ts'; +export type { TestmuOptions } from './provider.ts'; diff --git a/packages/testmu/src/provider.ts b/packages/testmu/src/provider.ts new file mode 100644 index 000000000..876d2a447 --- /dev/null +++ b/packages/testmu/src/provider.ts @@ -0,0 +1,233 @@ +/** + * TestMu AI's hosted Android emulators, iOS simulators, and real devices as a + * `DeviceProvider` for the mobile engine, through agent-device's `testmu` + * cloud provider: the provider allocates agent-device leases, and the daemon + * holding them runs each session over TestMu AI's Appium hub. + */ + +import { isAbsolute, resolve } from 'node:path'; +import type { DeviceLease, DeviceProvider, DeviceRequest } from '@e2e-dev/mobile'; +import { createAgentDeviceClient } from 'agent-device'; +import { ConfigurationError, rejectUnknownKeys } from 'e2e/engine'; +import { shareWithDaemon, testmuCredentials } from './credentials.ts'; + +/** agent-device's name for TestMu AI, as the lease provider and the tenant. */ +const PROVIDER = 'testmu'; + +/** Where each run's daemon keeps its state, under the project root. */ +const DEFAULT_STATE_DIR = '.e2e/testmu'; + +/** The dashboard project sessions are grouped under when `project` is absent. */ +const DEFAULT_PROJECT = 'e2e'; + +/** + * The inactivity window each lease asks for, agent-device's longest. Its + * 60-second default lapses while the worker's first command uploads the app + * and starts the session, which can take longer on an iOS simulator. + */ +const LEASE_TTL_MS = 10 * 60_000; + +/** How often the runner heartbeats each lease it holds, well inside `LEASE_TTL_MS`. */ +const HEARTBEAT_INTERVAL_MS = 2 * 60_000; + +const DEVICE_TYPES: ReadonlySet = new Set(['virtual', 'real']); + +/** Every option `testmu()` takes, kept equal to `TestmuOptions` by the compiler. */ +const OPTION_KEYS: readonly string[] = Object.keys({ + device: true, + osVersion: true, + app: true, + deviceType: true, + project: true, + build: true, + sessionName: true, + stateDir: true, +} satisfies Record); + +/** What `testmu()` takes: the device, its OS version, and the app TestMu AI installs on it. */ +export interface TestmuOptions { + /** Device name exactly as TestMu AI's catalog lists it: `Galaxy S22 Ultra 5G`, `iPhone 16`. */ + readonly device: string; + /** OS version exactly as the catalog lists it for that device: `14` on Android, `18.0` on a virtual iOS device, `18` on a real one. */ + readonly osVersion: string; + /** + * The build TestMu AI installs on every session: an `lt://` app id, an + * `https` URL, or a local path, resolved against the project root and + * uploaded when the session starts. + */ + readonly app: string; + /** `'virtual'` (default) for an emulator or simulator, `'real'` for a real device. */ + readonly deviceType?: 'virtual' | 'real' | undefined; + /** Dashboard project the sessions are grouped under. Defaults to `e2e`. */ + readonly project?: string | undefined; + /** Dashboard build the sessions are grouped under. Defaults to the run id. */ + readonly build?: string | undefined; + /** Name of every session on the dashboard; absent, TestMu AI names it. */ + readonly sessionName?: string | undefined; + /** Directory for the agent-device daemon each run starts, relative to the project root. Defaults to `.e2e/testmu`. */ + readonly stateDir?: string | undefined; +} + +type LeaseBackend = 'ios-instance' | 'android-instance'; + +/** The lease scope agent-device resolves a leased device by, as `leases.allocate` granted it. */ +interface LeaseScope { + readonly tenant: string; + readonly runId: string; + readonly leaseId: string; + readonly leaseBackend: LeaseBackend; + readonly leaseProvider: string; +} + +/** A lease's daemon and scope: what releasing it needs. */ +interface LeaseHandle { + readonly stateDir: string; + readonly scope: LeaseScope; +} + +/** + * TestMu AI devices for `mobile({ device: testmu({ device, osVersion, app }) })`: + * one hosted device per worker slot, leased when the run starts and released + * when it ends. Each lease comes from an agent-device daemon the provider + * starts for the run under `stateDir`; the worker's first command starts the + * TestMu AI session, which installs `app`, and releasing the lease ends it. + * It authenticates with `LT_USERNAME` and `LT_ACCESS_KEY` from the run's + * environment, and needs an agent-device with the `testmu` provider. + */ +export function testmu(options: TestmuOptions): DeviceProvider { + rejectUnknownKeys('testmu()', options, OPTION_KEYS); + for (const key of ['device', 'osVersion', 'app'] as const) { + const value: unknown = options[key]; + if (typeof value !== 'string' || value.trim() === '') { + throw new ConfigurationError('INVALID_CONFIG', `testmu: \`${key}\` is required, as a non-empty string`); + } + } + const deviceType = options.deviceType ?? 'virtual'; + if (!DEVICE_TYPES.has(deviceType)) { + throw new ConfigurationError('INVALID_CONFIG', `testmu: \`deviceType\` must be 'virtual' or 'real', not ${JSON.stringify(deviceType)}`); + } + const { device, osVersion, app, project, build, sessionName } = options; + /** One release per lease, shared by every caller: the engine's, and `acquire`'s own after a failure. */ + const releases = new Map>(); + /** Stops each held lease's heartbeat, by lease id. */ + const heartbeats = new Map void>(); + const release = (id: string, handle: LeaseHandle): Promise => { + heartbeats.get(id)?.(); + heartbeats.delete(id); + let pending = releases.get(id); + if (pending === undefined) { + pending = releaseLease(handle); + releases.set(id, pending); + // A failed release may be tried again. + pending.catch(() => releases.delete(id)); + } + return pending; + }; + return { + name: PROVIDER, + async acquire(request: DeviceRequest): Promise { + if (request.appPath !== undefined) throw new Error("TestMu AI installs the app from `app`; leave the target's `app.appPath` out"); + shareWithDaemon(testmuCredentials(request.env)); + if (request.signal.aborted) throw new Error('cancelled before a lease was allocated'); + const stateDir = resolve(request.projectRoot, options.stateDir ?? DEFAULT_STATE_DIR, request.runId); + const leaseBackend: LeaseBackend = request.platform === 'ios' ? 'ios-instance' : 'android-instance'; + const selectors = { + platform: request.platform, + target: 'mobile' as const, + device, + providerOsVersion: osVersion, + providerApp: appSource(app, request.projectRoot), + providerDeviceType: deviceType, + providerProject: project ?? DEFAULT_PROJECT, + providerBuild: build ?? request.runId, + ...(sessionName === undefined ? {} : { providerSessionName: sessionName }), + }; + // Not cancellable: the daemon may grant the lease after an interrupt, and only a lease this returns or releases is ever released. + const granted = await createAgentDeviceClient({ stateDir, session: `lease-${request.slot}` }).leases.allocate({ + tenant: PROVIDER, + runId: request.runId, + leaseBackend, + leaseProvider: PROVIDER, + ttlMs: LEASE_TTL_MS, + ...selectors, + }); + const scope: LeaseScope = { tenant: granted.tenantId, runId: granted.runId, leaseId: granted.leaseId, leaseBackend, leaseProvider: PROVIDER }; + heartbeats.set(scope.leaseId, keepAlive({ stateDir, scope }, request.log)); + try { + if (request.signal.aborted) throw new Error('cancelled'); + request.log(`lease ${scope.leaseId}: ${device}, ${request.platform} ${osVersion} (${deviceType}); the session starts on the first command`); + // The worker's client is created with these fields: the scope picks the lease, and the selectors start the session. + return { id: scope.leaseId, client: { stateDir, ...scope, ...selectors } }; + } catch (cause) { + // The engine releases only leases `acquire` returned. + const outcome = await release(scope.leaseId, { stateDir, scope }).then( + () => 'released it', + (releaseCause: unknown) => `releasing it failed (${messageOf(releaseCause)})`, + ); + throw new Error(`lease ${scope.leaseId} was not handed to the run: ${messageOf(cause)}; ${outcome}`, { cause }); + } + }, + async release(lease: DeviceLease): Promise { + const handle = leaseHandle(lease); + if (handle === undefined) throw new Error(`lease ${lease.id} carries no agent-device lease scope to release`); + await release(lease.id, handle); + }, + }; +} + +/** Releases a lease through the daemon that granted it, which ends its TestMu AI session. A lease the daemon no longer knows counts as released. */ +async function releaseLease({ stateDir, scope }: LeaseHandle): Promise { + await createAgentDeviceClient({ stateDir, session: 'release' }).leases.release(scope); +} + +/** + * Heartbeats a lease until the returned function is called. A command still + * running does not keep its lease alive, so without this a lease can lapse + * while the worker's first command is starting the session. A failed + * heartbeat is logged once and the next one tried; it never fails the run. + */ +function keepAlive({ stateDir, scope }: LeaseHandle, log: (line: string) => void): () => void { + const client = createAgentDeviceClient({ stateDir, session: 'heartbeat' }); + let warned = false; + const timer = setInterval(() => { + client.leases.heartbeat({ ...scope, ttlMs: LEASE_TTL_MS }).catch((cause: unknown) => { + if (warned) return; + warned = true; + try { + log(`lease ${scope.leaseId}: heartbeat failed (${messageOf(cause)}); agent-device ends the lease after ${LEASE_TTL_MS / 60_000} minutes without one`); + } catch { + // The run's log may be closed by now. + } + }); + }, HEARTBEAT_INTERVAL_MS); + timer.unref(); + return () => clearInterval(timer); +} + +/** `app` as the daemon reads it: an `lt://` id or URL as written, a local path resolved against the project root, never the daemon's working directory. */ +function appSource(app: string, projectRoot: string): string { + if (isAbsolute(app) || /^[a-z][a-z0-9+.-]*:\/\//i.test(app)) return app; + return resolve(projectRoot, app); +} + +/** The daemon and scope `acquire` put on a lease's `client`, when they are there. */ +function leaseHandle(lease: DeviceLease): LeaseHandle | undefined { + const client = lease.client as Record | undefined; + if (client === undefined) return undefined; + const { stateDir, tenant, runId, leaseId, leaseBackend, leaseProvider } = client; + if ( + typeof stateDir !== 'string' || + typeof tenant !== 'string' || + typeof runId !== 'string' || + typeof leaseId !== 'string' || + (leaseBackend !== 'ios-instance' && leaseBackend !== 'android-instance') || + typeof leaseProvider !== 'string' + ) { + return undefined; + } + return { stateDir, scope: { tenant, runId, leaseId, leaseBackend, leaseProvider } }; +} + +function messageOf(cause: unknown): string { + return cause instanceof Error ? cause.message : String(cause); +} diff --git a/packages/testmu/tests/unit/testmu.test.ts b/packages/testmu/tests/unit/testmu.test.ts new file mode 100644 index 000000000..de4c6fecf --- /dev/null +++ b/packages/testmu/tests/unit/testmu.test.ts @@ -0,0 +1,359 @@ +/** + * `testmu()` against a stubbed agent-device client: the options it refuses, + * the lease it allocates and hands the worker, the credentials it reads and + * shares with the daemon, the heartbeat that keeps it alive, and the release + * on every exit path. + */ + +import { join } from 'node:path'; +import type { DeviceLease, DeviceReleaseContext, DeviceRequest } from '@e2e-dev/mobile'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { testmu, type TestmuOptions } from '../../src/index.ts'; + +interface ClientCall { + readonly config: Record; + readonly operation: 'allocate' | 'heartbeat' | 'release'; + readonly options: Record; +} + +const daemon = { + calls: [] as ClientCall[], + /** Runs inside `allocate`, before it answers. */ + onAllocate: undefined as (() => void) | undefined, + allocateError: undefined as Error | undefined, + /** Errors `release` answers with, in order, before it succeeds. */ + releaseErrors: [] as Error[], + /** Errors `heartbeat` answers with, in order, before it succeeds. */ + heartbeatErrors: [] as Error[], +}; + +vi.mock('agent-device', () => ({ + createAgentDeviceClient: (config: Record) => ({ + leases: { + allocate: async (options: Record) => { + daemon.calls.push({ config, operation: 'allocate', options }); + daemon.onAllocate?.(); + if (daemon.allocateError !== undefined) throw daemon.allocateError; + return { leaseId: `lease-${daemon.calls.length}`, tenantId: options['tenant'], runId: options['runId'], backend: options['leaseBackend'], leaseProvider: options['leaseProvider'] }; + }, + heartbeat: async (options: Record) => { + daemon.calls.push({ config, operation: 'heartbeat', options }); + const error = daemon.heartbeatErrors.shift(); + if (error !== undefined) throw error; + return { leaseId: options['leaseId'], tenantId: options['tenant'], runId: options['runId'], backend: options['leaseBackend'] }; + }, + release: async (options: Record) => { + daemon.calls.push({ config, operation: 'release', options }); + const error = daemon.releaseErrors.shift(); + if (error !== undefined) throw error; + return { released: true }; + }, + }, + }), +})); + +const ROOT = join('/', 'work', 'shop'); +const env = { LT_USERNAME: 'ada', LT_ACCESS_KEY: 'lt-key' }; +const options: TestmuOptions = { device: 'Galaxy S22 Ultra 5G', osVersion: '14', app: 'https://example.com/app.apk' }; +const saved = { LT_USERNAME: process.env['LT_USERNAME'], LT_ACCESS_KEY: process.env['LT_ACCESS_KEY'] }; + +beforeEach(() => { + Object.assign(daemon, { calls: [], onAllocate: undefined, allocateError: undefined, releaseErrors: [], heartbeatErrors: [] }); +}); + +afterEach(() => { + vi.useRealTimers(); + for (const [name, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +}); + +function request(overrides: Partial = {}): DeviceRequest & { lines: string[] } { + const lines: string[] = []; + return { + platform: 'android', + runId: 'run-1', + targetName: 'android', + slot: 0, + slots: 2, + app: 'com.example.app', + agentDeviceVersion: '0.21.18', + projectRoot: ROOT, + env, + signal: new AbortController().signal, + log: (line) => lines.push(line), + lines, + ...overrides, + }; +} + +const context: DeviceReleaseContext = { runId: 'run-1', targetName: 'android', env, signal: new AbortController().signal, log: () => undefined }; + +const operations = () => daemon.calls.map((call) => call.operation); + +const MINUTE = 60_000; + +describe('testmu()', () => { + it('is a device provider named testmu that leaves recording to agent-device', () => { + const provider = testmu(options); + expect(provider.name).toBe('testmu'); + expect(provider.record).toBeUndefined(); + }); + + it('allocates an Android lease from a daemon under the project root, with the device selectors and dashboard labels', async () => { + await testmu(options).acquire(request({ slot: 1 })); + expect(daemon.calls).toEqual([ + { + config: { stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), session: 'lease-1' }, + operation: 'allocate', + options: { + tenant: 'testmu', + runId: 'run-1', + leaseBackend: 'android-instance', + leaseProvider: 'testmu', + platform: 'android', + target: 'mobile', + device: 'Galaxy S22 Ultra 5G', + providerOsVersion: '14', + providerApp: 'https://example.com/app.apk', + providerDeviceType: 'virtual', + providerProject: 'e2e', + providerBuild: 'run-1', + ttlMs: 10 * MINUTE, + }, + }, + ]); + }); + + it('allocates an iOS lease on a real device with its own project, build, session name, and state directory', async () => { + await testmu({ device: 'iPhone 16', osVersion: '18', app: 'lt://APP123', deviceType: 'real', project: 'shop', build: 'nightly', sessionName: 'checkout', stateDir: 'tmp/devices' }).acquire( + request({ platform: 'ios' }), + ); + expect(daemon.calls[0]?.config['stateDir']).toBe(join(ROOT, 'tmp', 'devices', 'run-1')); + expect(daemon.calls[0]?.options).toMatchObject({ + leaseBackend: 'ios-instance', + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18', + providerApp: 'lt://APP123', + providerDeviceType: 'real', + providerProject: 'shop', + providerBuild: 'nightly', + providerSessionName: 'checkout', + }); + }); + + it('resolves a local build against the project root, never the working directory', async () => { + await testmu({ ...options, app: 'build/app.apk' }).acquire(request()); + expect(daemon.calls[0]?.options['providerApp']).toBe(join(ROOT, 'build', 'app.apk')); + await testmu({ ...options, app: join('/', 'builds', 'app.apk') }).acquire(request()); + expect(daemon.calls[1]?.options['providerApp']).toBe(join('/', 'builds', 'app.apk')); + }); + + it('hands the worker the lease scope and the selectors as JSON client configuration', async () => { + const req = request(); + const lease = await testmu(options).acquire(req); + expect(lease).toEqual({ + id: 'lease-1', + client: { + stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), + tenant: 'testmu', + runId: 'run-1', + leaseId: 'lease-1', + leaseBackend: 'android-instance', + leaseProvider: 'testmu', + platform: 'android', + target: 'mobile', + device: 'Galaxy S22 Ultra 5G', + providerOsVersion: '14', + providerApp: 'https://example.com/app.apk', + providerDeviceType: 'virtual', + providerProject: 'e2e', + providerBuild: 'run-1', + }, + }); + const json = JSON.stringify(lease); + expect(JSON.parse(json)).toEqual(lease); + expect(Buffer.byteLength(json)).toBeLessThan(1024); + expect(json).not.toContain('lt-key'); + expect(req.lines).toEqual(['lease lease-1: Galaxy S22 Ultra 5G, android 14 (virtual); the session starts on the first command']); + }); + + it('shares the run\'s credentials with the daemon it starts', async () => { + delete process.env['LT_USERNAME']; + process.env['LT_ACCESS_KEY'] = 'stale'; + daemon.onAllocate = () => expect([process.env['LT_USERNAME'], process.env['LT_ACCESS_KEY']]).toEqual(['ada', 'lt-key']); + await testmu(options).acquire(request({ env: { LT_USERNAME: ' ada ', LT_ACCESS_KEY: 'lt-key' } })); + expect(operations()).toEqual(['allocate']); + }); + + it.each([ + [{}, 'LT_USERNAME and LT_ACCESS_KEY are not set'], + [{ LT_USERNAME: 'ada' }, 'LT_ACCESS_KEY is not set'], + [{ LT_USERNAME: ' ', LT_ACCESS_KEY: 'lt-key' }, 'LT_USERNAME is not set'], + ])('fails before any daemon starts without credentials in the run\'s environment (%j)', async (runEnv, message) => { + process.env['LT_USERNAME'] = 'from-the-runner'; + process.env['LT_ACCESS_KEY'] = 'from-the-runner'; + await expect(testmu(options).acquire(request({ env: runEnv }))).rejects.toThrow( + `${message}; set LT_USERNAME and LT_ACCESS_KEY to your TestMu AI username and access key in the environment \`e2e run\` starts in`, + ); + expect(daemon.calls).toEqual([]); + }); + + it("refuses the target's app.appPath, since TestMu AI installs `app`", async () => { + await expect(testmu(options).acquire(request({ appPath: join(ROOT, 'build', 'app.apk') }))).rejects.toThrow( + "TestMu AI installs the app from `app`; leave the target's `app.appPath` out", + ); + expect(daemon.calls).toEqual([]); + }); + + it('allocates nothing once the run is interrupted', async () => { + const controller = new AbortController(); + controller.abort(); + await expect(testmu(options).acquire(request({ signal: controller.signal }))).rejects.toThrow('cancelled before a lease was allocated'); + expect(daemon.calls).toEqual([]); + }); + + it('releases a lease granted after an interrupt instead of handing it over', async () => { + const controller = new AbortController(); + daemon.onAllocate = () => controller.abort(); + const req = request({ signal: controller.signal }); + await expect(testmu(options).acquire(req)).rejects.toThrow('lease lease-1 was not handed to the run: cancelled; released it'); + expect(daemon.calls[1]).toEqual({ + config: { stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), session: 'release' }, + operation: 'release', + options: { tenant: 'testmu', runId: 'run-1', leaseId: 'lease-1', leaseBackend: 'android-instance', leaseProvider: 'testmu' }, + }); + expect(req.lines).toEqual([]); + }); + + it('releases a granted lease when handing it over fails, and says when that release failed too', async () => { + daemon.releaseErrors = [new Error('daemon gone')]; + const req = request({ + log: () => { + throw new Error('reporter closed'); + }, + }); + await expect(testmu(options).acquire(req)).rejects.toThrow('lease lease-1 was not handed to the run: reporter closed; releasing it failed (daemon gone)'); + expect(operations()).toEqual(['allocate', 'release']); + }); + + it('passes an allocation failure through with nothing to release', async () => { + daemon.allocateError = new Error('unknown lease provider testmu'); + await expect(testmu(options).acquire(request())).rejects.toThrow('unknown lease provider testmu'); + expect(operations()).toEqual(['allocate']); + }); + + it('releases a lease through the daemon that granted it, once however often it is asked', async () => { + const provider = testmu(options); + const lease = await provider.acquire(request()); + await Promise.all([provider.release(lease, context), provider.release(lease, context)]); + await provider.release(lease, context); + expect(daemon.calls.slice(1)).toEqual([ + { + config: { stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), session: 'release' }, + operation: 'release', + options: { tenant: 'testmu', runId: 'run-1', leaseId: 'lease-1', leaseBackend: 'android-instance', leaseProvider: 'testmu' }, + }, + ]); + }); + + it('releases from the lease alone, after a round trip through JSON', async () => { + const lease = JSON.parse(JSON.stringify(await testmu(options).acquire(request({ platform: 'ios' })))) as DeviceLease; + await testmu(options).release(lease, context); + expect(daemon.calls[1]?.options).toEqual({ tenant: 'testmu', runId: 'run-1', leaseId: 'lease-1', leaseBackend: 'ios-instance', leaseProvider: 'testmu' }); + }); + + it('tries a release again after one failed', async () => { + daemon.releaseErrors = [new Error('daemon busy')]; + const provider = testmu(options); + const lease = await provider.acquire(request()); + await expect(provider.release(lease, context)).rejects.toThrow('daemon busy'); + await provider.release(lease, context); + expect(operations()).toEqual(['allocate', 'release', 'release']); + }); + + it('refuses to release a lease without an agent-device scope', async () => { + await expect(testmu(options).release({ id: 'other', client: { stateDir: '/tmp/x' } }, context)).rejects.toThrow('lease other carries no agent-device lease scope to release'); + expect(daemon.calls).toEqual([]); + }); + + it('heartbeats each lease it holds every two minutes, asking for the longest lease, until it is released', async () => { + vi.useFakeTimers(); + const provider = testmu(options); + const lease = await provider.acquire(request()); + await vi.advanceTimersByTimeAsync(2 * MINUTE - 1); + expect(operations()).toEqual(['allocate']); + await vi.advanceTimersByTimeAsync(1); + expect(daemon.calls[1]).toEqual({ + config: { stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), session: 'heartbeat' }, + operation: 'heartbeat', + options: { tenant: 'testmu', runId: 'run-1', leaseId: 'lease-1', leaseBackend: 'android-instance', leaseProvider: 'testmu', ttlMs: 10 * MINUTE }, + }); + await vi.advanceTimersByTimeAsync(2 * MINUTE); + expect(operations()).toEqual(['allocate', 'heartbeat', 'heartbeat']); + await provider.release(lease, context); + await vi.advanceTimersByTimeAsync(10 * MINUTE); + expect(operations()).toEqual(['allocate', 'heartbeat', 'heartbeat', 'release']); + }); + + it('stops heartbeating a lease it released because handing it over failed', async () => { + vi.useFakeTimers(); + const controller = new AbortController(); + daemon.onAllocate = () => controller.abort(); + await expect(testmu(options).acquire(request({ signal: controller.signal }))).rejects.toThrow('was not handed to the run'); + await vi.advanceTimersByTimeAsync(10 * MINUTE); + expect(operations()).toEqual(['allocate', 'release']); + }); + + it('heartbeats nothing when the allocation failed', async () => { + vi.useFakeTimers(); + daemon.allocateError = new Error('no capacity'); + await expect(testmu(options).acquire(request())).rejects.toThrow('no capacity'); + await vi.advanceTimersByTimeAsync(10 * MINUTE); + expect(operations()).toEqual(['allocate']); + }); + + it('logs the first failed heartbeat and keeps beating, without failing the run', async () => { + vi.useFakeTimers(); + daemon.heartbeatErrors = [new Error('Lease is not active'), new Error('daemon gone')]; + const req = request(); + await testmu(options).acquire(req); + await vi.advanceTimersByTimeAsync(6 * MINUTE); + expect(operations()).toEqual(['allocate', 'heartbeat', 'heartbeat', 'heartbeat']); + expect(req.lines.slice(1)).toEqual(['lease lease-1: heartbeat failed (Lease is not active); agent-device ends the lease after 10 minutes without one']); + }); + + it('survives a failed heartbeat when the run\'s log is closed', async () => { + vi.useFakeTimers(); + daemon.heartbeatErrors = [new Error('daemon gone')]; + let closed = false; + await testmu(options).acquire( + request({ + log: () => { + if (closed) throw new Error('reporter closed'); + }, + }), + ); + closed = true; + await vi.advanceTimersByTimeAsync(4 * MINUTE); + expect(operations()).toEqual(['allocate', 'heartbeat', 'heartbeat']); + }); + + it('rejects an option it does not take with INVALID_CONFIG, naming the nearest one', () => { + expect(() => testmu({ ...options, osVerison: '14' } as unknown as TestmuOptions)).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG', message: expect.stringContaining('osVersion') })); + }); + + it.each(['device', 'osVersion', 'app'] as const)('requires `%s` with INVALID_CONFIG', (key) => { + expect(() => testmu({ ...options, [key]: ' ' })).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG', message: `testmu: \`${key}\` is required, as a non-empty string` })); + const { [key]: _, ...rest } = options; + expect(() => testmu(rest as TestmuOptions)).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG' })); + }); + + it('refuses a device type other than virtual or real', () => { + expect(() => testmu({ ...options, deviceType: 'emulator' as 'virtual' })).toThrow( + expect.objectContaining({ code: 'INVALID_CONFIG', message: 'testmu: `deviceType` must be \'virtual\' or \'real\', not "emulator"' }), + ); + }); +}); diff --git a/packages/testmu/tsconfig.build.json b/packages/testmu/tsconfig.build.json new file mode 100644 index 000000000..65d5a8150 --- /dev/null +++ b/packages/testmu/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true + }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/testmu/tsconfig.json b/packages/testmu/tsconfig.json new file mode 100644 index 000000000..635a4f631 --- /dev/null +++ b/packages/testmu/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/testmu/vitest.config.ts b/packages/testmu/vitest.config.ts new file mode 100644 index 000000000..c7516095d --- /dev/null +++ b/packages/testmu/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config'; + +/** Pure unit tests: agent-device's client is a stub, no daemon and no network. */ +export default defineConfig({ + test: { + include: ['tests/unit/**/*.test.ts'], + testTimeout: 30_000, + pool: 'forks', + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7b810cfb4..e80250465 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -290,6 +290,9 @@ importers: '@e2e-dev/mobile': specifier: workspace:* version: link:../packages/mobile + '@e2e-dev/testmu': + specifier: workspace:* + version: link:../packages/testmu '@e2e-dev/web': specifier: workspace:* version: link:../packages/web @@ -473,6 +476,27 @@ importers: specifier: 5.0.1 version: 5.0.1(@types/node@26.6.2)(vite@8.1.5(@types/node@26.6.2)(esbuild@0.28.2)(jiti@1.21.7)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0)) + packages/testmu: + devDependencies: + '@e2e-dev/mobile': + specifier: workspace:* + version: link:../mobile + '@types/node': + specifier: 26.6.2 + version: 26.6.2 + agent-device: + specifier: 0.21.18 + version: 0.21.18(ai@7.0.107(zod@4.6.1)) + e2e: + specifier: workspace:* + version: link:../e2e + typescript: + specifier: 7.0.2 + version: 7.0.2 + vitest: + specifier: 5.0.1 + version: 5.0.1(@types/node@26.6.2)(vite@8.1.5(@types/node@26.6.2)(esbuild@0.28.2)(jiti@1.21.7)(terser@5.51.2)(tsx@4.23.14)(yaml@2.9.0)) + packages/web: devDependencies: '@types/node': diff --git a/scripts/check-docs-examples.ts b/scripts/check-docs-examples.ts index a19fd5832..aff292df2 100644 --- a/scripts/check-docs-examples.ts +++ b/scripts/check-docs-examples.ts @@ -27,6 +27,8 @@ const EXAMPLES: Record = { 'docs/examples/quickstart/e2e.command.config.ts': 'docs/starting-your-app.mdx', 'docs/examples/mobile/device-provider.ts': 'docs/mobile.mdx', 'docs/examples/mobile/device-cloud-provider.ts': 'docs/mobile.mdx', + 'docs/examples/mobile/testmu.config.ts': 'docs/integrations/testmu.mdx', + 'docs/examples/mobile/testmu.e2e.ts': 'docs/integrations/testmu.mdx', 'docs/examples/web/browser-provider.ts': 'docs/browser.mdx', 'docs/examples/skill/e2e.config.ts': 'skills/e2e/SKILL.md', 'docs/examples/skill/e2e.setup.config.ts': 'skills/e2e/references/setup.md', diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 7c42c1e01..4a4fffb52 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -299,6 +299,12 @@ export default { the app (omit `app.appPath`). A run must fit one session: `maxDurationMinutes`, absent, is the account's cap (40 on a standard plan). `videoTouches: false` on the engine for video there. + `testmu({ device, osVersion, app })` from `@e2e-dev/testmu` leases TestMu + AI (formerly LambdaTest) emulators, simulators, or real devices + (`deviceType: 'real'`); `device` and `osVersion` match its catalog + exactly, TestMu AI installs `app` (omit `app.appPath`), it reads + `LT_USERNAME` and `LT_ACCESS_KEY`, and needs an agent-device with the + `testmu` provider shared with `@e2e-dev/mobile` (override its pin). - Only a control that appeared or moved with the previous action waits out `transition` (default 500 ms); agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it.