From 125aa03612de12459c68a0d41ceafc1a38ce71a3 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:55:37 +0530 Subject: [PATCH 1/4] feat(testmu): add TestMu AI device provider package Add @e2e-dev/testmu, a DeviceProvider for the mobile engine that runs targets on TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real devices through agent-device's testmu cloud provider: mobile({ device: testmu({ device, osVersion, app }) }) Each worker slot allocates an agent-device lease from a daemon the provider starts for the run under stateDir (default .e2e/testmu). The lease's client configuration carries the lease scope and the device selectors, so the worker's first command starts the TestMu AI session and installs the app; releasing the lease ends it. - Credentials come from LT_USERNAME and LT_ACCESS_KEY in the run's environment; a missing one fails the lease before anything starts. agent-device's client spawns its daemon with process.env and takes no environment of its own, so the provider copies the run's values there before the first allocation. - A lease granted after an interrupt, or one that cannot be handed to the run, is released before acquire throws. Release is idempotent and works from the lease alone, after a round trip through JSON. - A relative local app path resolves against the project root, not the daemon's working directory. The target's app.appPath is refused, since TestMu AI installs the app itself. - Unknown options, missing device/osVersion/app, and a deviceType other than virtual or real fail with INVALID_CONFIG. agent-device is a peer (>0.21.18 <1): the provider and the engine's workers must share one agent-device, and no published release includes the testmu provider yet, so the range excludes only the releases known to lack it. No record hook: TestMu AI records whole sessions, not attempts, and its video is only available once the session ends. Co-Authored-By: Claude Opus 5.5 --- .changeset/testmu-devices.md | 5 + AGENTS.md | 6 + CONTRIBUTING.md | 2 +- README.md | 1 + package.json | 4 +- packages/testmu/LICENSE | 202 +++++++++++++++ packages/testmu/NOTICE | 5 + packages/testmu/README.md | 66 +++++ packages/testmu/package.json | 73 ++++++ packages/testmu/src/credentials.ts | 45 ++++ packages/testmu/src/index.ts | 8 + packages/testmu/src/provider.ts | 193 +++++++++++++++ packages/testmu/tests/unit/testmu.test.ts | 284 ++++++++++++++++++++++ packages/testmu/tsconfig.build.json | 11 + packages/testmu/tsconfig.json | 9 + packages/testmu/vitest.config.ts | 10 + pnpm-lock.yaml | 24 ++ 17 files changed, 945 insertions(+), 3 deletions(-) create mode 100644 .changeset/testmu-devices.md create mode 100644 packages/testmu/LICENSE create mode 100644 packages/testmu/NOTICE create mode 100644 packages/testmu/README.md create mode 100644 packages/testmu/package.json create mode 100644 packages/testmu/src/credentials.ts create mode 100644 packages/testmu/src/index.ts create mode 100644 packages/testmu/src/provider.ts create mode 100644 packages/testmu/tests/unit/testmu.test.ts create mode 100644 packages/testmu/tsconfig.build.json create mode 100644 packages/testmu/tsconfig.json create mode 100644 packages/testmu/vitest.config.ts diff --git a/.changeset/testmu-devices.md b/.changeset/testmu-devices.md new file mode 100644 index 000000000..c19d878ec --- /dev/null +++ b/.changeset/testmu-devices.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/testmu": minor +--- + +`@e2e-dev/testmu`: `mobile({ device: testmu({ device, osVersion, app }) })` runs a mobile target on TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real devices (`deviceType: 'real'`), one per worker slot, through agent-device's `testmu` cloud provider. Each slot leases a device from an agent-device daemon the provider starts for the run under `stateDir` (default `.e2e/testmu`); the worker's first command starts the TestMu AI session, which installs `app` (an `lt://` id, an `https` URL, or a local path), and releasing the lease ends it. It reads `LT_USERNAME` and `LT_ACCESS_KEY` from the run's environment and fails the lease before starting anything when either is missing, groups sessions on the dashboard under `project` (default `e2e`) and `build` (default the run id), releases a lease it allocated when the run was interrupted, and releases each lease once. It needs an agent-device that includes the `testmu` provider, which no published release does yet, with `@e2e-dev/mobile`'s agent-device overridden to the same version; `agent-device` and `@e2e-dev/mobile` are its peers. diff --git a/AGENTS.md b/AGENTS.md index 6340a3daf..c12d1fee0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,12 @@ suites that consume the built packages the way a user would. hosted iOS simulators and Android emulators for the mobile engine (`DeviceProvider`). Expo publishes no SDK for the sessions API, so it calls Expo's GraphQL API with `fetch`, and `@e2e-dev/mobile` is its only peer. +- `packages/testmu` - the published `@e2e-dev/testmu` package: TestMu AI + (formerly LambdaTest) hosted Android emulators, iOS simulators, and real + devices for the mobile engine (`DeviceProvider`). agent-device's `testmu` + cloud provider is the vendor client: the package allocates agent-device + leases from a daemon it starts per run, so `agent-device` (which must be + the same copy `@e2e-dev/mobile` uses) and `@e2e-dev/mobile` are its peers. - `apps/testbed` (`@e2e-dev/testbed`, private) — dogfood project that consumes the **built** packages like a real user would: the playground app where every runner feature (sessions, routes, downloads, frames, uploads, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cc2969edf..93827362c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -254,5 +254,5 @@ build against, and what a third-party engine builds against too. A change to that contract bumps all three packages together in one release, with a changeset for each, so an engine and a runner from the same release always match. Engines declare a peer range on `e2e` that points one way only -(engine to runner, `>=x <1`), and each integration (`@e2e-dev/kernel`, `@e2e-dev/eas`) does +(engine to runner, `>=x <1`), and each integration (`@e2e-dev/kernel`, `@e2e-dev/eas`, `@e2e-dev/testmu`) does the same on the engines it plugs into; do not make it mutual or narrow it. diff --git a/README.md b/README.md index 9ebbc17c1..0bcb1f5a2 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ config and an example test. The | [`@e2e-dev/github`](https://www.npmjs.com/package/@e2e-dev/github) | Reporter that posts results as a pull request comment. | | [`@e2e-dev/kernel`](https://www.npmjs.com/package/@e2e-dev/kernel) | Kernel hosted browsers for the web engine. | | [`@e2e-dev/eas`](https://www.npmjs.com/package/@e2e-dev/eas) | EAS Simulators hosted iOS simulators and Android emulators for the mobile engine. | +| [`@e2e-dev/testmu`](https://www.npmjs.com/package/@e2e-dev/testmu) | TestMu AI (formerly LambdaTest) hosted Android emulators, iOS simulators, and real devices for the mobile engine. | ## Documentation diff --git a/package.json b/package.json index 4d515218b..3fe17722d 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ ], "type": "module", "scripts": { - "build": "pnpm --filter e2e run build && pnpm --filter @e2e-dev/web run build && pnpm --filter @e2e-dev/mobile run build && pnpm --filter @e2e-dev/github run build && pnpm --filter @e2e-dev/kernel run build && pnpm --filter @e2e-dev/eas run build", + "build": "pnpm --filter e2e run build && pnpm --filter @e2e-dev/web run build && pnpm --filter @e2e-dev/mobile run build && pnpm --filter @e2e-dev/github run build && pnpm --filter @e2e-dev/kernel run build && pnpm --filter @e2e-dev/eas run build && pnpm --filter @e2e-dev/testmu run build", "check": "pnpm run lint && pnpm run check:dead-code && pnpm run typecheck && pnpm run docs:check-errors && pnpm run check:peer-ranges && pnpm run docs:check", "check:dead-code": "fallow dead-code", "check:peer-ranges": "node scripts/check-peer-ranges.ts", @@ -24,7 +24,7 @@ "canary": "node scripts/canary-changeset.ts && changeset version --snapshot canary && pnpm run build", "canary:publish": "changeset publish --tag canary --no-git-tag && node scripts/restore-peer-ranges.ts", "typecheck": "pnpm run build && pnpm -r run typecheck && tsc -p scripts", - "test": "pnpm run build && pnpm --filter e2e run test && pnpm --filter @e2e-dev/web run test && pnpm --filter @e2e-dev/mobile run test && pnpm --filter @e2e-dev/github run test && pnpm --filter @e2e-dev/kernel run test && pnpm --filter @e2e-dev/eas run test && pnpm run test:scripts", + "test": "pnpm run build && pnpm --filter e2e run test && pnpm --filter @e2e-dev/web run test && pnpm --filter @e2e-dev/mobile run test && pnpm --filter @e2e-dev/github run test && pnpm --filter @e2e-dev/kernel run test && pnpm --filter @e2e-dev/eas run test && pnpm --filter @e2e-dev/testmu run test && pnpm run test:scripts", "test:testbed": "pnpm run build && pnpm --filter @e2e-dev/testbed test", "test:web-benchmark": "pnpm run build && pnpm --filter @e2e-dev/web-benchmark test", "test:mobile-benchmark": "pnpm run build && pnpm --filter @e2e-dev/mobile-benchmark test", diff --git a/packages/testmu/LICENSE b/packages/testmu/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/packages/testmu/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/testmu/NOTICE b/packages/testmu/NOTICE new file mode 100644 index 000000000..a5f5c8473 --- /dev/null +++ b/packages/testmu/NOTICE @@ -0,0 +1,5 @@ +@e2e-dev/testmu +Copyright 2026 TesterArmy, Inc. + +This product includes software developed at TesterArmy, Inc. +(https://tester.army). diff --git a/packages/testmu/README.md b/packages/testmu/README.md new file mode 100644 index 000000000..09dea7f87 --- /dev/null +++ b/packages/testmu/README.md @@ -0,0 +1,66 @@ +# @e2e-dev/testmu + +[TestMu AI](https://www.lambdatest.com) (formerly LambdaTest) for [`e2e`](https://www.npmjs.com/package/e2e): +`mobile({ device: testmu({ device, osVersion, app }) })` runs a mobile target on +TestMu AI's hosted Android emulators, iOS simulators, and real devices. + +## Install + +```bash +npm install --save-dev @e2e-dev/testmu agent-device@ +``` + +The provider drives TestMu AI through agent-device's `testmu` cloud +provider, so the project needs an agent-device that includes it, and +`@e2e-dev/mobile` must use that same agent-device: override its pin (npm and +bun `overrides`, pnpm `overrides` in `pnpm-workspace.yaml`). No published +agent-device release includes the `testmu` provider yet; `` is the +first one that does. The peer range only excludes the releases known to lack +it. + +## Usage + +```ts title="e2e.config.ts" +import type { E2EConfig } from 'e2e'; +import { mobile } from '@e2e-dev/mobile'; +import { testmu } from '@e2e-dev/testmu'; + +export default { + targets: [ + { + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Galaxy S22 Ultra 5G', osVersion: '14', app: 'https://example.com/app.apk' }), + }), + app: { bundleId: 'com.example.app' }, + }, + ], + workers: 2, +} satisfies E2EConfig; +``` + +It authenticates with `LT_USERNAME` and `LT_ACCESS_KEY` from the run's +environment. Each worker slot leases one device from an agent-device daemon +the provider starts for the run; the worker's first command starts the +TestMu AI session, which installs `app`, and the lease is released when the +run ends, which ends the session. + +- `device` and `osVersion` must match TestMu AI's catalog exactly: `18.0` + for a virtual iOS device, `18` for a real one, `14` on Android. +- `app` is an `lt://` app id, an `https` URL, or a local path resolved + against the project root. Keep the target's `app.bundleId` and leave + `app.appPath` out. +- `deviceType: 'real'` picks a real device (default `'virtual'`). +- `project` (default `e2e`), `build` (default the run id), and `sessionName` + label the sessions on the dashboard. +- `stateDir` (default `.e2e/testmu`) holds each run's daemon. + +Sessions run over TestMu AI's Appium hub, so agent-device's device settings, +system alerts, recording, device logs, and port reverse are not available +there. TestMu AI records every session itself. + +Full documentation lives at [e2e.tester.army/docs/integrations/testmu](https://e2e.tester.army/docs/integrations/testmu). + +## License + +Apache-2.0 diff --git a/packages/testmu/package.json b/packages/testmu/package.json new file mode 100644 index 000000000..f4fc14f91 --- /dev/null +++ b/packages/testmu/package.json @@ -0,0 +1,73 @@ +{ + "name": "@e2e-dev/testmu", + "version": "0.0.0", + "description": "TestMu AI (formerly LambdaTest) for e2e: run mobile targets on hosted Android emulators, iOS simulators, and real devices", + "keywords": [ + "e2e", + "end-to-end", + "testing", + "mobile-testing", + "testmu", + "lambdatest", + "device-cloud", + "real-devices", + "ios-simulator", + "android-emulator", + "agent-device", + "ci" + ], + "license": "Apache-2.0", + "contributors": [ + "Oskar Kwaśniewski ", + "Szymon Rybczak " + ], + "repository": { + "type": "git", + "url": "git+https://github.com/tester-army/e2e.git", + "directory": "packages/testmu" + }, + "bugs": { + "url": "https://github.com/tester-army/e2e/issues" + }, + "homepage": "https://github.com/tester-army/e2e/tree/main/packages/testmu#readme", + "publishConfig": { + "registry": "https://registry.npmjs.org/", + "access": "public", + "tag": "latest" + }, + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "NOTICE" + ], + "scripts": { + "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsc --project tsconfig.build.json && node ../../scripts/stamp-dist.ts", + "prepublishOnly": "node ../../scripts/check-dist.ts", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest", + "test:unit": "vitest run tests/unit" + }, + "peerDependencies": { + "@e2e-dev/mobile": ">=0.9.0 <1", + "agent-device": ">0.21.18 <1", + "e2e": ">=0.15.0 <1" + }, + "devDependencies": { + "@e2e-dev/mobile": "workspace:*", + "@types/node": "26.6.2", + "agent-device": "0.21.18", + "e2e": "workspace:*", + "typescript": "7.0.2", + "vitest": "5.0.1" + }, + "engines": { + "node": ">=22.12.0" + } +} diff --git a/packages/testmu/src/credentials.ts b/packages/testmu/src/credentials.ts new file mode 100644 index 000000000..faa6b987e --- /dev/null +++ b/packages/testmu/src/credentials.ts @@ -0,0 +1,45 @@ +/** + * The TestMu AI credentials a run authenticates with: `LT_USERNAME` and + * `LT_ACCESS_KEY` from the run's environment. agent-device's `testmu` + * runtime reads them from the environment of the daemon that drives the + * sessions, not from a request. + */ + +const LT_USERNAME = 'LT_USERNAME'; +const LT_ACCESS_KEY = 'LT_ACCESS_KEY'; + +interface TestmuCredentials { + readonly username: string; + readonly accessKey: string; +} + +/** `LT_USERNAME` and `LT_ACCESS_KEY` from the run's environment; throws naming each one that is unset or blank. */ +export function testmuCredentials(env: Readonly>): TestmuCredentials { + const username = envValue(env, LT_USERNAME); + const accessKey = envValue(env, LT_ACCESS_KEY); + if (username === undefined || accessKey === undefined) { + const missing = [username === undefined ? LT_USERNAME : undefined, accessKey === undefined ? LT_ACCESS_KEY : undefined].filter((name) => name !== undefined); + throw new Error( + `${missing.join(' and ')} ${missing.length === 1 ? 'is' : 'are'} not set; set ${LT_USERNAME} and ${LT_ACCESS_KEY} to your TestMu AI username and access key in the environment \`e2e run\` starts in`, + ); + } + return { username, accessKey }; +} + +/** + * Puts the run's credentials in this process's environment, which is where + * the daemon gets them: agent-device's client starts its local daemon with + * `process.env` and takes no environment of its own, while a run's + * environment can differ from `process.env` (a host that passes `env`). + * Every worker of the run already starts with these values. + */ +export function shareWithDaemon(credentials: TestmuCredentials): void { + if (process.env[LT_USERNAME] !== credentials.username) process.env[LT_USERNAME] = credentials.username; + if (process.env[LT_ACCESS_KEY] !== credentials.accessKey) process.env[LT_ACCESS_KEY] = credentials.accessKey; +} + +/** A non-empty variable from the run's environment, trimmed, or `undefined`. */ +function envValue(env: Readonly>, name: string): string | undefined { + const value = env[name]?.trim(); + return value === undefined || value === '' ? undefined : value; +} diff --git a/packages/testmu/src/index.ts b/packages/testmu/src/index.ts new file mode 100644 index 000000000..65b060dfa --- /dev/null +++ b/packages/testmu/src/index.ts @@ -0,0 +1,8 @@ +/** + * `@e2e-dev/testmu` public surface: `testmu()`, a device provider that leases + * TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real + * devices for `@e2e-dev/mobile` through agent-device's `testmu` provider. + */ + +export { testmu } from './provider.ts'; +export type { TestmuOptions } from './provider.ts'; diff --git a/packages/testmu/src/provider.ts b/packages/testmu/src/provider.ts new file mode 100644 index 000000000..0198f890e --- /dev/null +++ b/packages/testmu/src/provider.ts @@ -0,0 +1,193 @@ +/** + * TestMu AI's hosted Android emulators, iOS simulators, and real devices as a + * `DeviceProvider` for the mobile engine, through agent-device's `testmu` + * cloud provider: the provider allocates agent-device leases, and the daemon + * holding them runs each session over TestMu AI's Appium hub. + */ + +import { isAbsolute, resolve } from 'node:path'; +import type { DeviceLease, DeviceProvider, DeviceRequest } from '@e2e-dev/mobile'; +import { createAgentDeviceClient } from 'agent-device'; +import { ConfigurationError, rejectUnknownKeys } from 'e2e/engine'; +import { shareWithDaemon, testmuCredentials } from './credentials.ts'; + +/** agent-device's name for TestMu AI, as the lease provider and the tenant. */ +const PROVIDER = 'testmu'; + +/** Where each run's daemon keeps its state, under the project root. */ +const DEFAULT_STATE_DIR = '.e2e/testmu'; + +/** The dashboard project sessions are grouped under when `project` is absent. */ +const DEFAULT_PROJECT = 'e2e'; + +const DEVICE_TYPES: ReadonlySet = new Set(['virtual', 'real']); + +/** Every option `testmu()` takes, kept equal to `TestmuOptions` by the compiler. */ +const OPTION_KEYS: readonly string[] = Object.keys({ + device: true, + osVersion: true, + app: true, + deviceType: true, + project: true, + build: true, + sessionName: true, + stateDir: true, +} satisfies Record); + +/** What `testmu()` takes: the device, its OS version, and the app TestMu AI installs on it. */ +export interface TestmuOptions { + /** Device name exactly as TestMu AI's catalog lists it: `Galaxy S22 Ultra 5G`, `iPhone 16`. */ + readonly device: string; + /** OS version exactly as the catalog lists it for that device: `14` on Android, `18.0` on a virtual iOS device, `18` on a real one. */ + readonly osVersion: string; + /** + * The build TestMu AI installs on every session: an `lt://` app id, an + * `https` URL, or a local path, resolved against the project root and + * uploaded when the session starts. + */ + readonly app: string; + /** `'virtual'` (default) for an emulator or simulator, `'real'` for a real device. */ + readonly deviceType?: 'virtual' | 'real' | undefined; + /** Dashboard project the sessions are grouped under. Defaults to `e2e`. */ + readonly project?: string | undefined; + /** Dashboard build the sessions are grouped under. Defaults to the run id. */ + readonly build?: string | undefined; + /** Name of every session on the dashboard; absent, TestMu AI names it. */ + readonly sessionName?: string | undefined; + /** Directory for the agent-device daemon each run starts, relative to the project root. Defaults to `.e2e/testmu`. */ + readonly stateDir?: string | undefined; +} + +type LeaseBackend = 'ios-instance' | 'android-instance'; + +/** The lease scope agent-device resolves a leased device by, as `leases.allocate` granted it. */ +interface LeaseScope { + readonly tenant: string; + readonly runId: string; + readonly leaseId: string; + readonly leaseBackend: LeaseBackend; + readonly leaseProvider: string; +} + +/** A lease's daemon and scope: what releasing it needs. */ +interface LeaseHandle { + readonly stateDir: string; + readonly scope: LeaseScope; +} + +/** + * TestMu AI devices for `mobile({ device: testmu({ device, osVersion, app }) })`: + * one hosted device per worker slot, leased when the run starts and released + * when it ends. Each lease comes from an agent-device daemon the provider + * starts for the run under `stateDir`; the worker's first command starts the + * TestMu AI session, which installs `app`, and releasing the lease ends it. + * It authenticates with `LT_USERNAME` and `LT_ACCESS_KEY` from the run's + * environment, and needs an agent-device with the `testmu` provider. + */ +export function testmu(options: TestmuOptions): DeviceProvider { + rejectUnknownKeys('testmu()', options, OPTION_KEYS); + for (const key of ['device', 'osVersion', 'app'] as const) { + const value: unknown = options[key]; + if (typeof value !== 'string' || value.trim() === '') { + throw new ConfigurationError('INVALID_CONFIG', `testmu: \`${key}\` is required, as a non-empty string`); + } + } + const deviceType = options.deviceType ?? 'virtual'; + if (!DEVICE_TYPES.has(deviceType)) { + throw new ConfigurationError('INVALID_CONFIG', `testmu: \`deviceType\` must be 'virtual' or 'real', not ${JSON.stringify(deviceType)}`); + } + const { device, osVersion, app, project, build, sessionName } = options; + /** One release per lease, shared by every caller: the engine's, and `acquire`'s own after a failure. */ + const releases = new Map>(); + const release = (id: string, handle: LeaseHandle): Promise => { + let pending = releases.get(id); + if (pending === undefined) { + pending = releaseLease(handle); + releases.set(id, pending); + // A failed release may be tried again. + pending.catch(() => releases.delete(id)); + } + return pending; + }; + return { + name: PROVIDER, + async acquire(request: DeviceRequest): Promise { + if (request.appPath !== undefined) throw new Error("TestMu AI installs the app from `app`; leave the target's `app.appPath` out"); + shareWithDaemon(testmuCredentials(request.env)); + if (request.signal.aborted) throw new Error('cancelled before a lease was allocated'); + const stateDir = resolve(request.projectRoot, options.stateDir ?? DEFAULT_STATE_DIR, request.runId); + const leaseBackend: LeaseBackend = request.platform === 'ios' ? 'ios-instance' : 'android-instance'; + const selectors = { + platform: request.platform, + target: 'mobile' as const, + device, + providerOsVersion: osVersion, + providerApp: appSource(app, request.projectRoot), + providerDeviceType: deviceType, + providerProject: project ?? DEFAULT_PROJECT, + providerBuild: build ?? request.runId, + ...(sessionName === undefined ? {} : { providerSessionName: sessionName }), + }; + // Not cancellable: the daemon may grant the lease after an interrupt, and only a lease this returns or releases is ever released. + const granted = await createAgentDeviceClient({ stateDir, session: `lease-${request.slot}` }).leases.allocate({ + tenant: PROVIDER, + runId: request.runId, + leaseBackend, + leaseProvider: PROVIDER, + ...selectors, + }); + const scope: LeaseScope = { tenant: granted.tenantId, runId: granted.runId, leaseId: granted.leaseId, leaseBackend, leaseProvider: PROVIDER }; + try { + if (request.signal.aborted) throw new Error('cancelled'); + request.log(`lease ${scope.leaseId}: ${device}, ${request.platform} ${osVersion} (${deviceType}); the session starts on the first command`); + // The worker's client is created with these fields: the scope picks the lease, and the selectors start the session. + return { id: scope.leaseId, client: { stateDir, ...scope, ...selectors } }; + } catch (cause) { + // The engine releases only leases `acquire` returned. + const outcome = await release(scope.leaseId, { stateDir, scope }).then( + () => 'released it', + (releaseCause: unknown) => `releasing it failed (${messageOf(releaseCause)})`, + ); + throw new Error(`lease ${scope.leaseId} was not handed to the run: ${messageOf(cause)}; ${outcome}`, { cause }); + } + }, + async release(lease: DeviceLease): Promise { + const handle = leaseHandle(lease); + if (handle === undefined) throw new Error(`lease ${lease.id} carries no agent-device lease scope to release`); + await release(lease.id, handle); + }, + }; +} + +/** Releases a lease through the daemon that granted it, which ends its TestMu AI session. A lease the daemon no longer knows counts as released. */ +async function releaseLease({ stateDir, scope }: LeaseHandle): Promise { + await createAgentDeviceClient({ stateDir, session: 'release' }).leases.release(scope); +} + +/** `app` as the daemon reads it: an `lt://` id or URL as written, a local path resolved against the project root, never the daemon's working directory. */ +function appSource(app: string, projectRoot: string): string { + if (isAbsolute(app) || /^[a-z][a-z0-9+.-]*:\/\//i.test(app)) return app; + return resolve(projectRoot, app); +} + +/** The daemon and scope `acquire` put on a lease's `client`, when they are there. */ +function leaseHandle(lease: DeviceLease): LeaseHandle | undefined { + const client = lease.client as Record | undefined; + if (client === undefined) return undefined; + const { stateDir, tenant, runId, leaseId, leaseBackend, leaseProvider } = client; + if ( + typeof stateDir !== 'string' || + typeof tenant !== 'string' || + typeof runId !== 'string' || + typeof leaseId !== 'string' || + (leaseBackend !== 'ios-instance' && leaseBackend !== 'android-instance') || + typeof leaseProvider !== 'string' + ) { + return undefined; + } + return { stateDir, scope: { tenant, runId, leaseId, leaseBackend, leaseProvider } }; +} + +function messageOf(cause: unknown): string { + return cause instanceof Error ? cause.message : String(cause); +} diff --git a/packages/testmu/tests/unit/testmu.test.ts b/packages/testmu/tests/unit/testmu.test.ts new file mode 100644 index 000000000..23f8e556c --- /dev/null +++ b/packages/testmu/tests/unit/testmu.test.ts @@ -0,0 +1,284 @@ +/** + * `testmu()` against a stubbed agent-device client: the options it refuses, + * the lease it allocates and hands the worker, the credentials it reads and + * shares with the daemon, and the release on every exit path. + */ + +import { join } from 'node:path'; +import type { DeviceLease, DeviceReleaseContext, DeviceRequest } from '@e2e-dev/mobile'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { testmu, type TestmuOptions } from '../../src/index.ts'; + +interface ClientCall { + readonly config: Record; + readonly operation: 'allocate' | 'release'; + readonly options: Record; +} + +const daemon = { + calls: [] as ClientCall[], + /** Runs inside `allocate`, before it answers. */ + onAllocate: undefined as (() => void) | undefined, + allocateError: undefined as Error | undefined, + /** Errors `release` answers with, in order, before it succeeds. */ + releaseErrors: [] as Error[], +}; + +vi.mock('agent-device', () => ({ + createAgentDeviceClient: (config: Record) => ({ + leases: { + allocate: async (options: Record) => { + daemon.calls.push({ config, operation: 'allocate', options }); + daemon.onAllocate?.(); + if (daemon.allocateError !== undefined) throw daemon.allocateError; + return { leaseId: `lease-${daemon.calls.length}`, tenantId: options['tenant'], runId: options['runId'], backend: options['leaseBackend'], leaseProvider: options['leaseProvider'] }; + }, + release: async (options: Record) => { + daemon.calls.push({ config, operation: 'release', options }); + const error = daemon.releaseErrors.shift(); + if (error !== undefined) throw error; + return { released: true }; + }, + }, + }), +})); + +const ROOT = join('/', 'work', 'shop'); +const env = { LT_USERNAME: 'ada', LT_ACCESS_KEY: 'lt-key' }; +const options: TestmuOptions = { device: 'Galaxy S22 Ultra 5G', osVersion: '14', app: 'https://example.com/app.apk' }; +const saved = { LT_USERNAME: process.env['LT_USERNAME'], LT_ACCESS_KEY: process.env['LT_ACCESS_KEY'] }; + +beforeEach(() => { + Object.assign(daemon, { calls: [], onAllocate: undefined, allocateError: undefined, releaseErrors: [] }); +}); + +afterEach(() => { + for (const [name, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +}); + +function request(overrides: Partial = {}): DeviceRequest & { lines: string[] } { + const lines: string[] = []; + return { + platform: 'android', + runId: 'run-1', + targetName: 'android', + slot: 0, + slots: 2, + app: 'com.example.app', + agentDeviceVersion: '0.21.18', + projectRoot: ROOT, + env, + signal: new AbortController().signal, + log: (line) => lines.push(line), + lines, + ...overrides, + }; +} + +const context: DeviceReleaseContext = { runId: 'run-1', targetName: 'android', env, signal: new AbortController().signal, log: () => undefined }; + +const operations = () => daemon.calls.map((call) => call.operation); + +describe('testmu()', () => { + it('is a device provider named testmu that leaves recording to agent-device', () => { + const provider = testmu(options); + expect(provider.name).toBe('testmu'); + expect(provider.record).toBeUndefined(); + }); + + it('allocates an Android lease from a daemon under the project root, with the device selectors and dashboard labels', async () => { + await testmu(options).acquire(request({ slot: 1 })); + expect(daemon.calls).toEqual([ + { + config: { stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), session: 'lease-1' }, + operation: 'allocate', + options: { + tenant: 'testmu', + runId: 'run-1', + leaseBackend: 'android-instance', + leaseProvider: 'testmu', + platform: 'android', + target: 'mobile', + device: 'Galaxy S22 Ultra 5G', + providerOsVersion: '14', + providerApp: 'https://example.com/app.apk', + providerDeviceType: 'virtual', + providerProject: 'e2e', + providerBuild: 'run-1', + }, + }, + ]); + }); + + it('allocates an iOS lease on a real device with its own project, build, session name, and state directory', async () => { + await testmu({ device: 'iPhone 16', osVersion: '18', app: 'lt://APP123', deviceType: 'real', project: 'shop', build: 'nightly', sessionName: 'checkout', stateDir: 'tmp/devices' }).acquire( + request({ platform: 'ios' }), + ); + expect(daemon.calls[0]?.config['stateDir']).toBe(join(ROOT, 'tmp', 'devices', 'run-1')); + expect(daemon.calls[0]?.options).toMatchObject({ + leaseBackend: 'ios-instance', + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18', + providerApp: 'lt://APP123', + providerDeviceType: 'real', + providerProject: 'shop', + providerBuild: 'nightly', + providerSessionName: 'checkout', + }); + }); + + it('resolves a local build against the project root, never the working directory', async () => { + await testmu({ ...options, app: 'build/app.apk' }).acquire(request()); + expect(daemon.calls[0]?.options['providerApp']).toBe(join(ROOT, 'build', 'app.apk')); + await testmu({ ...options, app: join('/', 'builds', 'app.apk') }).acquire(request()); + expect(daemon.calls[1]?.options['providerApp']).toBe(join('/', 'builds', 'app.apk')); + }); + + it('hands the worker the lease scope and the selectors as JSON client configuration', async () => { + const req = request(); + const lease = await testmu(options).acquire(req); + expect(lease).toEqual({ + id: 'lease-1', + client: { + stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), + tenant: 'testmu', + runId: 'run-1', + leaseId: 'lease-1', + leaseBackend: 'android-instance', + leaseProvider: 'testmu', + platform: 'android', + target: 'mobile', + device: 'Galaxy S22 Ultra 5G', + providerOsVersion: '14', + providerApp: 'https://example.com/app.apk', + providerDeviceType: 'virtual', + providerProject: 'e2e', + providerBuild: 'run-1', + }, + }); + const json = JSON.stringify(lease); + expect(JSON.parse(json)).toEqual(lease); + expect(Buffer.byteLength(json)).toBeLessThan(1024); + expect(json).not.toContain('lt-key'); + expect(req.lines).toEqual(['lease lease-1: Galaxy S22 Ultra 5G, android 14 (virtual); the session starts on the first command']); + }); + + it('shares the run\'s credentials with the daemon it starts', async () => { + delete process.env['LT_USERNAME']; + process.env['LT_ACCESS_KEY'] = 'stale'; + daemon.onAllocate = () => expect([process.env['LT_USERNAME'], process.env['LT_ACCESS_KEY']]).toEqual(['ada', 'lt-key']); + await testmu(options).acquire(request({ env: { LT_USERNAME: ' ada ', LT_ACCESS_KEY: 'lt-key' } })); + expect(operations()).toEqual(['allocate']); + }); + + it.each([ + [{}, 'LT_USERNAME and LT_ACCESS_KEY are not set'], + [{ LT_USERNAME: 'ada' }, 'LT_ACCESS_KEY is not set'], + [{ LT_USERNAME: ' ', LT_ACCESS_KEY: 'lt-key' }, 'LT_USERNAME is not set'], + ])('fails before any daemon starts without credentials in the run\'s environment (%j)', async (runEnv, message) => { + process.env['LT_USERNAME'] = 'from-the-runner'; + process.env['LT_ACCESS_KEY'] = 'from-the-runner'; + await expect(testmu(options).acquire(request({ env: runEnv }))).rejects.toThrow( + `${message}; set LT_USERNAME and LT_ACCESS_KEY to your TestMu AI username and access key in the environment \`e2e run\` starts in`, + ); + expect(daemon.calls).toEqual([]); + }); + + it("refuses the target's app.appPath, since TestMu AI installs `app`", async () => { + await expect(testmu(options).acquire(request({ appPath: join(ROOT, 'build', 'app.apk') }))).rejects.toThrow( + "TestMu AI installs the app from `app`; leave the target's `app.appPath` out", + ); + expect(daemon.calls).toEqual([]); + }); + + it('allocates nothing once the run is interrupted', async () => { + const controller = new AbortController(); + controller.abort(); + await expect(testmu(options).acquire(request({ signal: controller.signal }))).rejects.toThrow('cancelled before a lease was allocated'); + expect(daemon.calls).toEqual([]); + }); + + it('releases a lease granted after an interrupt instead of handing it over', async () => { + const controller = new AbortController(); + daemon.onAllocate = () => controller.abort(); + const req = request({ signal: controller.signal }); + await expect(testmu(options).acquire(req)).rejects.toThrow('lease lease-1 was not handed to the run: cancelled; released it'); + expect(daemon.calls[1]).toEqual({ + config: { stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), session: 'release' }, + operation: 'release', + options: { tenant: 'testmu', runId: 'run-1', leaseId: 'lease-1', leaseBackend: 'android-instance', leaseProvider: 'testmu' }, + }); + expect(req.lines).toEqual([]); + }); + + it('releases a granted lease when handing it over fails, and says when that release failed too', async () => { + daemon.releaseErrors = [new Error('daemon gone')]; + const req = request({ + log: () => { + throw new Error('reporter closed'); + }, + }); + await expect(testmu(options).acquire(req)).rejects.toThrow('lease lease-1 was not handed to the run: reporter closed; releasing it failed (daemon gone)'); + expect(operations()).toEqual(['allocate', 'release']); + }); + + it('passes an allocation failure through with nothing to release', async () => { + daemon.allocateError = new Error('unknown lease provider testmu'); + await expect(testmu(options).acquire(request())).rejects.toThrow('unknown lease provider testmu'); + expect(operations()).toEqual(['allocate']); + }); + + it('releases a lease through the daemon that granted it, once however often it is asked', async () => { + const provider = testmu(options); + const lease = await provider.acquire(request()); + await Promise.all([provider.release(lease, context), provider.release(lease, context)]); + await provider.release(lease, context); + expect(daemon.calls.slice(1)).toEqual([ + { + config: { stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), session: 'release' }, + operation: 'release', + options: { tenant: 'testmu', runId: 'run-1', leaseId: 'lease-1', leaseBackend: 'android-instance', leaseProvider: 'testmu' }, + }, + ]); + }); + + it('releases from the lease alone, after a round trip through JSON', async () => { + const lease = JSON.parse(JSON.stringify(await testmu(options).acquire(request({ platform: 'ios' })))) as DeviceLease; + await testmu(options).release(lease, context); + expect(daemon.calls[1]?.options).toEqual({ tenant: 'testmu', runId: 'run-1', leaseId: 'lease-1', leaseBackend: 'ios-instance', leaseProvider: 'testmu' }); + }); + + it('tries a release again after one failed', async () => { + daemon.releaseErrors = [new Error('daemon busy')]; + const provider = testmu(options); + const lease = await provider.acquire(request()); + await expect(provider.release(lease, context)).rejects.toThrow('daemon busy'); + await provider.release(lease, context); + expect(operations()).toEqual(['allocate', 'release', 'release']); + }); + + it('refuses to release a lease without an agent-device scope', async () => { + await expect(testmu(options).release({ id: 'other', client: { stateDir: '/tmp/x' } }, context)).rejects.toThrow('lease other carries no agent-device lease scope to release'); + expect(daemon.calls).toEqual([]); + }); + + it('rejects an option it does not take with INVALID_CONFIG, naming the nearest one', () => { + expect(() => testmu({ ...options, osVerison: '14' } as unknown as TestmuOptions)).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG', message: expect.stringContaining('osVersion') })); + }); + + it.each(['device', 'osVersion', 'app'] as const)('requires `%s` with INVALID_CONFIG', (key) => { + expect(() => testmu({ ...options, [key]: ' ' })).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG', message: `testmu: \`${key}\` is required, as a non-empty string` })); + const { [key]: _, ...rest } = options; + expect(() => testmu(rest as TestmuOptions)).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG' })); + }); + + it('refuses a device type other than virtual or real', () => { + expect(() => testmu({ ...options, deviceType: 'emulator' as 'virtual' })).toThrow( + expect.objectContaining({ code: 'INVALID_CONFIG', message: 'testmu: `deviceType` must be \'virtual\' or \'real\', not "emulator"' }), + ); + }); +}); diff --git a/packages/testmu/tsconfig.build.json b/packages/testmu/tsconfig.build.json new file mode 100644 index 000000000..65d5a8150 --- /dev/null +++ b/packages/testmu/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true + }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/testmu/tsconfig.json b/packages/testmu/tsconfig.json new file mode 100644 index 000000000..635a4f631 --- /dev/null +++ b/packages/testmu/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/testmu/vitest.config.ts b/packages/testmu/vitest.config.ts new file mode 100644 index 000000000..c7516095d --- /dev/null +++ b/packages/testmu/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config'; + +/** Pure unit tests: agent-device's client is a stub, no daemon and no network. */ +export default defineConfig({ + test: { + include: ['tests/unit/**/*.test.ts'], + testTimeout: 30_000, + pool: 'forks', + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7b810cfb4..e80250465 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -290,6 +290,9 @@ importers: '@e2e-dev/mobile': specifier: workspace:* version: link:../packages/mobile + '@e2e-dev/testmu': + specifier: workspace:* + version: link:../packages/testmu '@e2e-dev/web': specifier: workspace:* version: link:../packages/web @@ -473,6 +476,27 @@ importers: specifier: 5.0.1 version: 5.0.1(@types/node@26.6.2)(vite@8.1.5(@types/node@26.6.2)(esbuild@0.28.2)(jiti@1.21.7)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0)) + packages/testmu: + devDependencies: + '@e2e-dev/mobile': + specifier: workspace:* + version: link:../mobile + '@types/node': + specifier: 26.6.2 + version: 26.6.2 + agent-device: + specifier: 0.21.18 + version: 0.21.18(ai@7.0.107(zod@4.6.1)) + e2e: + specifier: workspace:* + version: link:../e2e + typescript: + specifier: 7.0.2 + version: 7.0.2 + vitest: + specifier: 5.0.1 + version: 5.0.1(@types/node@26.6.2)(vite@8.1.5(@types/node@26.6.2)(esbuild@0.28.2)(jiti@1.21.7)(terser@5.51.2)(tsx@4.23.14)(yaml@2.9.0)) + packages/web: devDependencies: '@types/node': From 98c60c1814d6d5d6dfdb0d93ffbd6695576c474e Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:55:37 +0530 Subject: [PATCH 2/4] docs(integrations): document @e2e-dev/testmu Add a TestMu AI (formerly LambdaTest) integration page for the testmu() device provider from @e2e-dev/testmu, in the shape of the EAS Simulators page: setup, authentication, installing the app, real devices, an options table, sessions, what the provider does, and limits. The page covers the requirement for an agent-device that includes the testmu provider and the override that makes @e2e-dev/mobile use the same copy, the catalog-exact device names and OS versions (18.0 for a virtual iOS device, 18 for a real one, 14 on Android), the app formats per device, and the limits of agent-device's WebDriver runtime. The config and test examples are shown verbatim and typecheck against the built package. The integrations overview, the hosted devices section of the mobile page, the environment reference (LT_USERNAME, LT_ACCESS_KEY), and the agent skill's setup reference name the package. Co-Authored-By: Claude Opus 5.5 --- docs/docs.json | 3 +- docs/examples/mobile/testmu.config.ts | 35 ++++ docs/examples/mobile/testmu.e2e.ts | 8 + docs/integrations/index.mdx | 3 + docs/integrations/testmu.mdx | 236 ++++++++++++++++++++++++++ docs/mobile.mdx | 6 +- docs/package.json | 1 + docs/reference/environment.mdx | 1 + scripts/check-docs-examples.ts | 2 + skills/e2e/references/setup.md | 6 + 10 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 docs/examples/mobile/testmu.config.ts create mode 100644 docs/examples/mobile/testmu.e2e.ts create mode 100644 docs/integrations/testmu.mdx diff --git a/docs/docs.json b/docs/docs.json index 19b432cee..52905d389 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -203,7 +203,8 @@ "pages": [ "integrations/index", "integrations/kernel", - "integrations/eas" + "integrations/eas", + "integrations/testmu" ] }, { diff --git a/docs/examples/mobile/testmu.config.ts b/docs/examples/mobile/testmu.config.ts new file mode 100644 index 000000000..eacf33ccf --- /dev/null +++ b/docs/examples/mobile/testmu.config.ts @@ -0,0 +1,35 @@ +import type { E2EConfig } from 'e2e'; +import { mobile } from '@e2e-dev/mobile'; +import { testmu } from '@e2e-dev/testmu'; + +const apk = 'https://prod-mobile-artefacts.lambdatest.com/assets/docs/proverbial_android.apk'; + +export default { + targets: [ + { + name: 'android-emulator', + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Galaxy S22 Ultra 5G', osVersion: '14', app: apk }), + }), + app: { bundleId: 'com.lambdatest.proverbial' }, + }, + { + name: 'ios-simulator', + engine: mobile({ + platform: 'ios', + device: testmu({ device: 'iPhone 16', osVersion: '18.0', app: './build/MyApp.zip' }), + }), + app: { bundleId: 'com.example.app' }, + }, + { + name: 'android-real', + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Pixel 6', osVersion: '14', app: apk, deviceType: 'real' }), + }), + app: { bundleId: 'com.lambdatest.proverbial' }, + }, + ], + workers: 1, +} satisfies E2EConfig; diff --git a/docs/examples/mobile/testmu.e2e.ts b/docs/examples/mobile/testmu.e2e.ts new file mode 100644 index 000000000..5602626dc --- /dev/null +++ b/docs/examples/mobile/testmu.e2e.ts @@ -0,0 +1,8 @@ +import { test } from '@e2e-dev/mobile'; +import { expect } from 'e2e'; + +test('Proverbial home screen', async ({ app, screen }) => { + await app.open(); + await expect(screen.getByText('Proverbial')).toBeVisible(); + await expect(screen.getByRole('button', 'GEOLOCATION')).toBeVisible(); +}); diff --git a/docs/integrations/index.mdx b/docs/integrations/index.mdx index 10f027b96..b6cebe12a 100644 --- a/docs/integrations/index.mdx +++ b/docs/integrations/index.mdx @@ -16,6 +16,9 @@ integration decides where it comes from. `@e2e-dev/eas`: hosted iOS simulators and Android emulators for mobile targets. + + `@e2e-dev/testmu`: hosted Android emulators, iOS simulators, and real devices for mobile targets. + A service without an integration is one small provider in your project: diff --git a/docs/integrations/testmu.mdx b/docs/integrations/testmu.mdx new file mode 100644 index 000000000..5549491ca --- /dev/null +++ b/docs/integrations/testmu.mdx @@ -0,0 +1,236 @@ +--- +title: TestMu AI +sidebarTitle: TestMu AI +description: Run mobile targets on TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real devices with the testmu() device provider from @e2e-dev/testmu. +--- + +`testmu()` from `@e2e-dev/testmu` is a +[device provider](/mobile#hosted-devices) for +[TestMu AI](https://www.lambdatest.com) (formerly LambdaTest). The run gets +hosted Android emulators, iOS simulators, or real devices, leased when it +starts and released when it ends. agent-device reaches them through its +`testmu` cloud provider, which drives each session over TestMu AI's Appium +hub. Tests, config, and CI stay the same. + + +No Mac, Xcode, or Android SDK on the machine that runs e2e. + + +## Setup + +The provider runs on agent-device's `testmu` provider, so install +`agent-device` beside the package, at a release that includes it: + + +```bash npm +npm install --save-dev @e2e-dev/testmu agent-device@ +``` + +```bash pnpm +pnpm add -D @e2e-dev/testmu agent-device@ +``` + +```bash bun +bun add -d @e2e-dev/testmu agent-device@ +``` + + +`` is an agent-device release that includes the `testmu` provider. +No published release does yet: until one does, install agent-device from a +build that includes it. The package's `agent-device` peer range excludes +only the releases known to lack the provider, so it cannot tell you whether +a newer one has it. + +`@e2e-dev/mobile` pins its own agent-device. The provider starts the daemon +from your copy and the engine's workers send commands through theirs, so +both must be the same agent-device. Override the engine's pin: + + +```json npm +{ + "overrides": { + "agent-device": "$agent-device" + } +} +``` + +```yaml pnpm +# pnpm-workspace.yaml +overrides: + agent-device: $agent-device +``` + +```json bun +{ + "overrides": { + "agent-device": "" + } +} +``` + + +For npm and bun the `overrides` field goes in `package.json`. + +```ts title="e2e.config.ts" +import type { E2EConfig } from 'e2e'; +import { mobile } from '@e2e-dev/mobile'; +import { testmu } from '@e2e-dev/testmu'; + +const apk = 'https://prod-mobile-artefacts.lambdatest.com/assets/docs/proverbial_android.apk'; + +export default { + targets: [ + { + name: 'android-emulator', + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Galaxy S22 Ultra 5G', osVersion: '14', app: apk }), + }), + app: { bundleId: 'com.lambdatest.proverbial' }, + }, + { + name: 'ios-simulator', + engine: mobile({ + platform: 'ios', + device: testmu({ device: 'iPhone 16', osVersion: '18.0', app: './build/MyApp.zip' }), + }), + app: { bundleId: 'com.example.app' }, + }, + { + name: 'android-real', + engine: mobile({ + platform: 'android', + device: testmu({ device: 'Pixel 6', osVersion: '14', app: apk, deviceType: 'real' }), + }), + app: { bundleId: 'com.lambdatest.proverbial' }, + }, + ], + workers: 1, +} satisfies E2EConfig; +``` + +`device` and `osVersion` must match TestMu AI's catalog exactly, as the +[capabilities generator](https://www.lambdatest.com/capabilities-generator) +lists them: a virtual iOS device takes a version like `18.0`, a real iOS +device one like `18`, and Android one like `14`. TestMu AI checks them when +the session starts and refuses a name or version it does not list. + +Keep `app.bundleId`: TestMu AI installs the app when the session starts, +and `app.open()` launches the installed bundle id or package, not the upload +name. Leave `app.appPath` out; a target that names one fails its lease. + +## Authenticate + +Set `LT_USERNAME` and `LT_ACCESS_KEY` to your TestMu AI username and access +key in the environment `e2e run` starts in, such as CI secrets. Without +either, the lease fails before anything starts, naming the missing one. + +agent-device's `testmu` runtime reads the credentials from the environment +of the daemon that drives the sessions. The provider starts that daemon from +the runner process, and agent-device starts a daemon with the process's own +environment, so the provider copies both values from the run's environment +into it first. + +## Install the app + +`app` names the build TestMu AI installs on every session: + +- **An app id**: an `lt://` reference to a build already in TestMu AI's app + storage. +- **A URL**: an `https` link to the build. +- **A local path**: a build on the machine that runs e2e, resolved against + the directory of `e2e.config.ts` and uploaded when the session starts. + +The build must suit the device: + +| Device | Build | +| --- | --- | +| Android emulator or real device | `.apk` or `.aab` | +| iOS simulator | a zipped `.app` bundle, not an `.ipa` | +| Real iOS device | a signed `.ipa` | + +## Real devices + +`deviceType: 'real'` leases a real device instead of an emulator or +simulator. Real and virtual devices have separate catalogs, so check +`device` and `osVersion` against the real-device one, and give a real iOS +device a signed `.ipa`. Real devices need an agent-device release whose +`testmu` provider supports them. + +## Options + +| Option | Default | Meaning | +| ------------- | --------------- | ------- | +| `device` | required | Device name exactly as TestMu AI's catalog lists it (`'Galaxy S22 Ultra 5G'`, `'iPhone 16'`). | +| `osVersion` | required | OS version exactly as the catalog lists it for that device: `'14'` on Android, `'18.0'` on a virtual iOS device, `'18'` on a real one. | +| `app` | required | The build TestMu AI installs: an `lt://` app id, an `https` URL, or a local path. See [Install the app](#install-the-app). | +| `deviceType` | `'virtual'` | `'virtual'` for an emulator or simulator, `'real'` for a real device. | +| `project` | `'e2e'` | Dashboard project the sessions are grouped under. | +| `build` | the run id | Dashboard build the sessions are grouped under. | +| `sessionName` | TestMu AI's | Name of every session on the dashboard. | +| `stateDir` | `'.e2e/testmu'` | Directory for the agent-device daemon each run starts, relative to the directory of `e2e.config.ts`. Keep it out of version control. | + +An option not in this table, a missing or empty `device`, `osVersion`, or +`app`, or another `deviceType` fails the config load with `INVALID_CONFIG`. + +## Write a test + +Tests are the same as on a local device: + +```ts title="tests/home.e2e.ts" +import { test } from '@e2e-dev/mobile'; +import { expect } from 'e2e'; + +test('Proverbial home screen', async ({ app, screen }) => { + await app.open(); + await expect(screen.getByText('Proverbial')).toBeVisible(); + await expect(screen.getByRole('button', 'GEOLOCATION')).toBeVisible(); +}); +``` + +## Sessions + +Each target leases up to `workers` devices (`--workers` overrides it), one +per worker slot, before the run's clock starts. A session takes about 30 to +75 seconds to start, allocation and app upload included. TestMu AI +runs as many sessions at once as the plan allows, so keep `workers` times +the number of TestMu AI targets within it. + +The provider logs each lease as it is granted: + +```text +ℹ leasing 1 android device(s) from testmu +ℹ testmu (1 of 1): lease 4f0c…: Galaxy S22 Ultra 5G, android 14 (virtual); the session starts on the first command +ℹ testmu: leased 4f0c… +``` + +The video, device logs, and network logs of every session are on the +TestMu AI dashboard, under the project and build the options name. + +## What the provider does + +- Starts one agent-device daemon per run under `stateDir/` and + allocates a `testmu` lease from it for each worker slot. +- Hands the worker the lease scope and the device selectors as agent-device + client configuration. The worker's first command starts the TestMu AI + session from them, which installs `app`. +- Releases each lease when the run ends, which ends its session. A lease + is released once however often the engine asks, and a failed release can + be tried again. +- Releases a lease it was granted after the run was interrupted, or could + not hand to the run, before failing the lease. + +## Limits + +Sessions run over TestMu AI's Appium hub through agent-device's WebDriver +runtime, so what agent-device does not support there is not available to +tests: + +- `device` settings: permissions, location, network, and appearance. +- System alerts. +- Recording through agent-device. Leave [`video`](/reference/config#video) + off for these targets. TestMu AI records every session itself, and the + video and device logs are on its dashboard. +- Device logs through agent-device; read them on the dashboard. +- Port reverse. To reach a server on your machine or network, use TestMu + AI Tunnel. diff --git a/docs/mobile.mdx b/docs/mobile.mdx index f1f94c8d8..27577eaf6 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -146,7 +146,8 @@ ends. The engine drives each lease through the agent-device daemon the lease names instead of the local one, so any service that runs an agent-device daemon next to a simulator, an emulator, or a physical device works. Nothing in the framework knows a vendor: for [EAS Simulators](/integrations/eas), -`easSimulators()` from `@e2e-dev/eas` is the provider, and for +`easSimulators()` from `@e2e-dev/eas` is the provider, for +[TestMu AI](/integrations/testmu) `testmu()` from `@e2e-dev/testmu`, and for any other service a provider is a few dozen lines in your project. ```ts @@ -302,7 +303,8 @@ on `app.bundleId`, so traces recorded locally replay on hosted devices. A device cloud agent-device speaks to itself needs no session API of your own: the provider allocates a lease through a daemon it starts for the run and -hands the worker the lease scope as `client`. +hands the worker the lease scope as `client`. `testmu()` from +`@e2e-dev/testmu` works this way for [TestMu AI](/integrations/testmu). ```ts title="device-cloud-provider.ts" import { createAgentDeviceClient } from 'agent-device'; diff --git a/docs/package.json b/docs/package.json index c55d4f486..8050c5c70 100644 --- a/docs/package.json +++ b/docs/package.json @@ -10,6 +10,7 @@ }, "devDependencies": { "@e2e-dev/mobile": "workspace:*", + "@e2e-dev/testmu": "workspace:*", "e2e": "workspace:*", "@e2e-dev/web": "workspace:*", "@types/node": "26.6.2", diff --git a/docs/reference/environment.mdx b/docs/reference/environment.mdx index 767e5d6d3..217ec2186 100644 --- a/docs/reference/environment.mdx +++ b/docs/reference/environment.mdx @@ -103,6 +103,7 @@ What is sent and how to read it is on the [Telemetry](/telemetry) page. | `KERNEL_API_KEY` | value | The API key `kernel()` from `@e2e-dev/kernel` creates browsers with; unset, the browser lease fails. See [Kernel](/integrations/kernel) | | `EXPO_TOKEN` | value | The Expo access token `easSimulators()` from `@e2e-dev/eas` starts and stops simulator sessions with; unset, it uses the eas-cli login in `.expo/state.json` under `HOME` (`USERPROFILE` on Windows), and with neither the device lease fails. See [EAS Simulators](/integrations/eas) | | `HOME`, `USERPROFILE` | value | The home `easSimulators()` finds the eas-cli login under when `EXPO_TOKEN` is unset: `USERPROFILE` on Windows, `HOME` elsewhere, the OS user's home when unset | +| `LT_USERNAME`, `LT_ACCESS_KEY` | value | The TestMu AI username and access key `testmu()` from `@e2e-dev/testmu` leases devices with; it copies them into the runner process's environment for the agent-device daemon it starts, and with either unset the device lease fails. See [TestMu AI](/integrations/testmu) | | `E2E_AGENT_DEVICE_POOL__` | value | Written by `@e2e-dev/mobile` in `prepare` and read by each worker to find its booted device. Not meant to be set by hand | ## What app processes inherit diff --git a/scripts/check-docs-examples.ts b/scripts/check-docs-examples.ts index a19fd5832..aff292df2 100644 --- a/scripts/check-docs-examples.ts +++ b/scripts/check-docs-examples.ts @@ -27,6 +27,8 @@ const EXAMPLES: Record = { 'docs/examples/quickstart/e2e.command.config.ts': 'docs/starting-your-app.mdx', 'docs/examples/mobile/device-provider.ts': 'docs/mobile.mdx', 'docs/examples/mobile/device-cloud-provider.ts': 'docs/mobile.mdx', + 'docs/examples/mobile/testmu.config.ts': 'docs/integrations/testmu.mdx', + 'docs/examples/mobile/testmu.e2e.ts': 'docs/integrations/testmu.mdx', 'docs/examples/web/browser-provider.ts': 'docs/browser.mdx', 'docs/examples/skill/e2e.config.ts': 'skills/e2e/SKILL.md', 'docs/examples/skill/e2e.setup.config.ts': 'skills/e2e/references/setup.md', diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 7c42c1e01..4a4fffb52 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -299,6 +299,12 @@ export default { the app (omit `app.appPath`). A run must fit one session: `maxDurationMinutes`, absent, is the account's cap (40 on a standard plan). `videoTouches: false` on the engine for video there. + `testmu({ device, osVersion, app })` from `@e2e-dev/testmu` leases TestMu + AI (formerly LambdaTest) emulators, simulators, or real devices + (`deviceType: 'real'`); `device` and `osVersion` match its catalog + exactly, TestMu AI installs `app` (omit `app.appPath`), it reads + `LT_USERNAME` and `LT_ACCESS_KEY`, and needs an agent-device with the + `testmu` provider shared with `@e2e-dev/mobile` (override its pin). - Only a control that appeared or moved with the previous action waits out `transition` (default 500 ms); agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. From 9a797ebdef0f2adc8182da8b2d45a04310e6dca8 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 20:26:28 +0530 Subject: [PATCH 3/4] fix(testmu): keep the lease alive while the session starts agent-device ends a lease after 60 seconds without activity by default, and a command still running does not count as activity. The worker's first command uploads the app, allocates the hosted session, and boots it; on an iOS simulator that takes about 70 seconds, so the lease lapsed mid-start and the engine's prepare failed with "boot failed: Lease is not active" (LEASE_NOT_FOUND). Each lease now asks for agent-device's longest inactivity window (10 minutes), and the provider heartbeats every lease it holds every 2 minutes from the runner process until the lease is released, including the release acquire does itself when it cannot hand a lease to the run. The timer is unref'd so it never keeps the runner alive. A failed heartbeat is logged once per lease and the next one is still tried; it never fails the run. Co-Authored-By: Claude Opus 5.5 --- .changeset/testmu-devices.md | 2 +- docs/integrations/testmu.mdx | 4 ++ packages/testmu/src/provider.ts | 40 +++++++++++ packages/testmu/tests/unit/testmu.test.ts | 81 ++++++++++++++++++++++- 4 files changed, 123 insertions(+), 4 deletions(-) diff --git a/.changeset/testmu-devices.md b/.changeset/testmu-devices.md index c19d878ec..f610a0367 100644 --- a/.changeset/testmu-devices.md +++ b/.changeset/testmu-devices.md @@ -2,4 +2,4 @@ "@e2e-dev/testmu": minor --- -`@e2e-dev/testmu`: `mobile({ device: testmu({ device, osVersion, app }) })` runs a mobile target on TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real devices (`deviceType: 'real'`), one per worker slot, through agent-device's `testmu` cloud provider. Each slot leases a device from an agent-device daemon the provider starts for the run under `stateDir` (default `.e2e/testmu`); the worker's first command starts the TestMu AI session, which installs `app` (an `lt://` id, an `https` URL, or a local path), and releasing the lease ends it. It reads `LT_USERNAME` and `LT_ACCESS_KEY` from the run's environment and fails the lease before starting anything when either is missing, groups sessions on the dashboard under `project` (default `e2e`) and `build` (default the run id), releases a lease it allocated when the run was interrupted, and releases each lease once. It needs an agent-device that includes the `testmu` provider, which no published release does yet, with `@e2e-dev/mobile`'s agent-device overridden to the same version; `agent-device` and `@e2e-dev/mobile` are its peers. +`@e2e-dev/testmu`: `mobile({ device: testmu({ device, osVersion, app }) })` runs a mobile target on TestMu AI (formerly LambdaTest) Android emulators, iOS simulators, and real devices (`deviceType: 'real'`), one per worker slot, through agent-device's `testmu` cloud provider. Each slot leases a device from an agent-device daemon the provider starts for the run under `stateDir` (default `.e2e/testmu`); the worker's first command starts the TestMu AI session, which installs `app` (an `lt://` id, an `https` URL, or a local path), and releasing the lease ends it. It reads `LT_USERNAME` and `LT_ACCESS_KEY` from the run's environment and fails the lease before starting anything when either is missing, groups sessions on the dashboard under `project` (default `e2e`) and `build` (default the run id), heartbeats each lease while the run holds it so a session slow to start keeps its lease, releases a lease it allocated when the run was interrupted, and releases each lease once. It needs an agent-device that includes the `testmu` provider, which no published release does yet, with `@e2e-dev/mobile`'s agent-device overridden to the same version; `agent-device` and `@e2e-dev/mobile` are its peers. diff --git a/docs/integrations/testmu.mdx b/docs/integrations/testmu.mdx index 5549491ca..ddd216fd0 100644 --- a/docs/integrations/testmu.mdx +++ b/docs/integrations/testmu.mdx @@ -211,6 +211,10 @@ TestMu AI dashboard, under the project and build the options name. - Starts one agent-device daemon per run under `stateDir/` and allocates a `testmu` lease from it for each worker slot. +- Keeps each lease alive while the run holds it. A lease asks for + agent-device's longest inactivity window, 10 minutes, and the provider + heartbeats it every 2 minutes, so a session that takes long to start keeps + its lease. A failed heartbeat is logged once and does not fail the run. - Hands the worker the lease scope and the device selectors as agent-device client configuration. The worker's first command starts the TestMu AI session from them, which installs `app`. diff --git a/packages/testmu/src/provider.ts b/packages/testmu/src/provider.ts index 0198f890e..876d2a447 100644 --- a/packages/testmu/src/provider.ts +++ b/packages/testmu/src/provider.ts @@ -20,6 +20,16 @@ const DEFAULT_STATE_DIR = '.e2e/testmu'; /** The dashboard project sessions are grouped under when `project` is absent. */ const DEFAULT_PROJECT = 'e2e'; +/** + * The inactivity window each lease asks for, agent-device's longest. Its + * 60-second default lapses while the worker's first command uploads the app + * and starts the session, which can take longer on an iOS simulator. + */ +const LEASE_TTL_MS = 10 * 60_000; + +/** How often the runner heartbeats each lease it holds, well inside `LEASE_TTL_MS`. */ +const HEARTBEAT_INTERVAL_MS = 2 * 60_000; + const DEVICE_TYPES: ReadonlySet = new Set(['virtual', 'real']); /** Every option `testmu()` takes, kept equal to `TestmuOptions` by the compiler. */ @@ -99,7 +109,11 @@ export function testmu(options: TestmuOptions): DeviceProvider { const { device, osVersion, app, project, build, sessionName } = options; /** One release per lease, shared by every caller: the engine's, and `acquire`'s own after a failure. */ const releases = new Map>(); + /** Stops each held lease's heartbeat, by lease id. */ + const heartbeats = new Map void>(); const release = (id: string, handle: LeaseHandle): Promise => { + heartbeats.get(id)?.(); + heartbeats.delete(id); let pending = releases.get(id); if (pending === undefined) { pending = releaseLease(handle); @@ -134,9 +148,11 @@ export function testmu(options: TestmuOptions): DeviceProvider { runId: request.runId, leaseBackend, leaseProvider: PROVIDER, + ttlMs: LEASE_TTL_MS, ...selectors, }); const scope: LeaseScope = { tenant: granted.tenantId, runId: granted.runId, leaseId: granted.leaseId, leaseBackend, leaseProvider: PROVIDER }; + heartbeats.set(scope.leaseId, keepAlive({ stateDir, scope }, request.log)); try { if (request.signal.aborted) throw new Error('cancelled'); request.log(`lease ${scope.leaseId}: ${device}, ${request.platform} ${osVersion} (${deviceType}); the session starts on the first command`); @@ -164,6 +180,30 @@ async function releaseLease({ stateDir, scope }: LeaseHandle): Promise { await createAgentDeviceClient({ stateDir, session: 'release' }).leases.release(scope); } +/** + * Heartbeats a lease until the returned function is called. A command still + * running does not keep its lease alive, so without this a lease can lapse + * while the worker's first command is starting the session. A failed + * heartbeat is logged once and the next one tried; it never fails the run. + */ +function keepAlive({ stateDir, scope }: LeaseHandle, log: (line: string) => void): () => void { + const client = createAgentDeviceClient({ stateDir, session: 'heartbeat' }); + let warned = false; + const timer = setInterval(() => { + client.leases.heartbeat({ ...scope, ttlMs: LEASE_TTL_MS }).catch((cause: unknown) => { + if (warned) return; + warned = true; + try { + log(`lease ${scope.leaseId}: heartbeat failed (${messageOf(cause)}); agent-device ends the lease after ${LEASE_TTL_MS / 60_000} minutes without one`); + } catch { + // The run's log may be closed by now. + } + }); + }, HEARTBEAT_INTERVAL_MS); + timer.unref(); + return () => clearInterval(timer); +} + /** `app` as the daemon reads it: an `lt://` id or URL as written, a local path resolved against the project root, never the daemon's working directory. */ function appSource(app: string, projectRoot: string): string { if (isAbsolute(app) || /^[a-z][a-z0-9+.-]*:\/\//i.test(app)) return app; diff --git a/packages/testmu/tests/unit/testmu.test.ts b/packages/testmu/tests/unit/testmu.test.ts index 23f8e556c..de4c6fecf 100644 --- a/packages/testmu/tests/unit/testmu.test.ts +++ b/packages/testmu/tests/unit/testmu.test.ts @@ -1,7 +1,8 @@ /** * `testmu()` against a stubbed agent-device client: the options it refuses, * the lease it allocates and hands the worker, the credentials it reads and - * shares with the daemon, and the release on every exit path. + * shares with the daemon, the heartbeat that keeps it alive, and the release + * on every exit path. */ import { join } from 'node:path'; @@ -11,7 +12,7 @@ import { testmu, type TestmuOptions } from '../../src/index.ts'; interface ClientCall { readonly config: Record; - readonly operation: 'allocate' | 'release'; + readonly operation: 'allocate' | 'heartbeat' | 'release'; readonly options: Record; } @@ -22,6 +23,8 @@ const daemon = { allocateError: undefined as Error | undefined, /** Errors `release` answers with, in order, before it succeeds. */ releaseErrors: [] as Error[], + /** Errors `heartbeat` answers with, in order, before it succeeds. */ + heartbeatErrors: [] as Error[], }; vi.mock('agent-device', () => ({ @@ -33,6 +36,12 @@ vi.mock('agent-device', () => ({ if (daemon.allocateError !== undefined) throw daemon.allocateError; return { leaseId: `lease-${daemon.calls.length}`, tenantId: options['tenant'], runId: options['runId'], backend: options['leaseBackend'], leaseProvider: options['leaseProvider'] }; }, + heartbeat: async (options: Record) => { + daemon.calls.push({ config, operation: 'heartbeat', options }); + const error = daemon.heartbeatErrors.shift(); + if (error !== undefined) throw error; + return { leaseId: options['leaseId'], tenantId: options['tenant'], runId: options['runId'], backend: options['leaseBackend'] }; + }, release: async (options: Record) => { daemon.calls.push({ config, operation: 'release', options }); const error = daemon.releaseErrors.shift(); @@ -49,10 +58,11 @@ const options: TestmuOptions = { device: 'Galaxy S22 Ultra 5G', osVersion: '14', const saved = { LT_USERNAME: process.env['LT_USERNAME'], LT_ACCESS_KEY: process.env['LT_ACCESS_KEY'] }; beforeEach(() => { - Object.assign(daemon, { calls: [], onAllocate: undefined, allocateError: undefined, releaseErrors: [] }); + Object.assign(daemon, { calls: [], onAllocate: undefined, allocateError: undefined, releaseErrors: [], heartbeatErrors: [] }); }); afterEach(() => { + vi.useRealTimers(); for (const [name, value] of Object.entries(saved)) { if (value === undefined) delete process.env[name]; else process.env[name] = value; @@ -82,6 +92,8 @@ const context: DeviceReleaseContext = { runId: 'run-1', targetName: 'android', e const operations = () => daemon.calls.map((call) => call.operation); +const MINUTE = 60_000; + describe('testmu()', () => { it('is a device provider named testmu that leaves recording to agent-device', () => { const provider = testmu(options); @@ -108,6 +120,7 @@ describe('testmu()', () => { providerDeviceType: 'virtual', providerProject: 'e2e', providerBuild: 'run-1', + ttlMs: 10 * MINUTE, }, }, ]); @@ -266,6 +279,68 @@ describe('testmu()', () => { expect(daemon.calls).toEqual([]); }); + it('heartbeats each lease it holds every two minutes, asking for the longest lease, until it is released', async () => { + vi.useFakeTimers(); + const provider = testmu(options); + const lease = await provider.acquire(request()); + await vi.advanceTimersByTimeAsync(2 * MINUTE - 1); + expect(operations()).toEqual(['allocate']); + await vi.advanceTimersByTimeAsync(1); + expect(daemon.calls[1]).toEqual({ + config: { stateDir: join(ROOT, '.e2e', 'testmu', 'run-1'), session: 'heartbeat' }, + operation: 'heartbeat', + options: { tenant: 'testmu', runId: 'run-1', leaseId: 'lease-1', leaseBackend: 'android-instance', leaseProvider: 'testmu', ttlMs: 10 * MINUTE }, + }); + await vi.advanceTimersByTimeAsync(2 * MINUTE); + expect(operations()).toEqual(['allocate', 'heartbeat', 'heartbeat']); + await provider.release(lease, context); + await vi.advanceTimersByTimeAsync(10 * MINUTE); + expect(operations()).toEqual(['allocate', 'heartbeat', 'heartbeat', 'release']); + }); + + it('stops heartbeating a lease it released because handing it over failed', async () => { + vi.useFakeTimers(); + const controller = new AbortController(); + daemon.onAllocate = () => controller.abort(); + await expect(testmu(options).acquire(request({ signal: controller.signal }))).rejects.toThrow('was not handed to the run'); + await vi.advanceTimersByTimeAsync(10 * MINUTE); + expect(operations()).toEqual(['allocate', 'release']); + }); + + it('heartbeats nothing when the allocation failed', async () => { + vi.useFakeTimers(); + daemon.allocateError = new Error('no capacity'); + await expect(testmu(options).acquire(request())).rejects.toThrow('no capacity'); + await vi.advanceTimersByTimeAsync(10 * MINUTE); + expect(operations()).toEqual(['allocate']); + }); + + it('logs the first failed heartbeat and keeps beating, without failing the run', async () => { + vi.useFakeTimers(); + daemon.heartbeatErrors = [new Error('Lease is not active'), new Error('daemon gone')]; + const req = request(); + await testmu(options).acquire(req); + await vi.advanceTimersByTimeAsync(6 * MINUTE); + expect(operations()).toEqual(['allocate', 'heartbeat', 'heartbeat', 'heartbeat']); + expect(req.lines.slice(1)).toEqual(['lease lease-1: heartbeat failed (Lease is not active); agent-device ends the lease after 10 minutes without one']); + }); + + it('survives a failed heartbeat when the run\'s log is closed', async () => { + vi.useFakeTimers(); + daemon.heartbeatErrors = [new Error('daemon gone')]; + let closed = false; + await testmu(options).acquire( + request({ + log: () => { + if (closed) throw new Error('reporter closed'); + }, + }), + ); + closed = true; + await vi.advanceTimersByTimeAsync(4 * MINUTE); + expect(operations()).toEqual(['allocate', 'heartbeat', 'heartbeat']); + }); + it('rejects an option it does not take with INVALID_CONFIG, naming the nearest one', () => { expect(() => testmu({ ...options, osVerison: '14' } as unknown as TestmuOptions)).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG', message: expect.stringContaining('osVersion') })); }); From 1ef8f7766178fb40359ac0adec7b6394f51c8930 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 20:27:31 +0530 Subject: [PATCH 4/4] fix(mobile): read XCUIElementType-prefixed iOS element types agent-device's native iOS runner reports element types as `Button`, `StaticText`, `NavigationBar`. Its WebDriver runtime, which hosted Appium clouds (TestMu AI, BrowserStack, AWS Device Farm) go through, reports the XCUITest class names instead: `XCUIElementTypeButton`, `XCUIElementTypeStaticText`. normalizeKind kebab-cased those to `xcuielement-type-button`, which maps to no role, so on a real iPhone `getByRole('button', 'CPU Load')` matched nothing and observe listed the node by its raw class name. normalizeKind now drops a leading `XCUIElementType` before kebab-casing, so both spellings land on one vocabulary: buttons, tab bar items as tabs, the navigation bar as the screen title. Only that exact prefix followed by a type name is stripped; Android class names are untouched, and the golden device trees project unchanged. Co-Authored-By: Claude Opus 5.5 --- .changeset/ios-webdriver-element-types.md | 5 ++++ packages/mobile/src/nodes.ts | 11 +++++---- packages/mobile/tests/unit/nodes.test.ts | 30 +++++++++++++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 .changeset/ios-webdriver-element-types.md diff --git a/.changeset/ios-webdriver-element-types.md b/.changeset/ios-webdriver-element-types.md new file mode 100644 index 000000000..18e659f7b --- /dev/null +++ b/.changeset/ios-webdriver-element-types.md @@ -0,0 +1,5 @@ +--- +'@e2e-dev/mobile': patch +--- + +iOS devices driven through agent-device's WebDriver runtime (hosted Appium clouds such as TestMu AI, BrowserStack, and AWS Device Farm) resolve roles like a local simulator does. That runtime reports XCUITest class names (`XCUIElementTypeButton`, `XCUIElementTypeStaticText`) where the native runner reports `Button` and `StaticText`, and the engine read them as unknown types, so `getByRole('button', 'CPU Load')` matched nothing and `observe` listed the node as `xcuielement-type-button`. The `XCUIElementType` prefix is now dropped before the type is mapped, so a button is a `button`, a tab bar's buttons are `tab`s, and the navigation bar titles the screen. diff --git a/packages/mobile/src/nodes.ts b/packages/mobile/src/nodes.ts index ea9e6212c..dcb5f350d 100644 --- a/packages/mobile/src/nodes.ts +++ b/packages/mobile/src/nodes.ts @@ -266,9 +266,10 @@ const ANDROID_TITLE_IDS = [':id/collapsing_toolbar', ':id/action_bar', ':id/tool /** * Platform element type of one raw node as a kebab-case token: XCTest sends - * `NavigationBar` and `StaticText`, Android sends `android.widget.TextView`; - * both read as one vocabulary here, the Android package prefix dropped. - * `role` is the fallback some platforms send instead. + * `NavigationBar` and `StaticText` (`XCUIElementTypeNavigationBar` over + * WebDriver), Android sends `android.widget.TextView`; all read as one + * vocabulary here, the class and package prefixes dropped. `role` is the + * fallback some platforms send instead. */ function kindOf(raw: RawNode): string { return normalizeKind(raw.type ?? raw.role ?? ''); @@ -279,9 +280,9 @@ function isAndroidClass(type: string | undefined): boolean { return type !== undefined && type.includes('.'); } -/** One element-type spelling for `NavigationBar`, `navigation-bar`, and `android.widget.NavigationBar` alike. */ +/** One element-type spelling for `NavigationBar`, `XCUIElementTypeNavigationBar`, `navigation-bar`, and `android.widget.NavigationBar` alike. */ export function normalizeKind(type: string): string { - const simple = type.slice(type.lastIndexOf('.') + 1); + const simple = type.slice(type.lastIndexOf('.') + 1).replace(/^XCUIElementType(?=[A-Z])/, ''); return simple .replaceAll(/([a-z0-9])([A-Z])/g, '$1-$2') .replaceAll(/[\s_]+/g, '-') diff --git a/packages/mobile/tests/unit/nodes.test.ts b/packages/mobile/tests/unit/nodes.test.ts index 45089d0a6..24f5f76fb 100644 --- a/packages/mobile/tests/unit/nodes.test.ts +++ b/packages/mobile/tests/unit/nodes.test.ts @@ -214,6 +214,36 @@ describe('snapshot projection', () => { expect(screenTitle(projected)).toBe('Settings'); }); + it('reads the XCUIElementType class names a WebDriver session reports as the native XCTest types', () => { + const native: RawNode[] = [ + { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'Profiler', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { ref: 'e2', index: 1, parentIndex: 0, depth: 1, type: 'NavigationBar', identifier: 'Profiler' }, + { ref: 'e3', index: 2, parentIndex: 1, depth: 2, type: 'StaticText', label: 'Profiler' }, + { ref: 'e4', index: 3, parentIndex: 0, depth: 1, type: 'Button', label: 'CPU Load' }, + { ref: 'e5', index: 4, parentIndex: 0, depth: 1, type: 'TextView', label: 'Notes', value: 'idle' }, + { ref: 'e6', index: 5, parentIndex: 0, depth: 1, type: 'TabBar' }, + { ref: 'e7', index: 6, parentIndex: 5, depth: 2, type: 'Button', label: 'Home', selected: true }, + { ref: 'e8', index: 7, parentIndex: 5, depth: 2, type: 'Button', label: 'Settings' }, + ]; + const webDriver = native.map((node) => ({ ...node, type: `XCUIElementType${node.type}` })); + const kindsAndRoles = (nodes: readonly RawNode[]) => project(nodes).index.map((entry) => [entry.kind, entry.node.role]); + expect(kindsAndRoles(webDriver)).toEqual(kindsAndRoles(native)); + const projected = project(webDriver); + expect(projected.index.map((entry) => entry.kind)).toEqual([ + 'application', + 'navigation-bar', + 'static-text', + 'button', + 'text-view', + 'tab-bar', + 'button', + 'button', + ]); + expect(projected.index.map((entry) => entry.node.role)).toEqual(['application', 'navigation', 'text', 'button', 'textbox', 'tablist', 'tab', 'tab']); + expect(projected.viewport).toEqual({ width: 390, height: 844 }); + expect(screenTitle(projected)).toBe('Profiler'); + }); + it('maps iOS composite widgets onto the vocabulary roles a browser reports for them', () => { const projected = project([ { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'Shop' },