From a2ecbaae7f16d23f95b281d522a84245a9237ff2 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 1/6] refactor(provider-webdriver): share hub upload and app-reference helpers BrowserStack's app upload, install adapter, --provider-app resolution, session-details URL artifacts, and orientation check were written for one vendor. A second hosted Appium hub needs the same mechanics with a different form field, reference scheme, and response shape, so move them into shared helpers and have BrowserStack use them: - webdriver-utils.ts: postHubAppUpload, createHubUploadApp, resolveHubAppReference, appFileUploadForm, asRecord, readProviderJsonBody, and requireProviderDeviceOrientation. - artifact-results.ts: urlArtifactFromDetails. - browserstack.ts: resolveBrowserStackAppReference, moved out of provider-definitions.ts. Two small BrowserStack behaviour changes come with the shared code: - An upload response that is not JSON (a gateway error page, an empty body) now fails with a typed COMMAND_FAILED that carries the HTTP status, instead of a raw JSON SyntaxError. - The http(s) scheme of a --provider-app URL is matched case-insensitively, so HTTPS://... is passed through to the hub rather than being treated as a local path. Co-Authored-By: Claude Opus 5.5 --- .../src/artifact-results.ts | 14 ++ .../src/browserstack-device-features.ts | 16 +- .../src/browserstack.test.ts | 57 +++++++- .../provider-webdriver/src/browserstack.ts | 101 ++++++------- .../src/provider-definitions.ts | 64 ++------ .../src/webdriver-utils.test.ts | 121 +++++++++++++++- .../provider-webdriver/src/webdriver-utils.ts | 137 +++++++++++++++++- 7 files changed, 386 insertions(+), 124 deletions(-) diff --git a/packages/provider-webdriver/src/artifact-results.ts b/packages/provider-webdriver/src/artifact-results.ts index bf02c7c77e..de29b4ab3a 100644 --- a/packages/provider-webdriver/src/artifact-results.ts +++ b/packages/provider-webdriver/src/artifact-results.ts @@ -28,3 +28,17 @@ export function unavailableCloudArtifactsResult(options: { message: options.error instanceof Error ? options.error.message : String(options.error), }; } + +/** A ready URL artifact read off a provider's session-details record, or nothing when the field is absent. */ +export function urlArtifactFromDetails( + provider: string, + providerSessionId: string, + details: Record, + field: string, + kind: CloudArtifact['kind'], + name: string, +): CloudArtifact | undefined { + const url = details[field]; + if (typeof url !== 'string' || url.length === 0) return undefined; + return { provider, providerSessionId, kind, name, url, availability: 'ready' }; +} diff --git a/packages/provider-webdriver/src/browserstack-device-features.ts b/packages/provider-webdriver/src/browserstack-device-features.ts index 38669957ac..3ea041add7 100644 --- a/packages/provider-webdriver/src/browserstack-device-features.ts +++ b/packages/provider-webdriver/src/browserstack-device-features.ts @@ -3,6 +3,7 @@ import { type CloudProviderProfileFields, } from '@agent-device/contracts/remote'; import { AppError } from '@agent-device/kernel/errors'; +import { requireProviderDeviceOrientation } from './webdriver-utils.ts'; import type { CloudWebDriverPlatform } from './runtime.ts'; /** @@ -199,26 +200,13 @@ function assignStringField( value: string, ): void { if (spec.field === 'providerDeviceOrientation') { - fields.providerDeviceOrientation = requireDeviceOrientation(spec, value); + fields.providerDeviceOrientation = requireProviderDeviceOrientation(spec, value); return; } if (spec.field === 'providerNoResignApp') return; fields[spec.field] = value; } -function requireDeviceOrientation( - spec: BrowserStackDeviceFeatureSpec, - value: string, -): (typeof PROVIDER_DEVICE_ORIENTATIONS)[number] { - const match = PROVIDER_DEVICE_ORIENTATIONS.find((orientation) => orientation === value); - if (match) return match; - throw new AppError('INVALID_ARGS', `Invalid ${spec.flag} value: ${value}.`, { - hint: `Use ${PROVIDER_DEVICE_ORIENTATIONS.join('|')}.`, - flag: spec.flag, - capability: spec.capability, - }); -} - function requireSupportedPlatform( spec: BrowserStackDeviceFeatureSpec, platform: CloudWebDriverPlatform, diff --git a/packages/provider-webdriver/src/browserstack.test.ts b/packages/provider-webdriver/src/browserstack.test.ts index 125bc8ea8c..61cf7bdbb6 100644 --- a/packages/provider-webdriver/src/browserstack.test.ts +++ b/packages/provider-webdriver/src/browserstack.test.ts @@ -3,7 +3,8 @@ import { promises as fs } from 'node:fs'; import path from 'node:path'; import { afterEach, test } from 'vitest'; -import { uploadBrowserStackApp } from './browserstack.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { resolveBrowserStackAppReference, uploadBrowserStackApp } from './browserstack.ts'; import { mkdtempForTest } from './tmp-dir.fixtures.ts'; const realFetch = globalThis.fetch; @@ -46,3 +47,57 @@ test('BrowserStack upload aborts while the provider request is in flight', async await fs.rm(tempDir, { recursive: true, force: true }); } }); + +const upload = { clientVersion: '0.0.0-test', username: 'user', accessKey: 'key' }; + +test('BrowserStack upload sends the file field and fails typed on a gateway error page', async () => { + const tempDir = await mkdtempForTest('agent-device-browserstack-upload-error-'); + const appPath = path.join(tempDir, 'App.apk'); + try { + await fs.writeFile(appPath, 'placeholder'); + globalThis.fetch = async (_input, init) => { + assert.ok(init?.body instanceof FormData); + assert.ok(init.body.get('file') instanceof Blob); + return new Response('502 Bad Gateway', { status: 502 }); + }; + await assert.rejects(uploadBrowserStackApp(appPath, upload), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'BrowserStack app upload failed.'); + assert.equal(error.details?.status, 502); + return true; + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('BrowserStack passes bs:// ids and URLs to the hub and uploads only local paths', async () => { + const tempDir = await mkdtempForTest('agent-device-browserstack-resolve-'); + try { + await fs.writeFile(path.join(tempDir, 'App.apk'), 'placeholder'); + const fetched: string[] = []; + globalThis.fetch = async (input) => { + fetched.push(String(input)); + return new Response(JSON.stringify({ app_url: 'bs://uploaded' }), { status: 200 }); + }; + const resolve = async (app: string) => + await resolveBrowserStackAppReference(app, { ...upload, cwd: tempDir }); + + assert.equal(await resolve('bs://preuploaded'), 'bs://preuploaded'); + assert.equal(await resolve('https://builds.example/App.apk'), 'https://builds.example/App.apk'); + assert.equal(fetched.length, 0); + assert.equal(await resolve('App.apk'), 'bs://uploaded'); + assert.deepEqual(fetched, ['https://api-cloud.browserstack.com/app-automate/upload']); + await assert.rejects(resolve('missing.apk'), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal( + error.message, + 'BrowserStack --provider-app must be a bs:// app id, URL, or existing local app path.', + ); + return true; + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index 6c8d10fd4b..74811f78e6 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -1,12 +1,18 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; import { AppError } from '@agent-device/kernel/errors'; import { agentDeviceRequestHeaders } from './request-headers.ts'; -import { cloudArtifactsReadyOrPending } from './artifact-results.ts'; -import { basicAuthHeader, trimTrailingSlash } from './webdriver-utils.ts'; +import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +import { + appFileUploadForm, + asRecord, + basicAuthHeader, + createHubUploadApp, + postHubAppUpload, + resolveHubAppReference, + trimTrailingSlash, +} from './webdriver-utils.ts'; export const BROWSERSTACK_APP_AUTOMATE_ENDPOINT = 'https://hub-cloud.browserstack.com/wd/hub/'; export const BROWSERSTACK_APP_UPLOAD_ENDPOINT = @@ -76,41 +82,41 @@ export async function uploadBrowserStackApp( signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - const file = await fs.readFile(appPath); - const form = new FormData(); - form.set('file', new Blob([file]), path.basename(appPath)); - const response = await fetch(options.endpoint ?? BROWSERSTACK_APP_UPLOAD_ENDPOINT, { - method: 'POST', - headers: { - ...agentDeviceRequestHeaders(options.clientVersion), - Authorization: basicAuthHeader(options), + return await postHubAppUpload( + await appFileUploadForm(appPath, 'file'), + { + service: 'BrowserStack', + endpoint: options.endpoint ?? BROWSERSTACK_APP_UPLOAD_ENDPOINT, + clientVersion: options.clientVersion, + auth: options, + readAppReference: readBrowserStackAppUrl, }, - body: form, signal, - }); - const json = (await response.json()) as unknown; - const appUrl = readBrowserStackAppUrl(json); - if (!response.ok || !appUrl) { - throw new AppError('COMMAND_FAILED', 'BrowserStack app upload failed.', { - status: response.status, - response: json, - }); - } - return appUrl; + ); } export function createBrowserStackUploadApp( options: Required, ): CloudWebDriverUploadApp { - return async ({ appPath, options: installOptions, signal }) => { - const appReference = await uploadBrowserStackApp(appPath, options, signal); - return { - appReference, - bundleId: installOptions?.appIdentifierHint, - packageName: installOptions?.packageNameHint, - launchTarget: installOptions?.appIdentifierHint ?? installOptions?.packageNameHint, - }; - }; + return createHubUploadApp( + async (appPath, signal) => await uploadBrowserStackApp(appPath, options, signal), + ); +} + +/** The hub fetches a public URL itself, so only a local path is uploaded. */ +export async function resolveBrowserStackAppReference( + app: string, + options: BrowserStackUploadOptions & { cwd?: string; signal?: AbortSignal }, +): Promise { + return await resolveHubAppReference({ + service: 'BrowserStack', + app, + cwd: options.cwd, + referenceScheme: 'bs://', + referenceLabel: 'a bs:// app id', + uploadFile: async (appPath, signal) => await uploadBrowserStackApp(appPath, options, signal), + signal: options.signal, + }); } /** @@ -138,17 +144,11 @@ export function buildBrowserStackCapabilities( buildName: options.buildName, sessionName: options.sessionName, ...(options.deviceFeatures ?? {}), - ...asRecord(configuredBstackOptions), + ...(asRecord(configuredBstackOptions) ?? {}), }, }; } -function asRecord(value: unknown): Record { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : {}; -} - async function fetchBrowserStackSessionDetails( sessionId: string, options: BrowserStackSessionDetailsOptions, @@ -179,7 +179,7 @@ function mapBrowserStackArtifacts( details: Record, ): CloudArtifact[] { return [ - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -187,7 +187,7 @@ function mapBrowserStackArtifacts( 'video', 'Session video', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -195,7 +195,7 @@ function mapBrowserStackArtifacts( 'appium-log', 'Appium logs', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -203,7 +203,7 @@ function mapBrowserStackArtifacts( 'device-log', 'Device logs', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -211,7 +211,7 @@ function mapBrowserStackArtifacts( 'provider-session', 'BrowserStack dashboard', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -222,19 +222,6 @@ function mapBrowserStackArtifacts( ].filter((artifact): artifact is CloudArtifact => artifact !== undefined); } -function browserStackUrlArtifact( - provider: string, - providerSessionId: string, - details: Record, - field: string, - kind: CloudArtifact['kind'], - name: string, -): CloudArtifact | undefined { - const url = details[field]; - if (typeof url !== 'string' || url.length === 0) return undefined; - return { provider, providerSessionId, kind, name, url, availability: 'ready' }; -} - function readBrowserStackAppUrl(value: unknown): string | undefined { if (!value || typeof value !== 'object') return undefined; const appUrl = (value as { app_url?: unknown }).app_url; diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index b2de85b689..713fd7d8ce 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -1,5 +1,3 @@ -import fs from 'node:fs'; -import path from 'node:path'; import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -17,7 +15,7 @@ import { buildBrowserStackCapabilities, createBrowserStackUploadApp, listBrowserStackCloudArtifacts, - uploadBrowserStackApp, + resolveBrowserStackAppReference, } from './browserstack.ts'; import { buildBrowserStackDeviceFeatureCapabilities, @@ -108,22 +106,24 @@ export function createCloudWebDriverProviderDefinitions( 'providerOsVersion', 'BrowserStack requires --provider-os-version .', ); - const app = await resolveBrowserStackAppReference({ - clientVersion: dependencies.clientVersion, - app: requireFlag( + const app = await resolveBrowserStackAppReference( + requireFlag( request, 'providerApp', 'BrowserStack requires --provider-app .', ), - cwd: request.cwd, - username, - accessKey, - uploadEndpoint: env.BROWSERSTACK_APP_UPLOAD_ENDPOINT, - // A local IPA/APK upload can run long (130 MB is routine); an - // upload is not a billed resource, so the request's cancellation - // may simply abort it — unlike the session creation that follows. - signal: request.signal, - }); + { + clientVersion: dependencies.clientVersion, + username, + accessKey, + endpoint: env.BROWSERSTACK_APP_UPLOAD_ENDPOINT, + cwd: request.cwd, + // A local IPA/APK upload can run long (130 MB is routine); an + // upload is not a billed resource, so the request's cancellation + // may simply abort it — unlike the session creation that follows. + signal: request.signal, + }, + ); return { ...base, platform, @@ -249,40 +249,6 @@ export function createCloudWebDriverProviderDefinitions( ]; } -async function resolveBrowserStackAppReference(options: { - clientVersion: string; - app: string; - cwd?: string; - username: string; - accessKey: string; - uploadEndpoint?: string; - signal?: AbortSignal; -}): Promise { - if (isProviderAppReference(options.app)) return options.app; - const appPath = path.resolve(options.cwd ?? process.cwd(), options.app); - if (!fs.existsSync(appPath)) { - throw new AppError( - 'INVALID_ARGS', - 'BrowserStack --provider-app must be a bs:// app id, URL, or existing local app path.', - { providerApp: options.app }, - ); - } - return await uploadBrowserStackApp( - appPath, - { - clientVersion: options.clientVersion, - username: options.username, - accessKey: options.accessKey, - endpoint: options.uploadEndpoint, - }, - options.signal, - ); -} - -function isProviderAppReference(value: string): boolean { - return value.startsWith('bs://') || /^https?:\/\//.test(value); -} - function requireRequest( req: LeaseLifecycleContext | undefined, providerLabel: string, diff --git a/packages/provider-webdriver/src/webdriver-utils.test.ts b/packages/provider-webdriver/src/webdriver-utils.test.ts index 58d326c241..122aafaf80 100644 --- a/packages/provider-webdriver/src/webdriver-utils.test.ts +++ b/packages/provider-webdriver/src/webdriver-utils.test.ts @@ -1,6 +1,23 @@ import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { trimLeadingSlash, trimTrailingSlash } from './webdriver-utils.ts'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + asRecord, + createHubUploadApp, + postHubAppUpload, + resolveHubAppReference, + trimLeadingSlash, + trimTrailingSlash, +} from './webdriver-utils.ts'; +import { mkdtempForTest } from './tmp-dir.fixtures.ts'; + +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); test('slash trimming utilities handle slash-heavy strings without regular expressions', () => { const slashRun = '/'.repeat(10_000); @@ -15,3 +32,103 @@ test('slash trimming utilities handle slash-heavy strings without regular expres assert.equal(trimLeadingSlash(slashRun), ''); assert.equal(trimTrailingSlash(slashRun), ''); }); + +test('asRecord admits plain objects only', () => { + assert.deepEqual(asRecord({ a: 1 }), { a: 1 }); + assert.equal(asRecord([]), undefined); + assert.equal(asRecord(null), undefined); + assert.equal(asRecord('x'), undefined); +}); + +const hub = { + service: 'Hub', + endpoint: 'https://upload.example.test/app', + clientVersion: '0.0.0-test', + auth: { username: 'user', accessKey: 'key' }, + readAppReference: (body: unknown) => asRecord(body)?.ref as string | undefined, +}; + +test('the hub upload helper posts with credentials and returns the vendor reference', async () => { + const form = new FormData(); + globalThis.fetch = async (input, init) => { + assert.equal(String(input), hub.endpoint); + assert.equal(init?.method, 'POST'); + assert.equal(init?.body, form); + const headers = init?.headers as Record; + assert.equal(headers.Authorization, `Basic ${Buffer.from('user:key').toString('base64')}`); + assert.equal(headers['x-agent-device-version'], '0.0.0-test'); + return new Response(JSON.stringify({ ref: 'hub://APP1' }), { status: 200 }); + }; + assert.equal(await postHubAppUpload(form, hub), 'hub://APP1'); +}); + +test('the hub upload helper fails typed with the status on an error page or a missing reference', async () => { + for (const response of [ + new Response('502 Bad Gateway', { status: 502 }), + new Response(JSON.stringify({ message: 'ok' }), { status: 200 }), + ]) { + globalThis.fetch = async () => response; + await assert.rejects(postHubAppUpload(new FormData(), hub), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'Hub app upload failed.'); + assert.equal(error.details?.status, response.status); + return true; + }); + } +}); + +test('the hub install adapter uploads the build and launches the hinted app', async () => { + const upload = vi.fn(async () => 'hub://APP2'); + const signal = new AbortController().signal; + const result = await createHubUploadApp(upload)({ + appPath: '/builds/App.ipa', + options: { appIdentifierHint: 'com.example.app' }, + signal, + }); + assert.deepEqual(upload.mock.calls, [['/builds/App.ipa', signal]]); + assert.deepEqual(result, { + appReference: 'hub://APP2', + bundleId: 'com.example.app', + packageName: undefined, + launchTarget: 'com.example.app', + }); +}); + +test('the hub app resolver passes references through, uploads local files, and routes URLs per hub', async () => { + const tempDir = await mkdtempForTest('agent-device-hub-resolve-'); + try { + await fs.writeFile(path.join(tempDir, 'App.apk'), 'placeholder'); + const uploadFile = vi.fn(async (appPath: string) => `hub://${path.basename(appPath)}`); + const resolve = (app: string, uploadUrl?: (url: string) => Promise) => + resolveHubAppReference({ + service: 'Hub', + app, + cwd: tempDir, + referenceScheme: 'hub://', + referenceLabel: 'a hub:// app id', + uploadFile, + uploadUrl, + }); + + assert.equal(await resolve('hub://APP3'), 'hub://APP3'); + assert.equal(await resolve('https://builds.example/App.apk'), 'https://builds.example/App.apk'); + assert.equal( + await resolve('https://builds.example/App.apk', async (url) => `fetched:${url}`), + 'fetched:https://builds.example/App.apk', + ); + assert.equal(await resolve('App.apk'), 'hub://App.apk'); + assert.deepEqual(uploadFile.mock.calls, [[path.join(tempDir, 'App.apk'), undefined]]); + await assert.rejects(resolve('missing.apk'), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal( + error.message, + 'Hub --provider-app must be a hub:// app id, URL, or existing local app path.', + ); + return true; + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index 5af69a9f4c..a6e833c8c9 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -1,5 +1,17 @@ -import type { DeviceLease } from '@agent-device/contracts/device'; +import fs from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { + DeviceLease, + ProviderDeviceInstallOptions, + ProviderDeviceInstallResult, +} from '@agent-device/contracts/device'; +import { + PROVIDER_DEVICE_ORIENTATIONS, + type ProviderDeviceOrientation, +} from '@agent-device/contracts/remote'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import { agentDeviceRequestHeaders } from './request-headers.ts'; export type LeaseValue = T | ((lease: DeviceLease) => T); @@ -50,3 +62,126 @@ export function withTrailingSlash(url: URL): URL { copy.pathname = `${copy.pathname}/`; return copy; } + +export function asRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +type HubCredentials = { username: string; accessKey: string }; + +/** A multipart form carrying the local app file under the hub's field name. */ +export async function appFileUploadForm(appPath: string, fileField: string): Promise { + const form = new FormData(); + form.set(fileField, new Blob([await readFile(appPath)]), path.basename(appPath)); + return form; +} + +/** + * POSTs an app upload to a hosted hub and returns the hub's app reference. A non-2xx answer, a + * body that is not JSON, or one without a reference is `COMMAND_FAILED` with the HTTP status. + */ +export async function postHubAppUpload( + form: FormData, + options: { + service: string; + endpoint: string | URL; + clientVersion: string; + auth: HubCredentials; + readAppReference: (body: unknown) => string | undefined; + }, + signal?: AbortSignal, +): Promise { + const response = await fetch(options.endpoint, { + method: 'POST', + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + Authorization: basicAuthHeader(options.auth), + }, + body: form, + signal, + }); + const json = await readProviderJsonBody(response); + const appReference = options.readAppReference(json); + if (!response.ok || !appReference) { + throw new AppError('COMMAND_FAILED', `${options.service} app upload failed.`, { + status: response.status, + response: json, + }); + } + return appReference; +} + +/** The `install` adapter of a hosted hub: upload the local build, then launch the hinted app. */ +export function createHubUploadApp( + upload: (appPath: string, signal?: AbortSignal) => Promise, +): (params: { + appPath: string; + options?: ProviderDeviceInstallOptions; + signal?: AbortSignal; +}) => Promise { + return async ({ appPath, options, signal }) => ({ + appReference: await upload(appPath, signal), + bundleId: options?.appIdentifierHint, + packageName: options?.packageNameHint, + launchTarget: options?.appIdentifierHint ?? options?.packageNameHint, + }); +} + +/** + * Turns `--provider-app` into a reference the hub accepts: its own reference scheme passes + * through, a public URL passes through unless the hub only takes its own references (then + * `uploadUrl` has the hub fetch it), and anything else must be a local file to upload. + */ +export async function resolveHubAppReference(options: { + service: string; + app: string; + cwd?: string; + referenceScheme: string; + /** How the scheme reads in the error message, e.g. `a bs:// app id`. */ + referenceLabel: string; + uploadFile: (appPath: string, signal?: AbortSignal) => Promise; + uploadUrl?: (url: string, signal?: AbortSignal) => Promise; + signal?: AbortSignal; +}): Promise { + const { app } = options; + if (app.startsWith(options.referenceScheme)) return app; + if (/^https?:\/\//i.test(app)) { + return options.uploadUrl ? await options.uploadUrl(app, options.signal) : app; + } + const appPath = path.resolve(options.cwd ?? process.cwd(), app); + if (!fs.existsSync(appPath)) { + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app must be ${options.referenceLabel}, URL, or existing local app path.`, + { providerApp: app }, + ); + } + return await options.uploadFile(appPath, options.signal); +} + +/** A provider response body parsed as JSON, or `undefined` when it is empty or not JSON (a gateway error page). */ +async function readProviderJsonBody(response: Response): Promise { + const text = await response.text(); + if (text.length === 0) return undefined; + try { + return JSON.parse(text) as unknown; + } catch { + return undefined; + } +} + +/** Validates a device-orientation flag against the shared enum before it reaches a hub that would ignore it. */ +export function requireProviderDeviceOrientation( + spec: { flag: string; capability: string }, + value: string, +): ProviderDeviceOrientation { + const match = PROVIDER_DEVICE_ORIENTATIONS.find((orientation) => orientation === value); + if (match) return match; + throw new AppError('INVALID_ARGS', `Invalid ${spec.flag} value: ${value}.`, { + hint: `Use ${PROVIDER_DEVICE_ORIENTATIONS.join('|')}.`, + flag: spec.flag, + capability: spec.capability, + }); +} From 364bfe7d596bb77002e4edc34faf266ecc3cb18f Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 2/6] fix(provider-webdriver): bound and type session-details lookups The BrowserStack session-details lookup behind `artifacts` had no deadline, so a stalled API call could hang the command indefinitely. A transport failure or a body that was not JSON surfaced as an untyped fetch or SyntaxError, and a JSON array passed the object check and was read as session details. Add fetchProviderSessionDetails to webdriver-utils.ts and use it for BrowserStack. It sends basic auth with a 15 second deadline and reports every failure as COMMAND_FAILED: a timeout or network error with a retry hint and the original error as its cause, and a non-2xx answer or a body that is not a JSON object with the HTTP status and the parsed response. Connection verification gets the same treatment through fetchProviderVerificationJson, which BrowserStack now uses in place of its private fetch. Behaviour is unchanged: 401/403 is UNAUTHORIZED with a credential hint, any other HTTP failure points at the provider's service status, and a transport failure points at network access. A new test pins the two non-credential hints. sameOsVersion moves alongside it. Co-Authored-By: Claude Opus 5.5 --- .../browserstack-connection-verification.ts | 54 ++------- .../src/browserstack.test.ts | 36 +++++- .../provider-webdriver/src/browserstack.ts | 23 +--- .../src/connection-verification.test.ts | 29 +++++ .../provider-webdriver/src/webdriver-utils.ts | 103 ++++++++++++++++++ 5 files changed, 183 insertions(+), 62 deletions(-) diff --git a/packages/provider-webdriver/src/browserstack-connection-verification.ts b/packages/provider-webdriver/src/browserstack-connection-verification.ts index efdfb4e3c8..56d36366cc 100644 --- a/packages/provider-webdriver/src/browserstack-connection-verification.ts +++ b/packages/provider-webdriver/src/browserstack-connection-verification.ts @@ -1,7 +1,6 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import { agentDeviceRequestHeaders } from './request-headers.ts'; -import { basicAuthHeader } from './webdriver-utils.ts'; +import { asRecord, fetchProviderVerificationJson, sameOsVersion } from './webdriver-utils.ts'; import type { CloudWebDriverConnectionVerification, CloudWebDriverConnectionVerificationOptions, @@ -105,42 +104,15 @@ async function fetchBrowserStackJson( auth: { username: string; accessKey: string }, clientVersion: string, ): Promise { - try { - const response = await fetch(endpoint, { - headers: { - ...agentDeviceRequestHeaders(clientVersion), - Authorization: basicAuthHeader(auth), - }, - signal: AbortSignal.timeout(15_000), - }); - if (!response.ok) { - const unauthorized = response.status === 401 || response.status === 403; - throw new AppError( - unauthorized ? 'UNAUTHORIZED' : 'COMMAND_FAILED', - 'BrowserStack rejected connection verification.', - { - status: response.status, - hint: unauthorized - ? 'Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.' - : 'Retry connect or check the BrowserStack service status.', - }, - ); - } - return (await response.json()) as unknown; - } catch (error) { - if (error instanceof AppError) throw error; - throw new AppError( - 'COMMAND_FAILED', - 'BrowserStack connection verification failed.', - { hint: 'Check network access to api-cloud.browserstack.com and retry connect.' }, - error, - ); - } -} - -function sameOsVersion(left: string, right: string): boolean { - const normalize = (value: string) => value.replace(/(?:\.0)+$/, ''); - return normalize(left) === normalize(right); + return await fetchProviderVerificationJson(endpoint, { + clientVersion, + auth, + hints: { + service: 'BrowserStack', + unauthorizedHint: 'Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.', + networkHint: 'Check network access to api-cloud.browserstack.com and retry connect.', + }, + }); } function readBrowserStackDevices( @@ -184,9 +156,3 @@ function readBrowserStackApps( ]; }); } - -function asRecord(value: unknown): Record | undefined { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : undefined; -} diff --git a/packages/provider-webdriver/src/browserstack.test.ts b/packages/provider-webdriver/src/browserstack.test.ts index 61cf7bdbb6..1d8ca76b1a 100644 --- a/packages/provider-webdriver/src/browserstack.test.ts +++ b/packages/provider-webdriver/src/browserstack.test.ts @@ -4,7 +4,11 @@ import { promises as fs } from 'node:fs'; import path from 'node:path'; import { afterEach, test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { resolveBrowserStackAppReference, uploadBrowserStackApp } from './browserstack.ts'; +import { + listBrowserStackCloudArtifacts, + resolveBrowserStackAppReference, + uploadBrowserStackApp, +} from './browserstack.ts'; import { mkdtempForTest } from './tmp-dir.fixtures.ts'; const realFetch = globalThis.fetch; @@ -101,3 +105,33 @@ test('BrowserStack passes bs:// ids and URLs to the hub and uploads only local p await fs.rm(tempDir, { recursive: true, force: true }); } }); + +test('BrowserStack session details lookup has a deadline and fails typed', async () => { + const lookup = async () => + await listBrowserStackCloudArtifacts('browserstack', 'SESSION1', upload); + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + const transportFailures: unknown[] = [timeout, new TypeError('fetch failed')]; + for (const failure of transportFailures) { + globalThis.fetch = async (_input, init) => { + assert.ok(init?.signal instanceof AbortSignal); + throw failure; + }; + await assert.rejects(lookup(), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'BrowserStack session details lookup failed.'); + assert.equal(error.cause, failure); + return true; + }); + } + + for (const body of ['gateway', '[]']) { + globalThis.fetch = async () => new Response(body, { status: 200 }); + await assert.rejects(lookup(), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.status, 200); + return true; + }); + } +}); diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index 74811f78e6..a5954eda00 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -1,14 +1,12 @@ import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; -import { AppError } from '@agent-device/kernel/errors'; -import { agentDeviceRequestHeaders } from './request-headers.ts'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; import { appFileUploadForm, asRecord, - basicAuthHeader, createHubUploadApp, + fetchProviderSessionDetails, postHubAppUpload, resolveHubAppReference, trimTrailingSlash, @@ -156,21 +154,12 @@ async function fetchBrowserStackSessionDetails( const endpoint = new URL( `${trimTrailingSlash(String(options.endpoint ?? BROWSERSTACK_SESSION_DETAILS_ENDPOINT))}/${sessionId}.json`, ); - const response = await fetch(endpoint, { - headers: { - ...agentDeviceRequestHeaders(options.clientVersion), - Authorization: basicAuthHeader(options), - }, + const json = await fetchProviderSessionDetails(endpoint, { + clientVersion: options.clientVersion, + auth: options, + service: 'BrowserStack', }); - const json = (await response.json()) as unknown; - if (!response.ok || !json || typeof json !== 'object') { - throw new AppError('COMMAND_FAILED', 'BrowserStack session details lookup failed.', { - status: response.status, - response: json, - }); - } - const details = (json as { automation_session?: unknown }).automation_session ?? json; - return details && typeof details === 'object' ? (details as Record) : {}; + return asRecord(json.automation_session) ?? json; } function mapBrowserStackArtifacts( diff --git a/packages/provider-webdriver/src/connection-verification.test.ts b/packages/provider-webdriver/src/connection-verification.test.ts index e9e7f9c726..c93e40109e 100644 --- a/packages/provider-webdriver/src/connection-verification.test.ts +++ b/packages/provider-webdriver/src/connection-verification.test.ts @@ -80,6 +80,35 @@ test('BrowserStack classifies rejected credentials without exposing them', async }); }); +test('BrowserStack points HTTP failures at its service status and transport failures at the network', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse({}, 503)), + ); + await assert.rejects(createProvider().verifyConnection(browserStackOptions), (error: unknown) => { + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.equal( + (error as { details?: { hint?: string } }).details?.hint, + 'Retry connect or check the BrowserStack service status.', + ); + return true; + }); + + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new TypeError('fetch failed'); + }), + ); + await assert.rejects(createProvider().verifyConnection(browserStackOptions), (error: unknown) => { + assert.equal( + (error as { details?: { hint?: string } }).details?.hint, + 'Check network access to api-cloud.browserstack.com and retry connect.', + ); + return true; + }); +}); + test('BrowserStack defers a bs app reference outside the recent upload window', async () => { vi.stubGlobal( 'fetch', diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index a6e833c8c9..4b750d5439 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -161,6 +161,103 @@ export async function resolveHubAppReference(options: { return await options.uploadFile(appPath, options.signal); } +const PROVIDER_API_TIMEOUT_MS = 15_000; + +/** The provider rejected or could not answer a verification call; typed so callers never sniff text. */ +export type ProviderJsonFailureHints = { + service: string; + unauthorizedHint: string; + networkHint: string; +}; + +/** + * Fetches JSON from a hosted provider's API during connection verification. A 401/403 is + * `UNAUTHORIZED` with a credential hint, any other non-2xx is `COMMAND_FAILED`, and a transport + * failure is wrapped so its cause survives without leaking the credentials. + */ +export async function fetchProviderVerificationJson( + endpoint: string | URL, + options: { + clientVersion: string; + auth?: { username: string; accessKey: string }; + hints: ProviderJsonFailureHints; + }, +): Promise { + const { service, unauthorizedHint, networkHint } = options.hints; + try { + const response = await fetch(endpoint, { + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + ...(options.auth ? { Authorization: basicAuthHeader(options.auth) } : {}), + }, + signal: AbortSignal.timeout(PROVIDER_API_TIMEOUT_MS), + }); + if (!response.ok) { + const unauthorized = response.status === 401 || response.status === 403; + throw new AppError( + unauthorized ? 'UNAUTHORIZED' : 'COMMAND_FAILED', + `${service} rejected connection verification.`, + { + status: response.status, + hint: unauthorized + ? unauthorizedHint + : `Retry connect or check the ${service} service status.`, + }, + ); + } + return (await response.json()) as unknown; + } catch (error) { + if (error instanceof AppError) throw error; + throw new AppError( + 'COMMAND_FAILED', + `${service} connection verification failed.`, + { hint: networkHint }, + error, + ); + } +} + +/** + * Fetches a provider's session-details JSON with basic auth under a deadline. A transport failure, + * a non-2xx answer, or a body that is not a JSON object is `COMMAND_FAILED`. + */ +export async function fetchProviderSessionDetails( + endpoint: string | URL, + options: { + clientVersion: string; + auth: { username: string; accessKey: string }; + service: string; + }, +): Promise> { + let response: Response; + let json: unknown; + try { + response = await fetch(endpoint, { + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + Authorization: basicAuthHeader(options.auth), + }, + signal: AbortSignal.timeout(PROVIDER_API_TIMEOUT_MS), + }); + json = await readProviderJsonBody(response); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `${options.service} session details lookup failed.`, + { hint: `Check network access to the ${options.service} API, then retry.` }, + error, + ); + } + const details = asRecord(json); + if (!response.ok || !details) { + throw new AppError('COMMAND_FAILED', `${options.service} session details lookup failed.`, { + status: response.status, + response: json, + }); + } + return details; +} + /** A provider response body parsed as JSON, or `undefined` when it is empty or not JSON (a gateway error page). */ async function readProviderJsonBody(response: Response): Promise { const text = await response.text(); @@ -172,6 +269,12 @@ async function readProviderJsonBody(response: Response): Promise { } } +/** `1.0` and `1` name the same OS release on BrowserStack's catalog. */ +export function sameOsVersion(left: string, right: string): boolean { + const normalize = (value: string) => value.replace(/(?:\.0)+$/, ''); + return normalize(left) === normalize(right); +} + /** Validates a device-orientation flag against the shared enum before it reaches a hub that would ignore it. */ export function requireProviderDeviceOrientation( spec: { flag: string; capability: string }, From 465a0a7a9fdcd79102227ddde2de5c5c425bc8f7 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 3/6] feat(provider-webdriver): add TestMu AI emulator and simulator provider Add `testmu`, a hosted WebDriver provider for TestMu AI (formerly LambdaTest) virtual devices: Android emulators and iOS simulators behind the TestMu AI Appium hub. It follows the BrowserStack shape: `connect testmu` verifies and saves a local profile, and `open` creates the hosted session. The service hostnames still carry the lambdatest.com domain. connect - Reads LT_USERNAME and LT_ACCESS_KEY, the variables TestMu AI SDKs use. - Checks the device and OS version against the public virtual-device catalog. The match is exact because the hub rejects `18` for a device listed as `18.0`; the error lists the versions the device offers. - Uses the authenticated app listing as the credential check, and looks an lt:// id up in the list for the session's runtime (`emulator` or `simulator`). An id that is not listed is reported as configured, since TestMu AI validates it at session creation. - Verifies against TESTMU_API_ENDPOINT when it is set, as the runtime does, and composes catalog and listing URLs with URL so a base or override that carries a query keeps it. - Never creates a session. Sessions - Standard Appium keys stay `appium:`-prefixed; everything vendor- specific goes in `lt:options`, merged per key with any configured `lt:options`. `isRealMobile: false` and `w3c: true` are applied last, so configuration cannot move the session to another device pool or off the W3C dialect agent-device speaks. - `appiumVersion` is sent only when --provider-appium-version pins one; otherwise TestMu AI starts its default server for the device. - Orientation, geo-location, timezone, Appium version, language, and locale map onto `lt:options` through a table. The BrowserStack-only network-profile, custom-network, and no-resign flags are refused by flag name at connect and at session preparation, instead of being silently dropped. Apps - --provider-app takes an lt:// id, an http(s) URL, or a local path. The hub only accepts lt:// references, so a URL is handed to the upload API to fetch (`storage=url`) and a local file is uploaded. - An unzipped iOS `.app` directory is rejected before any request, with a hint to zip it. - Only a well-formed lt:// reference or app id in the upload response counts as success. Artifacts - Read from the session-details API through the shared bounded lookup. A 404 reads as pending until TestMu AI publishes the details. `console_logs_url` is the device log on virtual devices. - Session video, Appium, network, and command logs, screenshots, and the dashboard link are returned once at least one artifact URL exists. TESTMU_WEBDRIVER_ENDPOINT, TESTMU_APP_UPLOAD_ENDPOINT, and TESTMU_API_ENDPOINT redirect the endpoints. The session, upload, verification, and device-feature modules are loaded with dynamic import, so the package entry's eager module graph does not grow. The CLI reaches the device-feature checks through a new `./testmu-device-features` subpath export. A .fallowrc entry covers the exports that are read only through the dynamic import. `connect testmu` is listed in the connect usage, the remote help topic, and the artifacts provider description. Co-authored-by: gautam-jain-dev Co-Authored-By: Claude Opus 5.5 --- .fallowrc.json | 5 + .../src/flag-definitions-connection.ts | 2 +- packages/provider-webdriver/package.json | 4 + .../src/connection-verification.test.ts | 248 ++++++++++ .../src/connection-verification.ts | 48 +- .../src/provider-definitions.ts | 125 +++++ packages/provider-webdriver/src/providers.ts | 1 + .../src/testmu-connection-verification.ts | 190 ++++++++ .../src/testmu-device-features.test.ts | 103 ++++ .../src/testmu-device-features.ts | 115 +++++ .../provider-webdriver/src/testmu.test.ts | 439 ++++++++++++++++++ packages/provider-webdriver/src/testmu.ts | 264 +++++++++++ .../provider-webdriver/src/webdriver-utils.ts | 2 +- scripts/layering/package-boundaries.test.ts | 1 + src/__tests__/cloud-connect-profile.test.ts | 47 +- src/__tests__/cloud-connect-testmu.test.ts | 192 ++++++++ src/cli/connection/cloud-webdriver-profile.ts | 70 ++- .../connection/connect-provider-adapters.ts | 23 + src/cli/connection/provider-policy.ts | 1 + src/commands/management/artifacts.ts | 4 +- src/commands/schema/cli-help-topics.test.ts | 12 +- src/commands/schema/cli-help.ts | 20 +- src/commands/schema/command-overrides.ts | 2 +- 23 files changed, 1864 insertions(+), 54 deletions(-) create mode 100644 packages/provider-webdriver/src/testmu-connection-verification.ts create mode 100644 packages/provider-webdriver/src/testmu-device-features.test.ts create mode 100644 packages/provider-webdriver/src/testmu-device-features.ts create mode 100644 packages/provider-webdriver/src/testmu.test.ts create mode 100644 packages/provider-webdriver/src/testmu.ts create mode 100644 src/__tests__/cloud-connect-testmu.test.ts diff --git a/.fallowrc.json b/.fallowrc.json index 659ad8259f..494c2f42f2 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -379,6 +379,11 @@ "file": "packages/provider-limrun/src/index.ts", "exports": ["LimrunIosCommandExecution"] }, + { + "comment": "TestMu session preparation reads these off the lazy loadTestMuDeviceFeatures() import in packages/provider-webdriver/src/provider-definitions.ts, which keeps the package entry's eager closure unchanged; Fallow cannot connect the dynamic member reads.", + "file": "packages/provider-webdriver/src/testmu-device-features.ts", + "exports": ["buildTestMuDeviceFeatureCapabilities", "readTestMuDeviceFeatureFields"] + }, { "comment": "Converting the contracts façades from `export *` to explicit named re-exports (the pin-table retirement) made these individually visible to --production analysis for the first time; a bare star previously hid them from this exact check. isRecord/IOS_SAFARI_BUNDLE_ID/REPLAY_DIVERGENCE_* have no production consumer. Kept rather than narrowed here so the façade's re-export surface stays byte-identical to the symbol set the retired pin table asserted — narrowing the surface is a follow-up with its own review, not a side effect of this mechanical conversion.", "file": "packages/contracts/src/facades/{client,command,divergence,recording}.ts", diff --git a/packages/command-registry/src/flag-definitions-connection.ts b/packages/command-registry/src/flag-definitions-connection.ts index 0b01b00d23..0c9a34b0f1 100644 --- a/packages/command-registry/src/flag-definitions-connection.ts +++ b/packages/command-registry/src/flag-definitions-connection.ts @@ -236,7 +236,7 @@ export const CONNECTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'string', usageLabel: '--provider-appium-version ', usageDescription: - 'Hosted cloud provider Appium server version, for example 3.2.0. Without it BrowserStack falls back to its default (Appium 1.x)', + 'Hosted cloud provider Appium server version, for example 3.2.0. Without it each provider starts its own default (Appium 1.x on BrowserStack)', projectConfig: false, recorded: false, }, diff --git a/packages/provider-webdriver/package.json b/packages/provider-webdriver/package.json index d4d56b8d0e..a9f01ebacb 100644 --- a/packages/provider-webdriver/package.json +++ b/packages/provider-webdriver/package.json @@ -19,6 +19,10 @@ "./providers": { "types": "./src/providers.ts", "default": "./src/providers.ts" + }, + "./testmu-device-features": { + "types": "./src/testmu-device-features.ts", + "default": "./src/testmu-device-features.ts" } } } diff --git a/packages/provider-webdriver/src/connection-verification.test.ts b/packages/provider-webdriver/src/connection-verification.test.ts index c93e40109e..5b2d872493 100644 --- a/packages/provider-webdriver/src/connection-verification.test.ts +++ b/packages/provider-webdriver/src/connection-verification.test.ts @@ -213,6 +213,254 @@ test('AWS Device Farm rejects a device from the wrong platform before allocation ); }); +const testMuOptions = { + provider: 'testmu' as const, + username: 'lt-user', + accessKey: 'lt-key', + platform: 'android' as const, + deviceName: 'Pixel 8', + osVersion: '14', + app: 'lt://APP1', + devicesEndpoint: 'https://testmu.test/capability/generator?isVirtualDevice=true', + appsEndpoint: 'https://testmu.test/app/data', +}; + +const testMuCatalog = { + app: { + devices: { + android: { + brands: { + Google: [ + { name: 'Pixel 8', osVersion: ['14', '15'] }, + { name: 'Pixel 4a', osVersion: ['13'] }, + ], + }, + }, + ios: { brands: { Apple: [{ name: 'iPhone 16', osVersion: ['18.0'] }] } }, + }, + }, +}; + +test('TestMu verifies the virtual device and uploaded app without creating a session', async () => { + const fetchMock = vi.fn(async (input, init) => { + const headers = (init?.headers ?? {}) as Record; + if (String(input).includes('capability/generator')) { + assert.equal(headers.Authorization, undefined); + return jsonResponse(testMuCatalog); + } + assert.match(String(headers.Authorization), /^Basic /); + return jsonResponse({ + data: [{ app_id: 'APP1', name: 'sample.apk', version: '1.2.3', type: 'android' }], + metaData: { total: 1 }, + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const result = await createProvider().verifyConnection(testMuOptions); + + assert.equal(result.provider, 'testmu'); + assert.equal(result.service, 'TestMu AI'); + assert.deepEqual(result.device, { + status: 'verified', + name: 'Pixel 8', + platform: 'android', + osVersion: '14', + }); + assert.deepEqual(result.app, { + status: 'verified', + name: 'sample.apk', + reference: 'lt://APP1', + version: '1.2.3', + }); + assert.deepEqual( + fetchMock.mock.calls.map(([input]) => String(input)), + [ + 'https://testmu.test/capability/generator?isVirtualDevice=true', + 'https://testmu.test/app/data?type=emulator&level=user', + ], + ); +}); + +test('TestMu checks the catalog of the configured API endpoint', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [{ app_id: 'APP1' }] }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...options } = testMuOptions; + + await createProvider().verifyConnection({ + ...options, + apiEndpoint: 'https://staging.testmu.test/mobile-automation/api/v1/', + }); + + assert.equal( + String(fetchMock.mock.calls[0]?.[0]), + 'https://staging.testmu.test/mobile-automation/api/v1/capability/generator?isVirtualDevice=true', + ); +}); + +test('TestMu keeps the query of an overridden endpoint and adds its own filters', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [{ app_id: 'APP1' }] }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...options } = testMuOptions; + + await createProvider().verifyConnection({ + ...options, + apiEndpoint: 'https://staging.testmu.test/api/v1/?region=eu', + appsEndpoint: 'https://staging.testmu.test/app/data?org=42', + }); + await createProvider().verifyConnection({ + ...testMuOptions, + devicesEndpoint: 'https://testmu.test/capability/generator?region=eu', + }); + + assert.deepEqual( + fetchMock.mock.calls.map(([input]) => String(input)), + [ + 'https://staging.testmu.test/api/v1/capability/generator?region=eu&isVirtualDevice=true', + 'https://staging.testmu.test/app/data?org=42&type=emulator&level=user', + 'https://testmu.test/capability/generator?region=eu&isVirtualDevice=true', + 'https://testmu.test/app/data?type=emulator&level=user', + ], + ); +}); + +test('TestMu rejects a device or OS version missing from the virtual-device catalog', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, osVersion: '12' }), + (error: unknown) => + error instanceof Error && /"Pixel 8" with android 12 is not available/.test(error.message), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, platform: 'ios', deviceName: 'Pixel 8' }), + /is not available/, + ); +}); + +// The hub rejects `platformVersion: '18'` for a catalog entry spelled `18.0`, so connect must too. +test('TestMu matches the catalog OS version spelling exactly and lists the offered versions', async () => { + const catalog = { + app: { + devices: { + ios: { + brands: { + Apple: [{ name: 'iPhone 16', osVersion: ['18.1', '26.0', '18.0', '18.5', '26.2'] }], + }, + }, + }, + }, + }; + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(catalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const iosOptions = { ...testMuOptions, platform: 'ios' as const, deviceName: 'iPhone 16' }; + + await assert.rejects( + createProvider().verifyConnection({ ...iosOptions, osVersion: '18' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); + assert.match(error.message, /iPhone 16 offers 18\.0, 18\.1, 18\.5, 26\.0, 26\.2/); + return true; + }, + ); + + const result = await createProvider().verifyConnection({ ...iosOptions, osVersion: '18.0' }); + assert.deepEqual(result.device, { + status: 'verified', + name: 'iPhone 16', + platform: 'ios', + osVersion: '18.0', + }); +}); + +test('TestMu classifies rejected credentials without exposing them', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ message: 'Unauthorized' }, 401), + ), + ); + await assert.rejects(createProvider().verifyConnection(testMuOptions), (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'UNAUTHORIZED'); + assert.doesNotMatch(error.message, /lt-key/); + return true; + }); +}); + +test('TestMu defers an lt:// reference it cannot find and a local path it will upload', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const unknownApp = await createProvider().verifyConnection(testMuOptions); + assert.equal(unknownApp.app.status, 'configured'); + assert.equal(unknownApp.app.reference, 'lt://APP1'); + + const localApp = await createProvider().verifyConnection({ + ...testMuOptions, + app: '/tmp/builds/App.apk', + }); + assert.deepEqual(localApp.app, { + status: 'configured', + name: 'App.apk', + reference: '/tmp/builds/App.apk', + message: 'Local app artifact is ready and will be uploaded when creating the session.', + }); +}); + +// The listing is keyed by runtime: an Android emulator upload is listed under `emulator` and an +// iOS simulator upload under `simulator`, never under the platform name. +test('TestMu checks an lt:// id against the virtual-device app list of its platform', async () => { + const cases = [ + { platform: 'android', deviceName: 'Pixel 8', osVersion: '14', listType: 'emulator' }, + { platform: 'ios', deviceName: 'iPhone 16', osVersion: '18.0', listType: 'simulator' }, + ] as const; + for (const { platform, deviceName, osVersion, listType } of cases) { + const fetchMock = vi.fn(async (input) => { + const url = String(input); + if (url.includes('capability/generator')) return jsonResponse(testMuCatalog); + return jsonResponse({ + data: new URL(url).searchParams.get('type') === listType ? [{ app_id: 'APP1' }] : [], + }); + }); + vi.stubGlobal('fetch', fetchMock); + const result = await createProvider().verifyConnection({ + ...testMuOptions, + platform, + deviceName, + osVersion, + }); + assert.equal(result.app.status, 'verified', platform); + assert.equal( + String(fetchMock.mock.calls[1]?.[0]), + `https://testmu.test/app/data?type=${listType}&level=user`, + ); + } +}); + function createProvider(runHostCommand: RunHostCommand = vi.fn()) { return createProviderWebDriver({ clientVersion: '1.2.3', runHostCommand }); } diff --git a/packages/provider-webdriver/src/connection-verification.ts b/packages/provider-webdriver/src/connection-verification.ts index 0ef8f5ce97..505fb2cf09 100644 --- a/packages/provider-webdriver/src/connection-verification.ts +++ b/packages/provider-webdriver/src/connection-verification.ts @@ -15,20 +15,32 @@ export type CloudWebDriverConnectionVerification = provider: 'aws-device-farm'; service: 'AWS Device Farm'; project: { name?: string; reference: string }; + }) + | (ProviderConnectionVerification & { + provider: 'testmu'; + service: 'TestMu AI'; + project?: never; }); +/** Credentials plus the exact device, OS, and app a hosted Appium hub session is created with. */ +type HubSelectionVerificationOptions = { + username: string; + accessKey: string; + platform: 'android' | 'ios'; + deviceName: string; + osVersion: string; + app: string; + devicesEndpoint?: string | URL; + appsEndpoint?: string | URL; +}; + export type CloudWebDriverConnectionVerificationOptions = - | { - provider: 'browserstack'; - username: string; - accessKey: string; - platform: 'android' | 'ios'; - deviceName: string; - osVersion: string; - app: string; - devicesEndpoint?: string | URL; - appsEndpoint?: string | URL; - } + | (HubSelectionVerificationOptions & { provider: 'browserstack' }) + | (HubSelectionVerificationOptions & { + provider: 'testmu'; + /** Base of the catalog API, as `TESTMU_API_ENDPOINT` sets it for the runtime. */ + apiEndpoint?: string | URL; + }) | { provider: 'aws-device-farm'; platform: 'android' | 'ios'; @@ -42,7 +54,15 @@ export async function verifyCloudWebDriverConnection( options: CloudWebDriverConnectionVerificationOptions, dependencies: ProviderWebDriverDependencies, ): Promise { - return options.provider === 'browserstack' - ? await verifyBrowserStackConnection(options, dependencies.clientVersion) - : await verifyAwsDeviceFarmConnection(options, dependencies.runHostCommand); + switch (options.provider) { + case 'browserstack': + return await verifyBrowserStackConnection(options, dependencies.clientVersion); + case 'testmu': { + // Loaded on demand: the package entry must not grow its eager closure for a new vendor. + const { verifyTestMuConnection } = await import('./testmu-connection-verification.ts'); + return await verifyTestMuConnection(options, dependencies.clientVersion); + } + case 'aws-device-farm': + return await verifyAwsDeviceFarmConnection(options, dependencies.runHostCommand); + } } diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index 713fd7d8ce..06e147c5eb 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -22,6 +22,7 @@ import { readBrowserStackDeviceFeatureFields, rejectBrowserStackOnlyDeviceFeatures, } from './browserstack-device-features.ts'; +import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import { CLOUD_WEBDRIVER_PROVIDERS, type CloudWebDriverKnownProviderName } from './providers.ts'; import { readAwsDeviceFarmRegionFromArn } from './connection-verification.ts'; import { @@ -37,11 +38,16 @@ export type DefaultCloudWebDriverArtifactEnv = { BROWSERSTACK_SESSION_DETAILS_ENDPOINT?: string; AWS_REGION?: string; AWS_DEFAULT_REGION?: string; + LT_USERNAME?: string; + LT_ACCESS_KEY?: string; + TESTMU_API_ENDPOINT?: string; }; export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtifactEnv & { BROWSERSTACK_WEBDRIVER_ENDPOINT?: string; BROWSERSTACK_APP_UPLOAD_ENDPOINT?: string; + TESTMU_WEBDRIVER_ENDPOINT?: string; + TESTMU_APP_UPLOAD_ENDPOINT?: string; AGENT_DEVICE_AWS_DEVICE_FARM_PROJECT_ARN?: string; AWS_DEVICE_FARM_PROJECT_ARN?: string; AGENT_DEVICE_AWS_DEVICE_FARM_DEVICE_ARN?: string; @@ -50,6 +56,30 @@ export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtif AWS_DEVICE_FARM_APP_ARN?: string; }; +/** + * TestMu (formerly LambdaTest) virtual devices: emulators and simulators behind one Appium hub. + * Only what `createRuntime` needs synchronously lives here; the session, upload, and artifact + * code loads on first use so the package entry stays as lean as it was. + */ +const TESTMU_WEBDRIVER_ENDPOINT = 'https://mobile-hub.lambdatest.com/wd/hub/'; +const TESTMU_CAPABILITY_OVERRIDES = { + install: { + support: 'partial', + note: 'Local app artifacts are uploaded to TestMu AI as virtual-device apps (lt://), then installed with Appium.', + }, + portReverse: { + support: 'unsupported', + note: 'Use the TestMu AI tunnel for network access to local hosts; agent-device port reverse is not available.', + }, + artifacts: { + support: 'supported', + note: 'TestMu AI session details expose provider-hosted video, Appium logs, device logs, network logs, and dashboard links.', + }, +} as const satisfies CloudWebDriverCapabilityOverrides; + +const loadTestMu = async () => await import('./testmu.ts'); +const loadTestMuDeviceFeatures = async () => await import('./testmu-device-features.ts'); + export type CloudWebDriverProviderDefinition = { provider: CloudWebDriverKnownProviderName; createRuntime: (env: DefaultCloudWebDriverProviderRuntimeEnv) => CloudWebDriverRuntime; @@ -246,7 +276,102 @@ export function createCloudWebDriverProviderDefinitions( ); }, }, + { + provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + createRuntime: (env) => + createCloudWebDriverRuntime({ + clientVersion: dependencies.clientVersion, + provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + platform: 'android', + deviceName: 'TestMu AI device', + endpoint: env.TESTMU_WEBDRIVER_ENDPOINT ?? TESTMU_WEBDRIVER_ENDPOINT, + capabilityOverrides: TESTMU_CAPABILITY_OVERRIDES, + listArtifacts: async ({ provider, providerSessionId }) => + await listTestMuArtifactsFromEnv(provider, providerSessionId, env), + prepareSession: async ({ req, lease, base }) => { + const request = requireRequest(req, 'TestMu AI'); + const { buildTestMuCapabilities, createTestMuUploadApp, resolveTestMuAppReference } = + await loadTestMu(); + const { + buildTestMuDeviceFeatureCapabilities, + readTestMuDeviceFeatureFields, + rejectUnsupportedTestMuDeviceFeatures, + } = await loadTestMuDeviceFeatures(); + rejectUnsupportedTestMuDeviceFeatures(request.flags); + const credentials = requireTestMuCredentials(env, 'TestMu AI'); + const platform = requireRequestPlatform(request, 'TestMu AI'); + const deviceName = requireFlag( + request, + 'device', + 'TestMu AI requires --device .', + ); + const osVersion = requireFlag( + request, + 'providerOsVersion', + 'TestMu AI requires --provider-os-version .', + ); + const upload = { + clientVersion: dependencies.clientVersion, + ...credentials, + endpoint: env.TESTMU_APP_UPLOAD_ENDPOINT, + }; + const app = await resolveTestMuAppReference( + requireFlag( + request, + 'providerApp', + 'TestMu AI requires --provider-app .', + ), + { ...upload, cwd: request.cwd, signal: request.signal }, + ); + return { + ...base, + platform, + deviceName, + auth: credentials, + uploadApp: createTestMuUploadApp(upload), + webdriverCapabilities: buildTestMuCapabilities({ + platform, + deviceName, + osVersion, + app, + projectName: readFlag(request, 'providerProject'), + buildName: readFlag(request, 'providerBuild') ?? lease.runId, + sessionName: readFlag(request, 'providerSessionName') ?? lease.leaseId, + deviceFeatures: buildTestMuDeviceFeatureCapabilities( + readTestMuDeviceFeatureFields(request.flags), + ), + configured: buildCloudWebDriverBaseCapabilities(platform, deviceName), + }), + }; + }, + }), + listArtifactsFromEnv: async (providerSessionId, env) => + await listTestMuArtifactsFromEnv(CLOUD_WEBDRIVER_PROVIDERS.testMu, providerSessionId, env), + }, ]; + + async function listTestMuArtifactsFromEnv( + provider: string, + providerSessionId: string | undefined, + env: DefaultCloudWebDriverArtifactEnv, + ): Promise { + const { listTestMuCloudArtifacts } = await loadTestMu(); + return await listTestMuCloudArtifacts(provider, providerSessionId, { + clientVersion: dependencies.clientVersion, + ...requireTestMuCredentials(env, 'TestMu AI artifact lookup'), + endpoint: env.TESTMU_API_ENDPOINT, + }); + } +} + +function requireTestMuCredentials( + env: DefaultCloudWebDriverArtifactEnv, + providerLabel: string, +): { username: string; accessKey: string } { + return { + username: requireEnv(env, 'LT_USERNAME', providerLabel), + accessKey: requireEnv(env, 'LT_ACCESS_KEY', providerLabel), + }; } function requireRequest( diff --git a/packages/provider-webdriver/src/providers.ts b/packages/provider-webdriver/src/providers.ts index 814a0c19ae..4160d465a5 100644 --- a/packages/provider-webdriver/src/providers.ts +++ b/packages/provider-webdriver/src/providers.ts @@ -1,6 +1,7 @@ export const CLOUD_WEBDRIVER_PROVIDERS = { browserStack: 'browserstack', awsDeviceFarm: 'aws-device-farm', + testMu: 'testmu', } as const; export type CloudWebDriverKnownProviderName = diff --git a/packages/provider-webdriver/src/testmu-connection-verification.ts b/packages/provider-webdriver/src/testmu-connection-verification.ts new file mode 100644 index 0000000000..91d913ce23 --- /dev/null +++ b/packages/provider-webdriver/src/testmu-connection-verification.ts @@ -0,0 +1,190 @@ +import path from 'node:path'; +import { AppError } from '@agent-device/kernel/errors'; +import { asRecord, fetchProviderVerificationJson, trimTrailingSlash } from './webdriver-utils.ts'; +import { TESTMU_API_ENDPOINT, TESTMU_APPS_ENDPOINT, isTestMuAppReference } from './testmu.ts'; +import type { + CloudWebDriverConnectionVerification, + CloudWebDriverConnectionVerificationOptions, +} from './connection-verification.ts'; +import type { ProviderConnectionResource } from '@agent-device/contracts/remote'; + +type TestMuOptions = Extract; + +type TestMuAuth = { username: string; accessKey: string }; + +/** `/app/data?type=` keys virtual-device uploads by runtime, not by platform. */ +const TESTMU_APP_LIST_TYPES: Record<'android' | 'ios', string> = { + android: 'emulator', + ios: 'simulator', +}; + +/** + * Verifies a TestMu virtual-device selection without creating a session: the public capability + * catalog confirms the device/OS pair exists in the emulator and simulator pool, and the + * authenticated app listing confirms the credentials and, for an `lt://` reference, the upload. + */ +export async function verifyTestMuConnection( + options: TestMuOptions, + clientVersion: string, +): Promise { + const auth = { username: options.username, accessKey: options.accessKey }; + const catalogUrl = options.devicesEndpoint + ? new URL(options.devicesEndpoint) + : apiUrl(options.apiEndpoint ?? TESTMU_API_ENDPOINT, 'capability/generator'); + catalogUrl.searchParams.set('isVirtualDevice', 'true'); + const catalog = await fetchTestMuJson(catalogUrl, undefined, clientVersion); + const namedDevices = readTestMuVirtualDevices(catalog, options.platform).filter( + (device) => device.name === options.deviceName, + ); + // Exact match on purpose: the hub rejects `18` for a device the catalog lists as `18.0`. + const matchedDevice = namedDevices.find((device) => + device.osVersions.includes(options.osVersion), + ); + if (!matchedDevice) { + const offered = [...new Set(namedDevices.flatMap((device) => device.osVersions))].sort( + (left, right) => left.localeCompare(right, undefined, { numeric: true }), + ); + throw new AppError( + 'INVALID_ARGS', + `TestMu AI virtual device "${options.deviceName}" with ${options.platform} ${options.osVersion} is not available${ + offered.length > 0 ? `; ${options.deviceName} offers ${offered.join(', ')}` : '' + }.`, + { + hint: 'Choose an exact device name and OS version from the TestMu AI virtual-device capability generator.', + ...(offered.length > 0 ? { availableOsVersions: offered } : {}), + }, + ); + } + + const app = await verifyTestMuApp(options, auth, clientVersion); + return { + provider: 'testmu', + service: 'TestMu AI', + verificationMessage: + app.status === 'verified' + ? 'Credentials, virtual device, and uploaded app verified.' + : 'Credentials and virtual device verified; app availability is checked when the session is created.', + device: { + status: 'verified', + name: matchedDevice.name, + platform: options.platform, + osVersion: options.osVersion, + }, + app, + }; +} + +async function verifyTestMuApp( + options: TestMuOptions, + auth: TestMuAuth, + clientVersion: string, +): Promise { + const { app } = options; + // The listing is authenticated, so it doubles as the credential check for every app kind. + const appsUrl = new URL(options.appsEndpoint ?? TESTMU_APPS_ENDPOINT); + appsUrl.searchParams.set('type', TESTMU_APP_LIST_TYPES[options.platform]); + appsUrl.searchParams.set('level', 'user'); + const apps = await fetchTestMuJson(appsUrl, auth, clientVersion); + if (isTestMuAppReference(app)) { + const matched = readTestMuApps(apps).find((entry) => entry.reference === app); + if (!matched) { + return { + status: 'configured', + reference: app, + message: + 'App reference was not found among your virtual-device uploads; TestMu AI validates it when creating the session.', + }; + } + return { status: 'verified', ...matched }; + } + if (/^https?:\/\//i.test(app)) { + return { + status: 'configured', + reference: app, + message: 'Public app URL configured; TestMu AI fetches it when creating the session.', + }; + } + return { + status: 'configured', + name: path.basename(app), + reference: app, + message: 'Local app artifact is ready and will be uploaded when creating the session.', + }; +} + +/** Appends `route` to the base's path; the base may carry a query, which is kept. */ +function apiUrl(base: string | URL, route: string): URL { + const url = new URL(base); + url.pathname = `${trimTrailingSlash(url.pathname)}/${route}`; + return url; +} + +async function fetchTestMuJson( + endpoint: string | URL, + auth: TestMuAuth | undefined, + clientVersion: string, +): Promise { + return await fetchProviderVerificationJson(endpoint, { + clientVersion, + auth, + hints: { + service: 'TestMu AI', + unauthorizedHint: 'Check LT_USERNAME and LT_ACCESS_KEY.', + networkHint: + 'Check network access to mobile-api.lambdatest.com and manual-api.lambdatest.com, then retry connect.', + }, + }); +} + +/** + * The capability generator lists virtual devices per platform under + * `app.devices..brands.[]` as `{ name, osVersion: string[] }`. + */ +function readTestMuVirtualDevices( + value: unknown, + platform: 'android' | 'ios', +): Array<{ name: string; osVersions: string[] }> { + const platformCatalog = asRecord(asRecord(asRecord(asRecord(value)?.app)?.devices)?.[platform]); + const brandRecord = asRecord(platformCatalog?.brands); + if (!brandRecord) { + throw new AppError( + 'COMMAND_FAILED', + 'TestMu AI virtual-device catalog response did not list devices for the platform.', + { platform }, + ); + } + return Object.values(brandRecord).flatMap((devices) => { + if (!Array.isArray(devices)) return []; + return devices.flatMap((entry) => { + const record = asRecord(entry); + if (!record || typeof record.name !== 'string' || !Array.isArray(record.osVersion)) return []; + const osVersions = record.osVersion.flatMap((osVersion) => + typeof osVersion === 'string' || typeof osVersion === 'number' ? [String(osVersion)] : [], + ); + return [{ name: record.name, osVersions }]; + }); + }); +} + +/** `/app/data` answers `{ data: [{ app_id, name, version, ... }], metaData }`. */ +function readTestMuApps( + value: unknown, +): Array<{ name?: string; reference: string; version?: string }> { + const record = asRecord(value); + const data = record?.data; + if (!Array.isArray(data)) { + throw new AppError('COMMAND_FAILED', 'TestMu AI app listing response was not a list.'); + } + return data.flatMap((entry) => { + const app = asRecord(entry); + if (!app || typeof app.app_id !== 'string') return []; + const reference = isTestMuAppReference(app.app_id) ? app.app_id : `lt://${app.app_id}`; + return [ + { + reference, + ...(typeof app.name === 'string' ? { name: app.name } : {}), + ...(typeof app.version === 'string' ? { version: app.version } : {}), + }, + ]; + }); +} diff --git a/packages/provider-webdriver/src/testmu-device-features.test.ts b/packages/provider-webdriver/src/testmu-device-features.test.ts new file mode 100644 index 0000000000..da38e77e3b --- /dev/null +++ b/packages/provider-webdriver/src/testmu-device-features.test.ts @@ -0,0 +1,103 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; + +import { AppError } from '@agent-device/kernel/errors'; +import { + TESTMU_DEVICE_FEATURE_SPECS, + buildTestMuDeviceFeatureCapabilities, + readTestMuDeviceFeatureFields, + rejectUnsupportedTestMuDeviceFeatures, +} from './testmu-device-features.ts'; + +// Every hosted-provider device-feature field is either a TestMu spec row or an explicit rejection: +// a field in neither parses off the CLI, rides the profile, and is silently dropped at the hub. +const SUPPORTED_FIELDS = [ + 'providerDeviceOrientation', + 'providerGeoLocation', + 'providerTimezone', + 'providerAppiumVersion', + 'providerLanguage', + 'providerLocale', +] as const; +const REJECTED_FIELDS = [ + 'providerNetworkProfile', + 'providerCustomNetwork', + 'providerNoResignApp', +] as const; + +test('every supported device-feature field maps to exactly one lt:options key', () => { + const fields = TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.field); + assert.deepEqual([...fields].sort(), [...SUPPORTED_FIELDS].sort()); + const capabilities = TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.capability); + assert.equal(new Set(capabilities).size, capabilities.length); +}); + +test('configured device features project onto TestMu capability keys', () => { + const capabilities = buildTestMuDeviceFeatureCapabilities({ + providerDeviceOrientation: 'landscape', + providerGeoLocation: 'US', + providerTimezone: 'UTC+05:30', + providerAppiumVersion: '2.16.2', + providerLanguage: 'fr', + providerLocale: 'fr_FR', + }); + assert.deepEqual(capabilities, { + deviceOrientation: 'LANDSCAPE', + geoLocation: 'US', + timezone: 'UTC+05:30', + appiumVersion: '2.16.2', + language: 'fr', + locale: 'fr_FR', + }); +}); + +test('unset and empty device features emit nothing', () => { + assert.deepEqual(buildTestMuDeviceFeatureCapabilities({}), {}); + assert.deepEqual(buildTestMuDeviceFeatureCapabilities({ providerGeoLocation: '' }), {}); +}); + +test('daemon flag bags are read through the same table with orientation validated', () => { + assert.deepEqual( + readTestMuDeviceFeatureFields({ + providerDeviceOrientation: 'portrait', + providerLocale: 'de_DE', + providerNetworkProfile: 'ignored-here', + providerGeoLocation: 7, + }), + { providerDeviceOrientation: 'portrait', providerLocale: 'de_DE' }, + ); + assert.throws( + () => readTestMuDeviceFeatureFields({ providerDeviceOrientation: 'sideways' }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.details?.flag === '--provider-device-orientation', + ); +}); + +test('BrowserStack-only flags are rejected by flag name instead of being dropped', () => { + assert.doesNotThrow(() => rejectUnsupportedTestMuDeviceFeatures(undefined)); + assert.doesNotThrow(() => + rejectUnsupportedTestMuDeviceFeatures({ + providerGeoLocation: 'US', + providerNoResignApp: false, + }), + ); + for (const field of REJECTED_FIELDS) { + assert.throws( + () => + rejectUnsupportedTestMuDeviceFeatures({ + [field]: field === 'providerNoResignApp' ? true : 'x', + }), + (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); + } + assert.throws( + () => + rejectUnsupportedTestMuDeviceFeatures({ + providerNetworkProfile: '4g-lte-good', + providerCustomNetwork: '1000', + }), + /--provider-network-profile, --provider-custom-network are not supported by TestMu AI/, + ); +}); diff --git a/packages/provider-webdriver/src/testmu-device-features.ts b/packages/provider-webdriver/src/testmu-device-features.ts new file mode 100644 index 0000000000..6f1ab049c4 --- /dev/null +++ b/packages/provider-webdriver/src/testmu-device-features.ts @@ -0,0 +1,115 @@ +import type { CloudProviderProfileFields } from '@agent-device/contracts/remote'; +import { AppError } from '@agent-device/kernel/errors'; +import { requireProviderDeviceOrientation } from './webdriver-utils.ts'; + +/** + * TestMu "device feature" session capabilities: the hosted-provider flags TestMu can act on, + * projected onto their `lt:options` keys. The table is the contract; adding a capability means + * adding a row, not a branch. + */ +export type TestMuDeviceFeatureFields = Pick< + CloudProviderProfileFields, + | 'providerDeviceOrientation' + | 'providerGeoLocation' + | 'providerTimezone' + | 'providerAppiumVersion' + | 'providerLanguage' + | 'providerLocale' +>; + +type TestMuDeviceFeatureSpec = { + field: keyof TestMuDeviceFeatureFields; + /** Key emitted inside `lt:options`. */ + capability: string; + /** Canonical CLI flag, so an error can name a recovery action. */ + flag: string; + /** Projects the validated flag value onto what the hub expects. */ + project?: (value: string) => unknown; +}; + +export const TESTMU_DEVICE_FEATURE_SPECS: readonly TestMuDeviceFeatureSpec[] = [ + { + field: 'providerDeviceOrientation', + capability: 'deviceOrientation', + flag: '--provider-device-orientation', + // The hub matches the orientation enum case-sensitively in upper case. + project: (value) => value.toUpperCase(), + }, + { field: 'providerGeoLocation', capability: 'geoLocation', flag: '--provider-geo-location' }, + { field: 'providerTimezone', capability: 'timezone', flag: '--provider-timezone' }, + { + field: 'providerAppiumVersion', + capability: 'appiumVersion', + flag: '--provider-appium-version', + }, + { field: 'providerLanguage', capability: 'language', flag: '--provider-language' }, + { field: 'providerLocale', capability: 'locale', flag: '--provider-locale' }, +]; + +/** Hosted-provider flags other vendors own and TestMu has no capability for. */ +const TESTMU_UNSUPPORTED_DEVICE_FEATURE_FLAGS: ReadonlyArray<{ + field: keyof CloudProviderProfileFields; + flag: string; +}> = [ + { field: 'providerNetworkProfile', flag: '--provider-network-profile' }, + { field: 'providerCustomNetwork', flag: '--provider-custom-network' }, + { field: 'providerNoResignApp', flag: '--provider-no-resign-app' }, +]; + +/** Builds the `lt:options` fragment for the configured device features. */ +export function buildTestMuDeviceFeatureCapabilities( + fields: TestMuDeviceFeatureFields, +): Record { + const capabilities: Record = {}; + for (const spec of TESTMU_DEVICE_FEATURE_SPECS) { + const value = fields[spec.field]; + if (value === undefined || value === '') continue; + capabilities[spec.capability] = spec.project ? spec.project(value) : value; + } + return capabilities; +} + +/** + * Fails when flags TestMu cannot act on were given. Called from both `connect testmu` (through the + * `./testmu-device-features` subpath, so the package entry stays lazy) and session preparation, + * since the typed client and hand-authored profiles skip `connect`. + */ +export function rejectUnsupportedTestMuDeviceFeatures( + flags: Record | undefined, +): void { + const configured = TESTMU_UNSUPPORTED_DEVICE_FEATURE_FLAGS.filter(({ field }) => { + const value = flags?.[field]; + return value !== undefined && value !== false && value !== ''; + }).map(({ flag }) => flag); + if (configured.length === 0) return; + const plural = configured.length !== 1; + throw new AppError( + 'INVALID_ARGS', + `${configured.join(', ')} ${plural ? 'are' : 'is'} not supported by TestMu AI.`, + { + hint: `Drop ${plural ? 'those flags' : 'the flag'}; TestMu AI has no equivalent capability.`, + provider: 'testmu', + flags: configured, + }, + ); +} + +/** + * Reads device-feature fields off an untyped flag bag (a daemon request). Enum values are + * validated here rather than forwarded to the hub, where an unrecognized value is ignored. + */ +export function readTestMuDeviceFeatureFields( + flags: Record | undefined, +): TestMuDeviceFeatureFields { + const fields: TestMuDeviceFeatureFields = {}; + for (const spec of TESTMU_DEVICE_FEATURE_SPECS) { + const value = flags?.[spec.field]; + if (typeof value !== 'string' || value.length === 0) continue; + if (spec.field === 'providerDeviceOrientation') { + fields.providerDeviceOrientation = requireProviderDeviceOrientation(spec, value); + continue; + } + fields[spec.field] = value; + } + return fields; +} diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-webdriver/src/testmu.test.ts new file mode 100644 index 0000000000..9b02320c57 --- /dev/null +++ b/packages/provider-webdriver/src/testmu.test.ts @@ -0,0 +1,439 @@ +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + buildTestMuCapabilities, + createTestMuUploadApp, + listTestMuCloudArtifacts, + resolveTestMuAppReference, + uploadTestMuApp, + uploadTestMuAppFromUrl, +} from './testmu.ts'; +import { buildCloudWebDriverBaseCapabilities } from './runtime.ts'; +import { mkdtempForTest } from './tmp-dir.fixtures.ts'; + +const realFetch = globalThis.fetch; +const auth = { clientVersion: '0.0.0-test', username: 'user', accessKey: 'key' }; + +afterEach(() => { + globalThis.fetch = realFetch; + vi.unstubAllGlobals(); +}); + +// `isRealMobile: false` is the one capability that routes to the emulator/simulator pool; a +// session without it lands on a real device and bills differently. +test('TestMu capabilities select the virtual-device pool and keep vendor keys in lt:options', () => { + const capabilities = buildTestMuCapabilities({ + platform: 'android', + deviceName: 'Pixel 8', + osVersion: '14', + app: 'lt://APP1', + projectName: 'agent-device', + buildName: 'run-1', + sessionName: 'lease-1', + deviceFeatures: { geoLocation: 'US' }, + configured: buildCloudWebDriverBaseCapabilities('android', 'Pixel 8'), + }); + + assert.deepEqual(capabilities, { + platformName: 'Android', + 'appium:deviceName': 'Pixel 8', + 'appium:platformVersion': '14', + 'appium:app': 'lt://APP1', + 'lt:options': { + isRealMobile: false, + w3c: true, + platformName: 'Android', + deviceName: 'Pixel 8', + platformVersion: '14', + app: 'lt://APP1', + project: 'agent-device', + build: 'run-1', + name: 'lease-1', + video: true, + devicelog: true, + geoLocation: 'US', + }, + }); + for (const key of Object.keys(capabilities)) { + assert.ok( + key === 'platformName' || key.startsWith('appium:') || key === 'lt:options', + `legacy top-level key ${key} would make the hub ignore lt:options`, + ); + } +}); + +// Unpinned, TestMu AI starts its own default Appium server for the device, as BrowserStack does. +test('a configured lt:options merges per key and only a pinned Appium version is sent', () => { + const capabilities = buildTestMuCapabilities({ + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18.0', + buildName: 'run-1', + sessionName: 'lease-1', + deviceFeatures: { appiumVersion: '2.16.2' }, + configured: { 'lt:options': { tunnel: true } }, + }); + const ltOptions = capabilities['lt:options'] as Record; + assert.equal(ltOptions.appiumVersion, '2.16.2'); + assert.equal(ltOptions.tunnel, true); + assert.equal(ltOptions.build, 'run-1'); + assert.equal(ltOptions.platformName, 'iOS'); + assert.equal('appium:app' in capabilities, false); + + const unpinned = buildTestMuCapabilities({ + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18.0', + buildName: 'run-1', + sessionName: 'lease-1', + }); + assert.equal('appiumVersion' in (unpinned['lt:options'] as Record), false); +}); + +test('a configured lt:options cannot turn off the W3C dialect', () => { + const capabilities = buildTestMuCapabilities({ + platform: 'android', + deviceName: 'Pixel 8', + osVersion: '14', + buildName: 'run-1', + sessionName: 'lease-1', + configured: { 'lt:options': { w3c: false, tunnel: true } }, + }); + const ltOptions = capabilities['lt:options'] as Record; + assert.equal(ltOptions.w3c, true); + assert.equal(ltOptions.tunnel, true); +}); + +// A configured `isRealMobile` would silently move the session to the real-device pool, which +// bills differently. +test('a configured lt:options cannot move the session off the virtual-device pool', () => { + const capabilities = buildTestMuCapabilities({ + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18.0', + buildName: 'run-1', + sessionName: 'lease-1', + configured: { 'lt:options': { isRealMobile: true, tunnel: true } }, + }); + const ltOptions = capabilities['lt:options'] as Record; + assert.equal(ltOptions.isRealMobile, false); + assert.equal(ltOptions.tunnel, true); +}); + +test('TestMu uploads go to the virtual-device upload API unless an endpoint is configured', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-upload-endpoint-'); + const appPath = path.join(tempDir, 'MyApp.apk'); + const endpoints: string[] = []; + try { + await fs.writeFile(appPath, 'placeholder'); + globalThis.fetch = async (input) => { + endpoints.push(String(input)); + return jsonResponse({ app_url: 'lt://APP1' }); + }; + await uploadTestMuApp(appPath, auth); + await uploadTestMuAppFromUrl('https://example.test/App.apk', auth); + await uploadTestMuApp(appPath, { ...auth, endpoint: 'https://upload.test/virtual' }); + assert.deepEqual(endpoints, [ + 'https://manual-api.lambdatest.com/app/upload/virtualDevice', + 'https://manual-api.lambdatest.com/app/upload/virtualDevice', + 'https://upload.test/virtual', + ]); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('TestMu upload reads the lt:// reference and aborts while the request is in flight', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-upload-'); + const appPath = path.join(tempDir, 'App.apk'); + const controller = new AbortController(); + const abortReason = new Error('request cancelled during TestMu AI upload'); + try { + await fs.writeFile(appPath, 'placeholder'); + globalThis.fetch = async (_input, init) => + await new Promise((_resolve, reject) => { + assert.equal(init?.signal, controller.signal); + if (init?.signal?.aborted) { + reject(init.signal.reason); + return; + } + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); + }); + + const pending = uploadTestMuApp(appPath, auth, controller.signal); + await Promise.resolve(); + controller.abort(abortReason); + await assert.rejects(pending, (error: unknown) => error === abortReason); + + globalThis.fetch = async (_input, init) => { + const body = init?.body as FormData; + assert.ok(body.get('appFile') instanceof Blob); + assert.equal(body.get('name'), 'App'); + return jsonResponse({ app_id: 'APP123', name: 'App' }); + }; + assert.equal(await uploadTestMuApp(appPath, auth), 'lt://APP123'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +// iOS simulator builds are `.app` directories; the upload API only takes a file. +test('TestMu upload rejects an unzipped .app bundle before calling the upload API', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-app-dir-'); + const appPath = path.join(tempDir, 'Demo.app'); + try { + await fs.mkdir(appPath); + const fetchMock = vi.fn(); + globalThis.fetch = fetchMock; + await assert.rejects(uploadTestMuApp(appPath, auth), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.match(String(error.details?.hint), /[Zz]ip the \.app bundle/); + return true; + }); + assert.equal(fetchMock.mock.calls.length, 0); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('the install adapter uploads the local build and launches the hinted app id', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-install-'); + const appPath = path.join(tempDir, 'Demo.apk'); + try { + await fs.writeFile(appPath, 'placeholder'); + globalThis.fetch = async () => jsonResponse({ app_url: 'lt://APP77' }); + const uploadApp = createTestMuUploadApp(auth); + const result = await uploadApp({ + provider: 'testmu', + lease: {} as never, + device: {} as never, + app: 'com.example.demo', + appPath, + options: { packageNameHint: 'com.example.demo' }, + }); + assert.deepEqual(result, { + appReference: 'lt://APP77', + bundleId: undefined, + packageName: 'com.example.demo', + launchTarget: 'com.example.demo', + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('TestMu passes lt:// ids through and has the upload API fetch a public URL', async () => { + const forms: FormData[] = []; + globalThis.fetch = async (_input, init) => { + forms.push(init?.body as FormData); + return jsonResponse({ app_id: 'APP9' }); + }; + assert.equal(await resolveTestMuAppReference('lt://APP1', auth), 'lt://APP1'); + assert.equal(forms.length, 0); + assert.equal( + await resolveTestMuAppReference('https://builds.example/App.apk', auth), + 'lt://APP9', + ); + assert.equal(forms[0]?.get('url'), 'https://builds.example/App.apk'); + await assert.rejects( + resolveTestMuAppReference('missing.apk', { ...auth, cwd: '/nonexistent' }), + /must be an lt:\/\/ app id, URL, or existing local app path/, + ); +}); + +test('TestMu upload accepts only an lt:// reference or a valid app id from the response', async () => { + const cases: Array<[unknown, string | undefined]> = [ + [{ app_url: 'lt://APP6' }, 'lt://APP6'], + [{ app_url: 'https://cdn.example/app.apk', app_id: 'APP5' }, 'lt://APP5'], + [{ app_id: 'lt://APP7' }, 'lt://APP7'], + [{ app_url: 'https://cdn.example/app.apk' }, undefined], + [{ app_url: 'lt://' }, undefined], + [{ app_id: 'bs://APP8' }, undefined], + ]; + for (const [body, expected] of cases) { + globalThis.fetch = async () => jsonResponse(body); + const pending = uploadTestMuAppFromUrl('https://builds.example/App.apk', auth); + if (expected) { + assert.equal(await pending, expected); + continue; + } + await assert.rejects(pending, (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.deepEqual(error.details?.response, body); + return true; + }); + } +}); + +test('TestMu URL upload hands the URL to the upload API and surfaces a failed upload', async () => { + globalThis.fetch = async (_input, init) => { + const body = init?.body as FormData; + assert.equal(body.get('url'), 'https://example.test/builds/App.apk'); + assert.equal(body.get('storage'), 'url'); + assert.equal(body.get('name'), 'App.apk'); + assert.equal(body.get('appFile'), null); + return jsonResponse({ app_url: 'lt://APP9' }); + }; + assert.equal( + await uploadTestMuAppFromUrl('https://example.test/builds/App.apk', auth), + 'lt://APP9', + ); + + globalThis.fetch = async () => jsonResponse({ message: 'invalid app' }, 400); + await assert.rejects( + uploadTestMuAppFromUrl('https://example.test/builds/App.apk', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 400, + ); +}); + +test('TestMu artifacts come from the jsend session payload and stay pending until a URL exists', async () => { + const calls: string[] = []; + globalThis.fetch = async (input, init) => { + calls.push(String(input)); + const headers = (init?.headers ?? {}) as Record; + assert.match(String(headers.Authorization), /^Basic /); + return jsonResponse({ + status: 'success', + data: { + test_id: 'SESSION1', + video_url: 'https://cdn.test/video.mp4', + appium_logs_url: 'https://api.test/sessions/SESSION1/log/appium', + device_logs_url: '', + }, + }); + }; + const result = await listTestMuCloudArtifacts('testmu', 'SESSION1', { + ...auth, + endpoint: 'https://api.test/mobile-automation/api/v1/', + }); + assert.deepEqual(calls, ['https://api.test/mobile-automation/api/v1/sessions/SESSION1']); + assert.equal(result?.status, 'ready'); + assert.deepEqual( + result?.cloudArtifacts.map((artifact) => [artifact.kind, artifact.url]), + [ + ['video', 'https://cdn.test/video.mp4'], + ['appium-log', 'https://api.test/sessions/SESSION1/log/appium'], + ['provider-session', 'https://appautomation.lambdatest.com/test?testID=SESSION1'], + ], + ); + + globalThis.fetch = async () => jsonResponse({ status: 'success', data: { test_id: 'SESSION1' } }); + const pending = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.equal(pending?.status, 'pending'); + assert.deepEqual(pending?.cloudArtifacts, []); +}); + +// Virtual-device session details carry the device log as `console_logs_url`. +test('TestMu reads the console log as the device log and falls back to device_logs_url', async () => { + globalThis.fetch = async () => + jsonResponse({ + status: 'success', + data: { + console_logs_url: 'https://api.test/sessions/SESSION1/log/console', + device_logs_url: 'https://api.test/sessions/SESSION1/log/device', + }, + }); + const consoleLog = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.deepEqual( + consoleLog?.cloudArtifacts + .filter((artifact) => artifact.kind === 'device-log') + .map((artifact) => artifact.url), + ['https://api.test/sessions/SESSION1/log/console'], + ); + + globalThis.fetch = async () => + jsonResponse({ + status: 'success', + data: { device_logs_url: 'https://api.test/sessions/SESSION1/log/device' }, + }); + const deviceLog = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.deepEqual( + deviceLog?.cloudArtifacts + .filter((artifact) => artifact.kind === 'device-log') + .map((artifact) => artifact.url), + ['https://api.test/sessions/SESSION1/log/device'], + ); +}); + +test('TestMu session details read as pending on 404 and fail typed on a body that is not JSON', async () => { + let signal: AbortSignal | undefined; + globalThis.fetch = async (_input, init) => { + signal = init?.signal ?? undefined; + return jsonResponse({ status: 'fail', message: 'session not found' }, 404); + }; + const notFound = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.equal(notFound?.status, 'pending'); + assert.deepEqual(notFound?.cloudArtifacts, []); + assert.ok(signal instanceof AbortSignal, 'session details lookup should carry a timeout'); + + globalThis.fetch = async () => new Response('Bad Gateway', { status: 502 }); + await assert.rejects( + listTestMuCloudArtifacts('testmu', 'SESSION1', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 502, + ); + + globalThis.fetch = async () => new Response('', { status: 200 }); + await assert.rejects( + listTestMuCloudArtifacts('testmu', 'SESSION1', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 200, + ); +}); + +test('TestMu session details require the jsend data envelope', async () => { + globalThis.fetch = async () => jsonResponse({ video_url: 'https://cdn.test/video.mp4' }); + await assert.rejects( + listTestMuCloudArtifacts('testmu', 'SESSION1', auth), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); +}); + +test('TestMu upload reports the HTTP status when the response is not JSON', async () => { + globalThis.fetch = async () => new Response('Bad Gateway', { status: 502 }); + await assert.rejects( + uploadTestMuAppFromUrl('https://example.test/builds/App.apk', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 502, + ); + + globalThis.fetch = async () => new Response('', { status: 200 }); + await assert.rejects( + uploadTestMuAppFromUrl('https://example.test/builds/App.apk', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 200, + ); +}); + +test('TestMu session details lookup types a timeout and a network failure', async () => { + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + globalThis.fetch = async () => { + throw timeout; + }; + await assert.rejects(listTestMuCloudArtifacts('testmu', 'SESSION1', auth), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.match(error.message, /TestMu AI session details lookup failed/); + assert.match(String(error.details?.hint), /retry/); + assert.equal(error.cause, timeout); + return true; + }); + + globalThis.fetch = async () => { + throw new TypeError('fetch failed'); + }; + await assert.rejects( + listTestMuCloudArtifacts('testmu', 'SESSION1', auth), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); +}); + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { status }); +} diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-webdriver/src/testmu.ts new file mode 100644 index 0000000000..a06b14a0dd --- /dev/null +++ b/packages/provider-webdriver/src/testmu.ts @@ -0,0 +1,264 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; +import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +import { + appFileUploadForm, + asRecord, + createHubUploadApp, + fetchProviderSessionDetails, + postHubAppUpload, + resolveHubAppReference, + trimTrailingSlash, +} from './webdriver-utils.ts'; + +/** + * TestMu session, upload, and artifact mechanics. Loaded on demand by the provider definition; + * `isRealMobile: false` in `lt:options` is what routes a session to the virtual-device pool, and + * the hostnames still carry the lambdatest.com brand. + */ +const TESTMU_APP_UPLOAD_ENDPOINT = 'https://manual-api.lambdatest.com/app/upload/virtualDevice'; +export const TESTMU_APPS_ENDPOINT = 'https://manual-api.lambdatest.com/app/data'; +export const TESTMU_API_ENDPOINT = 'https://mobile-api.lambdatest.com/mobile-automation/api/v1'; +const TESTMU_DASHBOARD_TEST_URL = 'https://appautomation.lambdatest.com/test?testID='; + +export type TestMuCapabilitiesOptions = { + platform: CloudWebDriverPlatform; + deviceName: string; + osVersion: string; + app?: string; + projectName?: string; + buildName: string; + sessionName: string; + /** Vendor device-feature capabilities, already projected onto their `lt:options` keys. */ + deviceFeatures?: Record; + configured?: Record; +}; + +export type TestMuAuth = { + username: string; + accessKey: string; +}; + +export type TestMuSessionDetailsOptions = TestMuAuth & { + clientVersion: string; + endpoint?: string | URL; +}; + +export async function listTestMuCloudArtifacts( + provider: string, + providerSessionId: string | undefined, + options: TestMuSessionDetailsOptions, +): Promise { + if (!providerSessionId) return undefined; + const details = await fetchTestMuSessionDetails(providerSessionId, options); + const artifacts = mapTestMuArtifacts(provider, providerSessionId, details); + return cloudArtifactsReadyOrPending({ + provider, + providerSessionId, + artifacts, + pendingMessage: 'TestMu AI artifacts are not ready yet.', + }); +} + +export type TestMuUploadOptions = TestMuAuth & { + clientVersion: string; + endpoint?: string | URL; +}; + +/** Uploads a local `.apk`, `.aab`, `.ipa`, or zipped simulator `.app` and returns its `lt://` reference. */ +export async function uploadTestMuApp( + appPath: string, + options: TestMuUploadOptions, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + if (!(await fs.stat(appPath)).isFile()) { + throw new AppError('INVALID_ARGS', `TestMu AI can only upload an app file: ${appPath}`, { + appPath, + hint: 'Zip the .app bundle of an iOS simulator build and pass the .zip.', + }); + } + const form = await appFileUploadForm(appPath, 'appFile'); + form.set('name', path.parse(appPath).name); + return await postTestMuUpload(form, options, signal); +} + +/** Has TestMu fetch a public app URL itself, returning its `lt://` reference. */ +export async function uploadTestMuAppFromUrl( + url: string, + options: TestMuUploadOptions, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + const form = new FormData(); + form.set('url', url); + form.set('storage', 'url'); + form.set('name', path.basename(new URL(url).pathname) || 'app'); + return await postTestMuUpload(form, options, signal); +} + +async function postTestMuUpload( + form: FormData, + options: TestMuUploadOptions, + signal?: AbortSignal, +): Promise { + return await postHubAppUpload( + form, + { + service: 'TestMu AI', + endpoint: options.endpoint ?? TESTMU_APP_UPLOAD_ENDPOINT, + clientVersion: options.clientVersion, + auth: options, + readAppReference: readTestMuAppReference, + }, + signal, + ); +} + +export function createTestMuUploadApp(options: TestMuUploadOptions): CloudWebDriverUploadApp { + return createHubUploadApp( + async (appPath, signal) => await uploadTestMuApp(appPath, options, signal), + ); +} + +/** The hub only accepts `lt://` references, so a public URL is handed to the upload API to fetch. */ +export async function resolveTestMuAppReference( + app: string, + options: TestMuUploadOptions & { cwd?: string; signal?: AbortSignal }, +): Promise { + return await resolveHubAppReference({ + service: 'TestMu AI', + app, + cwd: options.cwd, + referenceScheme: 'lt://', + referenceLabel: 'an lt:// app id', + uploadFile: async (appPath, signal) => await uploadTestMuApp(appPath, options, signal), + uploadUrl: async (url, signal) => await uploadTestMuAppFromUrl(url, options, signal), + signal: options.signal, + }); +} + +/** + * Builds the W3C `alwaysMatch` capabilities for a TestMu virtual-device session. + * + * Standard Appium keys stay `appium:`-prefixed at the top level; everything TestMu-specific lives + * in `lt:options`. `isRealMobile: false` selects an emulator or simulator, and `w3c: true` keeps + * the hub on the W3C dialect agent-device speaks. `appiumVersion` is sent only when the caller + * pins one; otherwise TestMu AI starts its default server for the device. + */ +export function buildTestMuCapabilities( + options: TestMuCapabilitiesOptions, +): Record { + const { 'lt:options': configuredLtOptions, ...configured } = options.configured ?? {}; + const deviceFeatures = options.deviceFeatures ?? {}; + return { + 'appium:deviceName': options.deviceName, + 'appium:platformVersion': options.osVersion, + ...(options.app ? { 'appium:app': options.app } : {}), + ...configured, + // Merged per key, never assigned: a configured `lt:options` must not drop the labels below. + 'lt:options': { + platformName: options.platform === 'ios' ? 'iOS' : 'Android', + deviceName: options.deviceName, + platformVersion: options.osVersion, + ...(options.app ? { app: options.app } : {}), + ...(options.projectName ? { project: options.projectName } : {}), + build: options.buildName, + name: options.sessionName, + video: true, + devicelog: true, + ...deviceFeatures, + ...(asRecord(configuredLtOptions) ?? {}), + // A configured value cannot switch the device pool or drop the W3C dialect agent-device speaks. + isRealMobile: false, + w3c: true, + }, + }; +} + +export function isTestMuAppReference(value: string): boolean { + return value.startsWith('lt://'); +} + +async function fetchTestMuSessionDetails( + sessionId: string, + options: TestMuSessionDetailsOptions, +): Promise> { + const endpoint = new URL( + `${trimTrailingSlash(String(options.endpoint ?? TESTMU_API_ENDPOINT))}/sessions/${encodeURIComponent(sessionId)}`, + ); + let json: Record; + try { + json = await fetchProviderSessionDetails(endpoint, { + clientVersion: options.clientVersion, + auth: options, + service: 'TestMu AI', + }); + } catch (error) { + // Details are published a little after the session ends; until then the API answers 404. + if (error instanceof AppError && error.details?.status === 404) return {}; + throw error; + } + // The API wraps the session in a jsend envelope: `{ status, data: {...}, message }`. + const details = asRecord(json.data); + if (!details) { + throw new AppError('COMMAND_FAILED', 'TestMu AI session details response had no data.', { + response: json, + }); + } + return details; +} + +function mapTestMuArtifacts( + provider: string, + providerSessionId: string, + details: Record, +): CloudArtifact[] { + // Virtual-device sessions report the device log as `console_logs_url`. + const deviceLogField = + typeof details.console_logs_url === 'string' && details.console_logs_url.length > 0 + ? 'console_logs_url' + : 'device_logs_url'; + const fromDetails = ( + [ + ['video_url', 'video', 'Session video'], + ['appium_logs_url', 'appium-log', 'Appium logs'], + [deviceLogField, 'device-log', 'Device logs'], + ['network_logs_url', 'raw', 'Network logs'], + ['command_logs_url', 'automation-log', 'Command logs'], + ['screenshot_url', 'raw', 'Screenshots'], + ] as const + ).map(([field, kind, name]) => + urlArtifactFromDetails(provider, providerSessionId, details, field, kind, name), + ); + const dashboard: CloudArtifact = { + provider, + providerSessionId, + kind: 'provider-session', + name: 'TestMu AI dashboard', + url: `${TESTMU_DASHBOARD_TEST_URL}${encodeURIComponent(providerSessionId)}`, + availability: 'ready', + }; + const ready = fromDetails.filter((artifact): artifact is CloudArtifact => artifact !== undefined); + // The dashboard link alone does not mean the session finished uploading; keep "pending" until + // the API reports at least one artifact URL. + return ready.length > 0 ? [...ready, dashboard] : []; +} + +const TESTMU_APP_ID = /^[\w.-]+$/; + +/** The upload answers with `app_url` (`lt://…`) and/or a bare `app_id`; anything else is a failed upload. */ +function readTestMuAppReference(value: unknown): string | undefined { + const { app_url: appUrl, app_id: appId } = asRecord(value) ?? {}; + if (typeof appUrl === 'string' && isValidTestMuAppReference(appUrl)) return appUrl; + if (typeof appId !== 'string') return undefined; + const reference = isTestMuAppReference(appId) ? appId : `lt://${appId}`; + return isValidTestMuAppReference(reference) ? reference : undefined; +} + +function isValidTestMuAppReference(value: string): boolean { + return isTestMuAppReference(value) && TESTMU_APP_ID.test(value.slice('lt://'.length)); +} diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index 4b750d5439..88724187dc 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -269,7 +269,7 @@ async function readProviderJsonBody(response: Response): Promise { } } -/** `1.0` and `1` name the same OS release on BrowserStack's catalog. */ +/** `1.0` and `1` name the same OS release on BrowserStack's catalog; TestMu's hub matches spellings exactly. */ export function sameOsVersion(left: string, right: string): boolean { const normalize = (value: string) => value.replace(/(?:\.0)+$/, ''); return normalize(left) === normalize(right); diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 3f41c6c817..de24acb872 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -756,6 +756,7 @@ test('the real tree parses, declares, and passes R11', () => { assert.deepEqual([...providerWebDriverPackage.exportTargets.keys()].sort(), [ '@agent-device/provider-webdriver', '@agent-device/provider-webdriver/providers', + '@agent-device/provider-webdriver/testmu-device-features', ]); assert.deepEqual([...providerWebDriverPackage.workspaceDependencies].sort(), [ '@agent-device/capture-kit', diff --git a/src/__tests__/cloud-connect-profile.test.ts b/src/__tests__/cloud-connect-profile.test.ts index fb3f486168..3b0118d9e9 100644 --- a/src/__tests__/cloud-connect-profile.test.ts +++ b/src/__tests__/cloud-connect-profile.test.ts @@ -70,24 +70,37 @@ beforeEach(() => { }, app: { status: 'verified', reference: options.app }, } - : { - provider: 'aws-device-farm', - service: 'AWS Device Farm', - verificationMessage: 'Credentials, project, and device verified.', - project: { name: 'Agent Device', reference: options.projectArn }, - device: { - status: 'verified', - name: 'iPhone 15', - reference: options.deviceArn, - platform: options.platform, - osVersion: '17', - }, - app: { - status: 'missing', - message: - 'No app upload is attached; AWS Device Farm does not support install after allocation.', + : options.provider === 'testmu' + ? { + provider: 'testmu', + service: 'TestMu AI', + verificationMessage: 'Credentials, virtual device, and uploaded app verified.', + device: { + status: 'verified', + name: options.deviceName, + platform: options.platform, + osVersion: options.osVersion, + }, + app: { status: 'verified', reference: options.app }, + } + : { + provider: 'aws-device-farm', + service: 'AWS Device Farm', + verificationMessage: 'Credentials, project, and device verified.', + project: { name: 'Agent Device', reference: options.projectArn }, + device: { + status: 'verified', + name: 'iPhone 15', + reference: options.deviceArn, + platform: options.platform, + osVersion: '17', + }, + app: { + status: 'missing', + message: + 'No app upload is attached; AWS Device Farm does not support install after allocation.', + }, }, - }, ); }); diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts new file mode 100644 index 0000000000..1255a4af6a --- /dev/null +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -0,0 +1,192 @@ +import { afterEach, beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { connectCommand } from '../cli/commands/connection.ts'; +import { + readActiveConnectionState, + type RemoteConnectionState, +} from '../remote/remote-connection-state.ts'; +import type { AgentDeviceClient } from '../agent-device-client.ts'; +import { resolveCloudWebDriverConnectProfile } from '../cli/connection/cloud-webdriver-profile.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { providerWebDriver } from '../provider-webdriver.ts'; +import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; + +vi.mock('../provider-webdriver.ts', () => ({ + providerWebDriver: { verifyConnection: vi.fn() }, +})); + +afterEach(() => { + vi.clearAllMocks(); + vi.unstubAllEnvs(); +}); + +const mockedVerifyWebDriverConnection = vi.mocked(providerWebDriver.verifyConnection); + +beforeEach(() => { + mockedVerifyWebDriverConnection.mockImplementation(async (options) => { + assert.equal(options.provider, 'testmu'); + return { + provider: 'testmu', + service: 'TestMu AI', + verificationMessage: 'Credentials, device, and uploaded app verified.', + device: { + status: 'verified', + name: options.deviceName, + platform: options.platform, + osVersion: options.osVersion, + }, + app: { status: 'verified', reference: options.app }, + }; + }); +}); + +test('connect testmu generates a local provider profile and verifies the virtual device', async () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-'); + const stateDir = path.join(tempRoot, '.state'); + vi.stubEnv('LT_USERNAME', 'lt-user'); + vi.stubEnv('LT_ACCESS_KEY', 'lt-key'); + + try { + await connectWithGeneratedProviderProfile({ + stateDir, + positionals: ['testmu'], + flags: { + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18.0', + providerApp: 'lt://APP1', + providerBuild: 'build-a', + }, + }); + + assert.deepEqual(mockedVerifyWebDriverConnection.mock.calls[0]?.[0], { + provider: 'testmu', + username: 'lt-user', + accessKey: 'lt-key', + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18.0', + app: 'lt://APP1', + }); + const state = readRequiredActiveState(stateDir); + assert.equal(state.tenant, 'testmu'); + assert.equal(state.leaseProvider, 'testmu'); + assert.match(state.remoteConfigPath, /generated\/testmu-[a-f0-9]{16}\.json$/); + const generated = readGeneratedConfig(state.remoteConfigPath); + assert.equal(generated.providerApp, 'lt://APP1'); + assert.equal(generated.providerOsVersion, '18.0'); + assert.equal(generated.providerBuild, 'build-a'); + assert.equal(JSON.stringify(generated).includes('lt-key'), false); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test('connect testmu verifies against TESTMU_API_ENDPOINT', async () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-endpoint-'); + vi.stubEnv('LT_USERNAME', 'lt-user'); + vi.stubEnv('LT_ACCESS_KEY', 'lt-key'); + vi.stubEnv('TESTMU_API_ENDPOINT', 'https://staging.testmu.test/mobile-automation/api/v1'); + + try { + await connectWithGeneratedProviderProfile({ + stateDir: path.join(tempRoot, '.state'), + positionals: ['testmu'], + flags: { + platform: 'android', + device: 'Pixel 8', + providerOsVersion: '14', + providerApp: 'lt://APP1', + }, + }); + + const options = mockedVerifyWebDriverConnection.mock.calls[0]?.[0]; + assert.equal(options?.provider, 'testmu'); + assert.equal(options.apiEndpoint, 'https://staging.testmu.test/mobile-automation/api/v1'); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test('connect testmu rejects BrowserStack network and re-sign flags before saving a profile', () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-reject-'); + + try { + assert.throws( + () => + resolveCloudWebDriverConnectProfile({ + provider: 'testmu', + stateDir: path.join(tempRoot, '.state'), + cwd: tempRoot, + env: { LT_USERNAME: 'lt-user', LT_ACCESS_KEY: 'lt-key' }, + flags: { + json: false, + help: false, + version: false, + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18.0', + providerApp: 'lt://APP1', + providerNetworkProfile: '3g-lossy', + providerNoResignApp: true, + }, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.match(error.message, /not supported by TestMu AI/); + assert.deepEqual(error.details?.flags, [ + '--provider-network-profile', + '--provider-no-resign-app', + ]); + return true; + }, + ); + assert.equal(fs.existsSync(path.join(tempRoot, '.state')), false); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +async function connectWithGeneratedProviderProfile(options: { + stateDir: string; + positionals: string[]; + flags: Partial[0]['flags']>; +}): Promise { + const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + await connectCommand({ + positionals: options.positionals, + flags: { + json: true, + help: false, + version: false, + stateDir: options.stateDir, + ...options.flags, + }, + client: {} as AgentDeviceClient, + }); + } finally { + stdoutWrite.mockRestore(); + } +} + +function readGeneratedConfig(configPath: string): { + providerApp?: string; + providerOsVersion?: string; + providerBuild?: string; +} { + return JSON.parse(fs.readFileSync(configPath, 'utf8')) as { + providerApp?: string; + providerOsVersion?: string; + providerBuild?: string; + }; +} + +function readRequiredActiveState(stateDir: string): RemoteConnectionState { + const state = readActiveConnectionState({ stateDir }); + assert.ok(state); + return state; +} diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index ffa64093af..031d48c8c7 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -4,6 +4,7 @@ import { rejectBrowserStackOnlyDeviceFeatures, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; +import { rejectUnsupportedTestMuDeviceFeatures } from '@agent-device/provider-webdriver/testmu-device-features'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { PlatformSelector } from '@agent-device/kernel/device'; @@ -70,6 +71,10 @@ const CLOUD_WEBDRIVER_CONNECT_PROFILE_BUILDERS: readonly { provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, buildProfileFields: awsDeviceFarmProfileFields, }, + { + provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + buildProfileFields: testMuProfileFields, + }, ]; function requireConnectProfileBuilder( @@ -82,29 +87,68 @@ function requireConnectProfileBuilder( throw new AppError('INVALID_ARGS', `Unsupported cloud WebDriver provider "${provider}".`); } +/** A hosted Appium hub picks its device by exact name + OS version and installs one app reference. */ +type HubProviderProfile = { + command: string; + label: string; + credentialEnv: readonly [string, string]; + /** Scheme of the provider's own app references, e.g. `bs://` or `lt://`. */ + appScheme: string; + appHint: string; +}; + +const BROWSERSTACK_HUB_PROFILE: HubProviderProfile = { + command: 'connect browserstack', + label: 'BrowserStack', + credentialEnv: ['BROWSERSTACK_USERNAME', 'BROWSERSTACK_ACCESS_KEY'], + appScheme: 'bs://', + appHint: '', +}; + +const TESTMU_HUB_PROFILE: HubProviderProfile = { + command: 'connect testmu', + label: 'TestMu AI', + credentialEnv: ['LT_USERNAME', 'LT_ACCESS_KEY'], + appScheme: 'lt://', + appHint: '', +}; + function browserStackProfileFields(options: { flags: CliFlags; env?: EnvMap; cwd: string; }): RemoteConfigProfile { - requireEnv(options.env, 'BROWSERSTACK_USERNAME', 'connect browserstack'); - requireEnv(options.env, 'BROWSERSTACK_ACCESS_KEY', 'connect browserstack'); + return hubProviderProfileFields(BROWSERSTACK_HUB_PROFILE, options); +} + +function testMuProfileFields(options: { + flags: CliFlags; + env?: EnvMap; + cwd: string; +}): RemoteConfigProfile { + rejectUnsupportedTestMuDeviceFeatures(options.flags); + return hubProviderProfileFields(TESTMU_HUB_PROFILE, options); +} + +function hubProviderProfileFields( + hub: HubProviderProfile, + options: { flags: CliFlags; env?: EnvMap; cwd: string }, +): RemoteConfigProfile { + for (const name of hub.credentialEnv) requireEnv(options.env, name, hub.command); const platform = requireCloudWebDriverPlatform( options.flags.platform, - 'connect browserstack requires --platform ios|android.', - ); - const device = requireFlag( - options.flags.device, - 'connect browserstack requires --device .', + `${hub.command} requires --platform ios|android.`, ); + const device = requireFlag(options.flags.device, `${hub.command} requires --device .`); const providerOsVersion = requireFlag( options.flags.providerOsVersion, - 'connect browserstack requires --provider-os-version .', + `${hub.command} requires --provider-os-version .`, ); - const providerApp = normalizeBrowserStackAppReference( + const providerApp = normalizeHubAppReference( + hub, requireFlag( options.flags.providerApp, - 'connect browserstack requires --provider-app .', + `${hub.command} requires --provider-app ${hub.appHint}.`, ), options.cwd, ); @@ -120,15 +164,15 @@ function browserStackProfileFields(options: { }; } -function normalizeBrowserStackAppReference(app: string, cwd: string): string { - if (app.startsWith('bs://') || /^https?:\/\//i.test(app)) return app; +function normalizeHubAppReference(hub: HubProviderProfile, app: string, cwd: string): string { + if (app.startsWith(hub.appScheme) || /^https?:\/\//i.test(app)) return app; const resolvedPath = path.resolve(cwd, app); try { if (fs.statSync(resolvedPath).isFile()) return resolvedPath; } catch { // Report one stable profile error below. } - throw new AppError('INVALID_ARGS', `BrowserStack app file not found: ${resolvedPath}`); + throw new AppError('INVALID_ARGS', `${hub.label} app file not found: ${resolvedPath}`); } function awsDeviceFarmProfileFields(options: { diff --git a/src/cli/connection/connect-provider-adapters.ts b/src/cli/connection/connect-provider-adapters.ts index 27d9c5c564..a83c548365 100644 --- a/src/cli/connection/connect-provider-adapters.ts +++ b/src/cli/connection/connect-provider-adapters.ts @@ -69,6 +69,10 @@ const CONNECT_PROVIDER_ADAPTERS = { resolveCloudWebDriverConnectProfile({ provider: 'aws-device-farm', ...context }), verify: verifyAwsDeviceFarm, }, + testmu: { + resolve: (context) => resolveCloudWebDriverConnectProfile({ provider: 'testmu', ...context }), + verify: verifyTestMu, + }, limrun: { resolve: resolveLimrunConnectProfile, verify: verifyLimrun, @@ -153,6 +157,25 @@ async function verifyBrowserStack( }); } +async function verifyTestMu( + context: Pick, +): Promise { + const { flags, env } = context; + return await providerWebDriver.verifyConnection({ + provider: 'testmu', + username: requiredResolvedValue(env.LT_USERNAME, 'TestMu AI profile missed LT_USERNAME.'), + accessKey: requiredResolvedValue(env.LT_ACCESS_KEY, 'TestMu AI profile missed LT_ACCESS_KEY.'), + platform: requiredResolvedPlatform(flags.platform, 'TestMu AI'), + deviceName: requiredResolvedValue(flags.device, 'TestMu AI profile missed device.'), + osVersion: requiredResolvedValue( + flags.providerOsVersion, + 'TestMu AI profile missed OS version.', + ), + app: requiredResolvedValue(flags.providerApp, 'TestMu AI profile missed app.'), + ...(env.TESTMU_API_ENDPOINT ? { apiEndpoint: env.TESTMU_API_ENDPOINT } : {}), + }); +} + async function verifyAwsDeviceFarm( context: Pick, ): Promise { diff --git a/src/cli/connection/provider-policy.ts b/src/cli/connection/provider-policy.ts index ebb86bc6d4..1756751af5 100644 --- a/src/cli/connection/provider-policy.ts +++ b/src/cli/connection/provider-policy.ts @@ -33,6 +33,7 @@ export function connectProviderNamesForError(): string { 'proxy', CLOUD_WEBDRIVER_PROVIDERS.browserStack, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + CLOUD_WEBDRIVER_PROVIDERS.testMu, 'limrun', ].join(', '); } diff --git a/src/commands/management/artifacts.ts b/src/commands/management/artifacts.ts index 11cea82c1b..1fbb683529 100644 --- a/src/commands/management/artifacts.ts +++ b/src/commands/management/artifacts.ts @@ -11,7 +11,9 @@ const artifactsCommandMetadata = defineFieldCommandMetadata( 'artifacts', 'List daemon or cloud provider artifacts for an active or completed session.', { - provider: stringField('Cloud provider name, for example browserstack or aws-device-farm.'), + provider: stringField( + 'Cloud provider name, for example browserstack, aws-device-farm, or testmu.', + ), providerSessionId: stringField('Cloud provider session id or ARN.'), }, ); diff --git a/src/commands/schema/cli-help-topics.test.ts b/src/commands/schema/cli-help-topics.test.ts index 2364272b00..cc92b8b6e7 100644 --- a/src/commands/schema/cli-help-topics.test.ts +++ b/src/commands/schema/cli-help-topics.test.ts @@ -436,7 +436,17 @@ test('usageForCommand resolves remote help topic', async () => { assert.match(help, /AGENT_DEVICE_HTTP_AUTH_HOOK configured treats HTTP requests as remote/); assert.match(help, /host-path install sources are rejected/); assert.match(help, /uploaded artifacts remain supported/); - assert.match(help, /Limrun, BrowserStack, and AWS Device Farm through local provider profiles/); + assert.match( + help, + /Limrun, BrowserStack, AWS Device Farm, and TestMu AI through local provider profiles/, + ); + assert.match(help, /TestMu AI uses LT_USERNAME and LT_ACCESS_KEY/); + const testMuFlow = help.slice( + help.indexOf('TestMu AI virtual-device flow'), + help.indexOf('BrowserStack hosted-device flow'), + ); + assert.match(testMuFlow, /--device "iPhone 16" --provider-os-version 18\.0/); + assert.match(testMuFlow, /agent-device disconnect/); assert.match(help, /Limrun uses LIMRUN_API_KEY/); assert.match(help, /BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY/); assert.match(help, /Generated connection profiles store app\/device selectors and ARNs/); diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 24957278e4..1211c54f88 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -577,17 +577,18 @@ Providers: Direct proxy: agent-device connect proxy --daemon-base-url stores the shared proxy profile and client identity. BrowserStack: agent-device connect browserstack verifies credentials, the exact device, and a bs:// app reference, then stores a local provider profile. It does not create an App Automate session. AWS Device Farm: agent-device connect aws-device-farm verifies credentials and the exact project, device, and optional app upload, then stores a local provider profile. It does not create a remote access session. + TestMu AI: agent-device connect testmu verifies credentials, the exact virtual device (emulator or simulator) and OS version, and an lt:// app reference, then stores a local provider profile. It does not create a hub session. Limrun: agent-device connect limrun verifies access to the selected iOS or Android instance service, then stores a local provider profile. It does not create an instance. After direct-provider connect: Read the printed Device, App, Next, and workflow-note lines. They are also available as verification/device/app/liveSession/nextSteps/notes in --json output. - BrowserStack and AWS Device Farm create the hosted session on open. open needs the installed package or bundle identifier, not the app artifact name or ARN. + BrowserStack, AWS Device Farm, and TestMu AI create the hosted session on open. open needs the installed package or bundle identifier, not the app artifact name, ARN, or lt:// id. Before provider allocation, apps lists compatible uploaded app assets without creating an instance when the selected provider exposes a catalog. open creates the instance with that asset, resolves its installed app id, and launches it. install remains available when the app comes from a fresh local path or URL. AWS Device Farm cannot install after allocation. If connect reports no attached app, run its printed reconnect command, which includes --session --force, before open. Do not run devices as a pre-open catalog probe for direct providers; it can allocate the deferred provider session. Limrun is the exception for apps: before allocation it lists uploaded assets for the selected platform. Device cloud interfaces: - CLI is the canonical bootstrap path: connect limrun/browserstack/aws-device-farm, then use normal open/snapshot/click/close/artifacts/disconnect commands. + CLI is the canonical bootstrap path: connect limrun/browserstack/aws-device-farm/testmu, then use normal open/snapshot/click/close/artifacts/disconnect commands. JavaScript can skip persisted connect state by passing leaseProvider plus provider fields to createAgentDeviceClient or per-command options. MCP exposes operational tools such as open, snapshot, click, close, and artifacts. It does not expose connect/disconnect; run CLI connect first in the same state dir before relying on MCP tools. @@ -617,6 +618,15 @@ Cloud profile flow: agent-device snapshot agent-device disconnect +TestMu AI virtual-device flow (emulators and simulators): + LT_USERNAME=... LT_ACCESS_KEY=... + agent-device connect testmu --platform ios --device "iPhone 16" --provider-os-version 18.0 --provider-app lt://APP-id + agent-device open com.example.app + agent-device snapshot -i + agent-device close + agent-device artifacts --json + agent-device disconnect + BrowserStack hosted-device flow: BROWSERSTACK_USERNAME=... BROWSERSTACK_ACCESS_KEY=... agent-device connect browserstack --platform android --device "Google Pixel 8" --provider-os-version 14.0 --provider-app bs://app-id @@ -663,12 +673,12 @@ Rules: Use connect without --remote-config when the cloud control plane owns the connection profile. Prefer connect --remote-config over --daemon-base-url, --tenant, --run-id, and --lease-id when using a local profile. Use agent-device proxy for direct tunnel access to a Mac you control. Expose the printed proxy URL through cloudflared/ngrok, then run agent-device connect proxy with the tunnel URL and printed token before normal commands. - Use Limrun, BrowserStack, and AWS Device Farm through local provider profiles; they do not accept a remote agent-device daemon URL. - Device cloud credentials must be available before the command starts. Limrun uses LIMRUN_API_KEY. BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY. AWS Device Farm uses the AWS CLI credential chain, including CI-provided AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN, AWS profiles, or web identity role variables. + Use Limrun, BrowserStack, AWS Device Farm, and TestMu AI through local provider profiles; they do not accept a remote agent-device daemon URL. + Device cloud credentials must be available before the command starts. Limrun uses LIMRUN_API_KEY. BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY. TestMu AI uses LT_USERNAME and LT_ACCESS_KEY. AWS Device Farm uses the AWS CLI credential chain, including CI-provided AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN, AWS profiles, or web identity role variables. Direct-provider connect performs read-only provider calls and saves active connection state only after verification succeeds. It never creates a device, instance, App Automate session, or AWS remote access session. connect without --session always creates a fresh remote session and prints that session in its next-step commands. Concurrent callers must pass the returned --session on every command; the ambient active connection is only a single-workflow convenience. To replace an existing connection, pass its returned session explicitly with --session --force. --force without --session creates another fresh session and does not release or overwrite an unrelated active connection. - Prefer short-lived AWS role credentials in CI. Generated connection profiles store app/device selectors and ARNs, not Limrun API keys, BrowserStack access keys, or AWS credentials. + Prefer short-lived AWS role credentials in CI. Generated connection profiles store app/device selectors and ARNs, not Limrun API keys, BrowserStack or TestMu AI access keys, or AWS credentials. Limrun Android supports direct ADB port reverse for local Metro. Limrun iOS requires a public Metro/React DevTools URL because it cannot reach local host ports directly. After closing a device cloud session, run agent-device artifacts --json to retrieve provider video/log/dashboard URLs when the provider has made them available. connect proxy stores the connection profile and client identity. Proxy device leases are acquired on open and expire after five minutes without commands; devices may inspect proxy inventory without allocating. diff --git a/src/commands/schema/command-overrides.ts b/src/commands/schema/command-overrides.ts index 728c32ffd1..f69ccda125 100644 --- a/src/commands/schema/command-overrides.ts +++ b/src/commands/schema/command-overrides.ts @@ -64,7 +64,7 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { 'Configure remote access without allocating a device. Direct providers validate credentials/resources before saving state and print the exact device/app preparation needed before open. AGENT_DEVICE_CLOUD_BASE_URL is the bridge/control-plane API origin; use AGENT_DEVICE_DAEMON_AUTH_TOKEN=adc_live_... for CI/service-token automation.', }, usageOverride: - 'connect [cloud|proxy|limrun|browserstack|aws-device-farm] [--remote-config ] [--daemon-base-url ] [--tenant ] [--run-id ] [--lease-id ] [--lease-backend ] [--force] [--no-login]', + 'connect [cloud|proxy|limrun|browserstack|aws-device-farm|testmu] [--remote-config ] [--daemon-base-url ] [--tenant ] [--run-id ] [--lease-id ] [--lease-backend ] [--force] [--no-login]', usageFlags: [], listUsageOverride: 'connect', positionalArgs: ['provider?'], From 95c21f7fc910cfaabd8dc17347c5ace74b840993 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 4/6] feat(provider-webdriver): support TestMu AI real devices TestMu AI serves real devices and virtual devices from the same Appium hub; `lt:options.isRealMobile` selects the pool. Add `--provider-device-type real|virtual` to choose it. The default is `virtual`, so existing `testmu` profiles and sessions keep their current behaviour. The value is a cloud provider profile field like the others. It is defined in contracts as PROVIDER_DEVICE_TYPES and carried through the lease_allocate projection, request overrides, the remote-config schema, `connect`, the CLI request flags, and the session doctor's provider keys. It reaches session preparation from a `connect testmu` profile and from `client.leases.allocate({ providerDeviceType })`. For `real`, TestMu AI sessions: - Set `isRealMobile: true`. It is applied after any configured `lt:options`, so configuration cannot switch pools. - Upload through the real-device upload API. It has its own override, TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT; TESTMU_APP_UPLOAD_ENDPOINT keeps redirecting virtual-device uploads only. An `.app` directory is refused with a hint to pass a signed .ipa. - Verify against the real-device catalog (`capability/generator?isVirtualDevice=false`). It lists devices directly under the platform key rather than under `app.devices`. The OS version must still match exactly, and real iOS devices are listed by major version, such as `18`. A catalog response without the expected pool shape fails with a typed error. - Look an lt:// id up in the real-device app list (`type=android` or `type=ios`), the same way virtual uploads are looked up under `emulator` or `simulator`. BrowserStack, AWS Device Farm, and Limrun refuse the flag at connect. BrowserStack and AWS Device Farm also refuse it at session preparation, which the typed client and hand-written profiles reach without `connect`. Co-Authored-By: Claude Opus 5.5 --- .fallowrc.json | 6 +- .../src/flag-definitions-connection.ts | 16 +- packages/command-registry/src/flag-groups.ts | 1 + .../src/__tests__/lease-scope.test.ts | 2 + packages/contracts/src/client-connection.ts | 1 + packages/contracts/src/facades/remote.ts | 3 +- packages/contracts/src/lease-scope.ts | 1 + .../contracts/src/remote-config-fields.ts | 5 + .../src/connection-verification.test.ts | 165 +++++++++++++++++- .../src/connection-verification.ts | 7 +- .../src/provider-definitions.ts | 35 +++- .../src/testmu-connection-verification.ts | 59 ++++--- .../src/testmu-device-features.test.ts | 34 ++++ .../src/testmu-device-features.ts | 41 ++++- .../provider-webdriver/src/testmu.test.ts | 76 ++++++-- packages/provider-webdriver/src/testmu.ts | 27 ++- scripts/integration-progress-model.ts | 1 + src/__tests__/client-leases.test.ts | 35 ++++ src/__tests__/cloud-connect-testmu.test.ts | 116 ++++++++++++ src/cli.ts | 1 + src/cli/commands/connection-runtime.ts | 1 + src/cli/connection/cloud-webdriver-profile.ts | 12 +- .../connection/connect-provider-adapters.ts | 1 + src/cli/connection/limrun-profile.ts | 2 + .../args-parse-provider-device-type.test.ts | 17 ++ src/commands/command-flags.ts | 1 + src/commands/schema/cli-help-topics.test.ts | 5 + src/commands/schema/cli-help.ts | 6 +- src/commands/schema/command-overrides.ts | 1 + src/daemon-client/daemon-client-rpc.test.ts | 2 + src/daemon/handlers/session-doctor-options.ts | 1 + src/remote/remote-config-schema.ts | 2 + .../cloud-webdriver-provider-adapters.test.ts | 81 +++++++++ .../daemon-http-lease-allocate.test.ts | 2 + 34 files changed, 696 insertions(+), 70 deletions(-) create mode 100644 src/__tests__/client-leases.test.ts create mode 100644 src/cli/parser/__tests__/args-parse-provider-device-type.test.ts diff --git a/.fallowrc.json b/.fallowrc.json index 494c2f42f2..865583d4d3 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -382,7 +382,11 @@ { "comment": "TestMu session preparation reads these off the lazy loadTestMuDeviceFeatures() import in packages/provider-webdriver/src/provider-definitions.ts, which keeps the package entry's eager closure unchanged; Fallow cannot connect the dynamic member reads.", "file": "packages/provider-webdriver/src/testmu-device-features.ts", - "exports": ["buildTestMuDeviceFeatureCapabilities", "readTestMuDeviceFeatureFields"] + "exports": [ + "buildTestMuDeviceFeatureCapabilities", + "readTestMuDeviceFeatureFields", + "readTestMuDeviceType" + ] }, { "comment": "Converting the contracts façades from `export *` to explicit named re-exports (the pin-table retirement) made these individually visible to --production analysis for the first time; a bare star previously hid them from this exact check. isRecord/IOS_SAFARI_BUNDLE_ID/REPLAY_DIVERGENCE_* have no production consumer. Kept rather than narrowed here so the façade's re-export surface stays byte-identical to the symbol set the retired pin table asserted — narrowing the surface is a follow-up with its own review, not a side effect of this mechanical conversion.", diff --git a/packages/command-registry/src/flag-definitions-connection.ts b/packages/command-registry/src/flag-definitions-connection.ts index 0c9a34b0f1..dd698ce7e3 100644 --- a/packages/command-registry/src/flag-definitions-connection.ts +++ b/packages/command-registry/src/flag-definitions-connection.ts @@ -1,4 +1,7 @@ -import { PROVIDER_DEVICE_ORIENTATIONS } from '@agent-device/contracts/remote'; +import { + PROVIDER_DEVICE_ORIENTATIONS, + PROVIDER_DEVICE_TYPES, +} from '@agent-device/contracts/remote'; import type { FlagDefinition } from './flag-types.ts'; export const CONNECTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ @@ -174,6 +177,17 @@ export const CONNECTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ projectConfig: false, recorded: false, }, + { + key: 'providerDeviceType', + names: ['--provider-device-type'], + type: 'enum', + enumValues: PROVIDER_DEVICE_TYPES, + usageLabel: '--provider-device-type real|virtual', + usageDescription: + 'TestMu AI device pool: real devices or virtual devices (emulators and simulators). Defaults to virtual', + projectConfig: false, + recorded: false, + }, { key: 'providerProject', names: ['--provider-project'], diff --git a/packages/command-registry/src/flag-groups.ts b/packages/command-registry/src/flag-groups.ts index c8dc16507d..f3f4ae1804 100644 --- a/packages/command-registry/src/flag-groups.ts +++ b/packages/command-registry/src/flag-groups.ts @@ -78,6 +78,7 @@ export const COMMON_COMMAND_SUPPORTED_FLAG_KEYS = flagKeys( 'device', 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/packages/contracts/src/__tests__/lease-scope.test.ts b/packages/contracts/src/__tests__/lease-scope.test.ts index 3da34f2be7..2aa8ed53df 100644 --- a/packages/contracts/src/__tests__/lease-scope.test.ts +++ b/packages/contracts/src/__tests__/lease-scope.test.ts @@ -182,6 +182,7 @@ test('readLeaseAllocateProviderFlags carries the provider-allocation flags and d device: 'iPhone 15', providerApp: 'bs://abc', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-1', providerSessionName: 'smoke', @@ -198,6 +199,7 @@ test('readLeaseAllocateProviderFlags carries the provider-allocation flags and d device: 'iPhone 15', providerApp: 'bs://abc', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-1', providerSessionName: 'smoke', diff --git a/packages/contracts/src/client-connection.ts b/packages/contracts/src/client-connection.ts index 7568768a33..a0eee1368b 100644 --- a/packages/contracts/src/client-connection.ts +++ b/packages/contracts/src/client-connection.ts @@ -63,6 +63,7 @@ export type AgentDeviceRequestOverrides = Pick< | 'clientId' | 'providerApp' | 'providerOsVersion' + | 'providerDeviceType' | 'providerProject' | 'providerBuild' | 'providerSessionName' diff --git a/packages/contracts/src/facades/remote.ts b/packages/contracts/src/facades/remote.ts index 852c8272cb..bfaecc5086 100644 --- a/packages/contracts/src/facades/remote.ts +++ b/packages/contracts/src/facades/remote.ts @@ -14,10 +14,11 @@ export type { ProviderConnectionResource, ProviderConnectionVerification, } from '../provider-connection.ts'; -export { PROVIDER_DEVICE_ORIENTATIONS } from '../remote-config-fields.ts'; +export { PROVIDER_DEVICE_ORIENTATIONS, PROVIDER_DEVICE_TYPES } from '../remote-config-fields.ts'; export type { CloudProviderProfileFields, ProviderDeviceOrientation, + ProviderDeviceType, RemoteConfigMetroOptions, RemoteConnectionProfileFields, } from '../remote-config-fields.ts'; diff --git a/packages/contracts/src/lease-scope.ts b/packages/contracts/src/lease-scope.ts index 51bb05550c..9f27b661cc 100644 --- a/packages/contracts/src/lease-scope.ts +++ b/packages/contracts/src/lease-scope.ts @@ -212,6 +212,7 @@ const LEASE_ALLOCATE_PROVIDER_FLAG_KEYS = [ // The Cloud provider profile fields; pinned exhaustive against that vocabulary below. 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/packages/contracts/src/remote-config-fields.ts b/packages/contracts/src/remote-config-fields.ts index 5ab0b6a6b2..a2fe91b62d 100644 --- a/packages/contracts/src/remote-config-fields.ts +++ b/packages/contracts/src/remote-config-fields.ts @@ -20,9 +20,14 @@ import type { MetroPrepareKind } from './metro.ts'; export const PROVIDER_DEVICE_ORIENTATIONS = ['portrait', 'landscape'] as const; export type ProviderDeviceOrientation = (typeof PROVIDER_DEVICE_ORIENTATIONS)[number]; +/** Device pool a hosted provider session is created in: physical devices or emulators/simulators. */ +export const PROVIDER_DEVICE_TYPES = ['real', 'virtual'] as const; +export type ProviderDeviceType = (typeof PROVIDER_DEVICE_TYPES)[number]; + export type CloudProviderProfileFields = { providerApp?: string; providerOsVersion?: string; + providerDeviceType?: ProviderDeviceType; providerProject?: string; providerBuild?: string; providerSessionName?: string; diff --git a/packages/provider-webdriver/src/connection-verification.test.ts b/packages/provider-webdriver/src/connection-verification.test.ts index 5b2d872493..fbd3978638 100644 --- a/packages/provider-webdriver/src/connection-verification.test.ts +++ b/packages/provider-webdriver/src/connection-verification.test.ts @@ -431,17 +431,134 @@ test('TestMu defers an lt:// reference it cannot find and a local path it will u }); }); -// The listing is keyed by runtime: an Android emulator upload is listed under `emulator` and an -// iOS simulator upload under `simulator`, never under the platform name. -test('TestMu checks an lt:// id against the virtual-device app list of its platform', async () => { +// The real-device catalog is keyed by platform at the top level, not under `app.devices`. +const testMuRealCatalog = { + android: { + brands: { + Google: [ + { name: 'Pixel 6', osVersion: ['12', '13', '14', '15', '16'] }, + { name: 'Pixel 8', osVersion: ['14'] }, + ], + }, + }, + ios: { + brands: { + Apple: [ + { name: 'iPhone 16', osVersion: ['18'] }, + { name: 'iPhone 15', osVersion: ['17', '18', '26'] }, + ], + }, + }, + roku: { brands: {} }, + tvos: { brands: {} }, +}; + +test('TestMu verifies a real device against the real-device catalog shape', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuRealCatalog) + : jsonResponse({ data: [{ app_id: 'APP1', name: 'MyApp.ipa' }], metaData: { total: 1 } }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...defaultCatalog } = testMuOptions; + + const result = await createProvider().verifyConnection({ + ...defaultCatalog, + deviceType: 'real', + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18', + }); + + assert.equal(result.verificationMessage, 'Credentials, real device, and uploaded app verified.'); + assert.deepEqual(result.device, { + status: 'verified', + name: 'iPhone 16', + platform: 'ios', + osVersion: '18', + }); + assert.equal( + String(fetchMock.mock.calls[0]?.[0]), + 'https://mobile-api.lambdatest.com/mobile-automation/api/v1/capability/generator?isVirtualDevice=false', + ); + + const android = await createProvider().verifyConnection({ + ...testMuOptions, + deviceType: 'real', + deviceName: 'Pixel 6', + osVersion: '14', + }); + assert.equal(android.device.name, 'Pixel 6'); +}); + +// Real iOS devices are listed by major version, so `18.0` is the wrong spelling for the real pool. +test('TestMu matches real-device OS versions exactly and lists what the device offers', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuRealCatalog)), + ); + const realIos = { + ...testMuOptions, + deviceType: 'real' as const, + platform: 'ios' as const, + deviceName: 'iPhone 15', + }; + await assert.rejects( + createProvider().verifyConnection({ ...realIos, deviceName: 'iPhone 16', osVersion: '18.0' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); + assert.match( + error.message, + /TestMu AI real device "iPhone 16" with ios 18\.0 is not available/, + ); + assert.match(error.message, /iPhone 16 offers 18\.$/); + return true; + }, + ); + await assert.rejects( + createProvider().verifyConnection({ ...realIos, osVersion: '16' }), + /iPhone 15 offers 17, 18, 26/, + ); +}); + +test('TestMu fails typed when a catalog does not have the selected pool shape', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, deviceType: 'real' }), + (error: unknown) => + error instanceof Error && + (error as { code?: string }).code === 'COMMAND_FAILED' && + /real-device catalog response did not list devices/.test(error.message), + ); + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuRealCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection(testMuOptions), + /virtual-device catalog response did not list devices/, + ); +}); + +// The listing is keyed by pool: `emulator`/`simulator` hold virtual uploads, `android`/`ios` real +// ones, so an id must be looked up in the list of the pool the session will run on. +test('TestMu checks an lt:// id against the app list of the selected pool and platform', async () => { const cases = [ - { platform: 'android', deviceName: 'Pixel 8', osVersion: '14', listType: 'emulator' }, - { platform: 'ios', deviceName: 'iPhone 16', osVersion: '18.0', listType: 'simulator' }, + { deviceType: 'virtual', platform: 'android', deviceName: 'Pixel 8', listType: 'emulator' }, + { deviceType: 'virtual', platform: 'ios', deviceName: 'iPhone 16', listType: 'simulator' }, + { deviceType: 'real', platform: 'android', deviceName: 'Pixel 6', listType: 'android' }, + { deviceType: 'real', platform: 'ios', deviceName: 'iPhone 16', listType: 'ios' }, ] as const; - for (const { platform, deviceName, osVersion, listType } of cases) { + for (const { deviceType, platform, deviceName, listType } of cases) { const fetchMock = vi.fn(async (input) => { const url = String(input); - if (url.includes('capability/generator')) return jsonResponse(testMuCatalog); + if (url.includes('capability/generator')) { + return jsonResponse(deviceType === 'real' ? testMuRealCatalog : testMuCatalog); + } return jsonResponse({ data: new URL(url).searchParams.get('type') === listType ? [{ app_id: 'APP1' }] : [], }); @@ -449,11 +566,19 @@ test('TestMu checks an lt:// id against the virtual-device app list of its platf vi.stubGlobal('fetch', fetchMock); const result = await createProvider().verifyConnection({ ...testMuOptions, + deviceType, platform, deviceName, - osVersion, + osVersion: + deviceType === 'real' + ? platform === 'ios' + ? '18' + : '14' + : platform === 'ios' + ? '18.0' + : '14', }); - assert.equal(result.app.status, 'verified', platform); + assert.equal(result.app.status, 'verified', `${deviceType} ${platform}`); assert.equal( String(fetchMock.mock.calls[1]?.[0]), `https://testmu.test/app/data?type=${listType}&level=user`, @@ -461,6 +586,28 @@ test('TestMu checks an lt:// id against the virtual-device app list of its platf } }); +test('TestMu defers a real-device lt:// id missing from the real-device app list', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuRealCatalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const result = await createProvider().verifyConnection({ + ...testMuOptions, + deviceType: 'real', + deviceName: 'Pixel 6', + }); + assert.equal(result.app.status, 'configured'); + assert.match(String(result.app.message), /not found among your real-device uploads/); + assert.equal( + result.verificationMessage, + 'Credentials and real device verified; app availability is checked when the session is created.', + ); +}); + function createProvider(runHostCommand: RunHostCommand = vi.fn()) { return createProviderWebDriver({ clientVersion: '1.2.3', runHostCommand }); } diff --git a/packages/provider-webdriver/src/connection-verification.ts b/packages/provider-webdriver/src/connection-verification.ts index 505fb2cf09..7e8685da1b 100644 --- a/packages/provider-webdriver/src/connection-verification.ts +++ b/packages/provider-webdriver/src/connection-verification.ts @@ -1,5 +1,8 @@ import type { ProviderWebDriverDependencies } from './dependencies.ts'; -import type { ProviderConnectionVerification } from '@agent-device/contracts/remote'; +import type { + ProviderConnectionVerification, + ProviderDeviceType, +} from '@agent-device/contracts/remote'; import { verifyAwsDeviceFarmConnection } from './aws-device-farm-connection-verification.ts'; import { verifyBrowserStackConnection } from './browserstack-connection-verification.ts'; @@ -38,6 +41,8 @@ export type CloudWebDriverConnectionVerificationOptions = | (HubSelectionVerificationOptions & { provider: 'browserstack' }) | (HubSelectionVerificationOptions & { provider: 'testmu'; + /** Defaults to `virtual`. */ + deviceType?: ProviderDeviceType; /** Base of the catalog API, as `TESTMU_API_ENDPOINT` sets it for the runtime. */ apiEndpoint?: string | URL; }) diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index 06e147c5eb..dbc2092f7a 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -1,5 +1,6 @@ import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; +import type { ProviderDeviceType } from '@agent-device/contracts/remote'; import { AppError } from '@agent-device/kernel/errors'; import type { ProviderWebDriverDependencies } from './dependencies.ts'; import { @@ -48,6 +49,7 @@ export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtif BROWSERSTACK_APP_UPLOAD_ENDPOINT?: string; TESTMU_WEBDRIVER_ENDPOINT?: string; TESTMU_APP_UPLOAD_ENDPOINT?: string; + TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT?: string; AGENT_DEVICE_AWS_DEVICE_FARM_PROJECT_ARN?: string; AWS_DEVICE_FARM_PROJECT_ARN?: string; AGENT_DEVICE_AWS_DEVICE_FARM_DEVICE_ARN?: string; @@ -57,15 +59,15 @@ export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtif }; /** - * TestMu (formerly LambdaTest) virtual devices: emulators and simulators behind one Appium hub. - * Only what `createRuntime` needs synchronously lives here; the session, upload, and artifact - * code loads on first use so the package entry stays as lean as it was. + * TestMu (formerly LambdaTest) real devices, and virtual devices (emulators and simulators), behind + * one Appium hub. Only what `createRuntime` needs synchronously lives here; the session, + * upload, and artifact code loads on first use so the package entry stays as lean as it was. */ const TESTMU_WEBDRIVER_ENDPOINT = 'https://mobile-hub.lambdatest.com/wd/hub/'; const TESTMU_CAPABILITY_OVERRIDES = { install: { support: 'partial', - note: 'Local app artifacts are uploaded to TestMu AI as virtual-device apps (lt://), then installed with Appium.', + note: 'Local app artifacts are uploaded to TestMu AI as real- or virtual-device apps (lt://), then installed with Appium.', }, portReverse: { support: 'unsupported', @@ -123,6 +125,10 @@ export function createCloudWebDriverProviderDefinitions( }, prepareSession: async ({ req, lease, base }) => { const request = requireRequest(req, 'BrowserStack'); + (await loadTestMuDeviceFeatures()).rejectTestMuOnlyProviderFlags( + request.flags, + CLOUD_WEBDRIVER_PROVIDERS.browserStack, + ); const username = requireEnv(env, 'BROWSERSTACK_USERNAME', 'BrowserStack'); const accessKey = requireEnv(env, 'BROWSERSTACK_ACCESS_KEY', 'BrowserStack'); const platform = requireRequestPlatform(request, 'BrowserStack'); @@ -229,6 +235,10 @@ export function createCloudWebDriverProviderDefinitions( request.flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, ); + (await loadTestMuDeviceFeatures()).rejectTestMuOnlyProviderFlags( + request.flags, + CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + ); const platform = requireRequestPlatform(request, 'AWS Device Farm'); const sessionOptions = { client: createAwsCliDeviceFarmClient({ @@ -295,9 +305,12 @@ export function createCloudWebDriverProviderDefinitions( const { buildTestMuDeviceFeatureCapabilities, readTestMuDeviceFeatureFields, + readTestMuDeviceType, rejectUnsupportedTestMuDeviceFeatures, } = await loadTestMuDeviceFeatures(); rejectUnsupportedTestMuDeviceFeatures(request.flags); + const deviceType = readTestMuDeviceType(request.flags); + const uploadEndpoint = testMuAppUploadEndpoint(env, deviceType); const credentials = requireTestMuCredentials(env, 'TestMu AI'); const platform = requireRequestPlatform(request, 'TestMu AI'); const deviceName = requireFlag( @@ -313,7 +326,8 @@ export function createCloudWebDriverProviderDefinitions( const upload = { clientVersion: dependencies.clientVersion, ...credentials, - endpoint: env.TESTMU_APP_UPLOAD_ENDPOINT, + deviceType, + endpoint: uploadEndpoint, }; const app = await resolveTestMuAppReference( requireFlag( @@ -331,6 +345,7 @@ export function createCloudWebDriverProviderDefinitions( uploadApp: createTestMuUploadApp(upload), webdriverCapabilities: buildTestMuCapabilities({ platform, + deviceType, deviceName, osVersion, app, @@ -374,6 +389,16 @@ function requireTestMuCredentials( }; } +/** Each pool has its own upload API, so each has its own override. */ +function testMuAppUploadEndpoint( + env: DefaultCloudWebDriverProviderRuntimeEnv, + deviceType: ProviderDeviceType, +): string | undefined { + return deviceType === 'real' + ? env.TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT + : env.TESTMU_APP_UPLOAD_ENDPOINT; +} + function requireRequest( req: LeaseLifecycleContext | undefined, providerLabel: string, diff --git a/packages/provider-webdriver/src/testmu-connection-verification.ts b/packages/provider-webdriver/src/testmu-connection-verification.ts index 91d913ce23..f54c28358f 100644 --- a/packages/provider-webdriver/src/testmu-connection-verification.ts +++ b/packages/provider-webdriver/src/testmu-connection-verification.ts @@ -6,21 +6,24 @@ import type { CloudWebDriverConnectionVerification, CloudWebDriverConnectionVerificationOptions, } from './connection-verification.ts'; -import type { ProviderConnectionResource } from '@agent-device/contracts/remote'; +import type { + ProviderConnectionResource, + ProviderDeviceType, +} from '@agent-device/contracts/remote'; type TestMuOptions = Extract; type TestMuAuth = { username: string; accessKey: string }; -/** `/app/data?type=` keys virtual-device uploads by runtime, not by platform. */ -const TESTMU_APP_LIST_TYPES: Record<'android' | 'ios', string> = { - android: 'emulator', - ios: 'simulator', +/** `/app/data?type=` keys uploads by pool: real-device apps by platform, virtual ones by runtime. */ +const TESTMU_APP_LIST_TYPES: Record> = { + real: { android: 'android', ios: 'ios' }, + virtual: { android: 'emulator', ios: 'simulator' }, }; /** - * Verifies a TestMu virtual-device selection without creating a session: the public capability - * catalog confirms the device/OS pair exists in the emulator and simulator pool, and the + * Verifies a TestMu device selection without creating a session: the public capability catalog + * of the selected pool (real or virtual) confirms the device/OS pair exists, and the * authenticated app listing confirms the credentials and, for an `lt://` reference, the upload. */ export async function verifyTestMuConnection( @@ -28,15 +31,17 @@ export async function verifyTestMuConnection( clientVersion: string, ): Promise { const auth = { username: options.username, accessKey: options.accessKey }; + const deviceType = options.deviceType ?? 'virtual'; const catalogUrl = options.devicesEndpoint ? new URL(options.devicesEndpoint) : apiUrl(options.apiEndpoint ?? TESTMU_API_ENDPOINT, 'capability/generator'); - catalogUrl.searchParams.set('isVirtualDevice', 'true'); + catalogUrl.searchParams.set('isVirtualDevice', String(deviceType === 'virtual')); const catalog = await fetchTestMuJson(catalogUrl, undefined, clientVersion); - const namedDevices = readTestMuVirtualDevices(catalog, options.platform).filter( + const namedDevices = readTestMuCatalogDevices(catalog, options.platform, deviceType).filter( (device) => device.name === options.deviceName, ); - // Exact match on purpose: the hub rejects `18` for a device the catalog lists as `18.0`. + // Exact match on purpose: the hub rejects `18` for a virtual device the catalog lists as `18.0`, + // and real iOS devices are listed by major version only. const matchedDevice = namedDevices.find((device) => device.osVersions.includes(options.osVersion), ); @@ -46,24 +51,25 @@ export async function verifyTestMuConnection( ); throw new AppError( 'INVALID_ARGS', - `TestMu AI virtual device "${options.deviceName}" with ${options.platform} ${options.osVersion} is not available${ + `TestMu AI ${deviceType} device "${options.deviceName}" with ${options.platform} ${options.osVersion} is not available${ offered.length > 0 ? `; ${options.deviceName} offers ${offered.join(', ')}` : '' }.`, { - hint: 'Choose an exact device name and OS version from the TestMu AI virtual-device capability generator.', + hint: `Choose an exact device name and OS version from the TestMu AI ${deviceType}-device capability generator.`, + deviceType, ...(offered.length > 0 ? { availableOsVersions: offered } : {}), }, ); } - const app = await verifyTestMuApp(options, auth, clientVersion); + const app = await verifyTestMuApp(options, deviceType, auth, clientVersion); return { provider: 'testmu', service: 'TestMu AI', verificationMessage: app.status === 'verified' - ? 'Credentials, virtual device, and uploaded app verified.' - : 'Credentials and virtual device verified; app availability is checked when the session is created.', + ? `Credentials, ${deviceType} device, and uploaded app verified.` + : `Credentials and ${deviceType} device verified; app availability is checked when the session is created.`, device: { status: 'verified', name: matchedDevice.name, @@ -76,13 +82,14 @@ export async function verifyTestMuConnection( async function verifyTestMuApp( options: TestMuOptions, + deviceType: ProviderDeviceType, auth: TestMuAuth, clientVersion: string, ): Promise { const { app } = options; // The listing is authenticated, so it doubles as the credential check for every app kind. const appsUrl = new URL(options.appsEndpoint ?? TESTMU_APPS_ENDPOINT); - appsUrl.searchParams.set('type', TESTMU_APP_LIST_TYPES[options.platform]); + appsUrl.searchParams.set('type', TESTMU_APP_LIST_TYPES[deviceType][options.platform]); appsUrl.searchParams.set('level', 'user'); const apps = await fetchTestMuJson(appsUrl, auth, clientVersion); if (isTestMuAppReference(app)) { @@ -91,8 +98,7 @@ async function verifyTestMuApp( return { status: 'configured', reference: app, - message: - 'App reference was not found among your virtual-device uploads; TestMu AI validates it when creating the session.', + message: `App reference was not found among your ${deviceType}-device uploads; TestMu AI validates it when creating the session.`, }; } return { status: 'verified', ...matched }; @@ -137,20 +143,25 @@ async function fetchTestMuJson( } /** - * The capability generator lists virtual devices per platform under - * `app.devices..brands.[]` as `{ name, osVersion: string[] }`. + * The capability generator lists devices per platform as `brands.[]` of + * `{ name, osVersion: string[] }`: under `app.devices.` for the virtual pool + * (`isVirtualDevice=true`), and directly under `` for the real pool. */ -function readTestMuVirtualDevices( +function readTestMuCatalogDevices( value: unknown, platform: 'android' | 'ios', + deviceType: ProviderDeviceType, ): Array<{ name: string; osVersions: string[] }> { - const platformCatalog = asRecord(asRecord(asRecord(asRecord(value)?.app)?.devices)?.[platform]); + const platformCatalog = + deviceType === 'real' + ? asRecord(asRecord(value)?.[platform]) + : asRecord(asRecord(asRecord(asRecord(value)?.app)?.devices)?.[platform]); const brandRecord = asRecord(platformCatalog?.brands); if (!brandRecord) { throw new AppError( 'COMMAND_FAILED', - 'TestMu AI virtual-device catalog response did not list devices for the platform.', - { platform }, + `TestMu AI ${deviceType}-device catalog response did not list devices for the platform.`, + { platform, deviceType }, ); } return Object.values(brandRecord).flatMap((devices) => { diff --git a/packages/provider-webdriver/src/testmu-device-features.test.ts b/packages/provider-webdriver/src/testmu-device-features.test.ts index da38e77e3b..33eb5afd49 100644 --- a/packages/provider-webdriver/src/testmu-device-features.test.ts +++ b/packages/provider-webdriver/src/testmu-device-features.test.ts @@ -6,6 +6,8 @@ import { TESTMU_DEVICE_FEATURE_SPECS, buildTestMuDeviceFeatureCapabilities, readTestMuDeviceFeatureFields, + readTestMuDeviceType, + rejectTestMuOnlyProviderFlags, rejectUnsupportedTestMuDeviceFeatures, } from './testmu-device-features.ts'; @@ -101,3 +103,35 @@ test('BrowserStack-only flags are rejected by flag name instead of being dropped /--provider-network-profile, --provider-custom-network are not supported by TestMu AI/, ); }); + +test('the device type defaults to the virtual pool and rejects unknown values', () => { + assert.equal(readTestMuDeviceType(undefined), 'virtual'); + assert.equal(readTestMuDeviceType({ providerDeviceType: '' }), 'virtual'); + assert.equal(readTestMuDeviceType({ providerDeviceType: 'virtual' }), 'virtual'); + assert.equal(readTestMuDeviceType({ providerDeviceType: 'real' }), 'real'); + assert.throws( + () => readTestMuDeviceType({ providerDeviceType: 'physical' }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.details?.flag === '--provider-device-type', + ); +}); + +test('other providers refuse --provider-device-type by flag name', () => { + assert.doesNotThrow(() => rejectTestMuOnlyProviderFlags(undefined, 'browserstack')); + assert.doesNotThrow(() => + rejectTestMuOnlyProviderFlags({ providerGeoLocation: 'US' }, 'browserstack'), + ); + for (const providerDeviceType of ['real', 'virtual']) { + assert.throws( + () => rejectTestMuOnlyProviderFlags({ providerDeviceType }, 'aws-device-farm'), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /--provider-device-type is only supported by TestMu AI, not aws-device-farm/.test( + error.message, + ), + ); + } +}); diff --git a/packages/provider-webdriver/src/testmu-device-features.ts b/packages/provider-webdriver/src/testmu-device-features.ts index 6f1ab049c4..84ea7d8e95 100644 --- a/packages/provider-webdriver/src/testmu-device-features.ts +++ b/packages/provider-webdriver/src/testmu-device-features.ts @@ -1,4 +1,8 @@ -import type { CloudProviderProfileFields } from '@agent-device/contracts/remote'; +import { + PROVIDER_DEVICE_TYPES, + type CloudProviderProfileFields, + type ProviderDeviceType, +} from '@agent-device/contracts/remote'; import { AppError } from '@agent-device/kernel/errors'; import { requireProviderDeviceOrientation } from './webdriver-utils.ts'; @@ -113,3 +117,38 @@ export function readTestMuDeviceFeatureFields( } return fields; } + +/** Reads the TestMu device pool off an untyped flag bag; an unset value keeps the virtual pool. */ +export function readTestMuDeviceType( + flags: Record | undefined, +): ProviderDeviceType { + const value = flags?.providerDeviceType; + if (value === undefined || value === '') return 'virtual'; + const match = PROVIDER_DEVICE_TYPES.find((deviceType) => deviceType === value); + if (match) return match; + throw new AppError('INVALID_ARGS', `Invalid --provider-device-type value: ${String(value)}.`, { + hint: `Use ${PROVIDER_DEVICE_TYPES.join('|')}.`, + flag: '--provider-device-type', + }); +} + +/** + * Fails when another provider was given a TestMu-only flag. Like the BrowserStack-only check, it + * runs in both the connect profile builder and session preparation. + */ +export function rejectTestMuOnlyProviderFlags( + flags: Record | undefined, + provider: string, +): void { + const value = flags?.providerDeviceType; + if (value === undefined || value === '') return; + throw new AppError( + 'INVALID_ARGS', + `--provider-device-type is only supported by TestMu AI, not ${provider}.`, + { + hint: 'Drop the flag or use the testmu provider.', + provider, + flags: ['--provider-device-type'], + }, + ); +} diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-webdriver/src/testmu.test.ts index 9b02320c57..fd00b6782e 100644 --- a/packages/provider-webdriver/src/testmu.test.ts +++ b/packages/provider-webdriver/src/testmu.test.ts @@ -107,25 +107,40 @@ test('a configured lt:options cannot turn off the W3C dialect', () => { assert.equal(ltOptions.tunnel, true); }); -// A configured `isRealMobile` would silently move the session to the real-device pool, which -// bills differently. -test('a configured lt:options cannot move the session off the virtual-device pool', () => { - const capabilities = buildTestMuCapabilities({ - platform: 'ios', +// A configured `isRealMobile` would silently move the session to the other pool, which bills +// differently. +test('the device type selects the TestMu pool and a configured lt:options cannot override it', () => { + const base = { + platform: 'ios' as const, deviceName: 'iPhone 16', - osVersion: '18.0', + osVersion: '18', buildName: 'run-1', sessionName: 'lease-1', - configured: { 'lt:options': { isRealMobile: true, tunnel: true } }, + }; + const real = buildTestMuCapabilities({ + ...base, + deviceType: 'real', + configured: { 'lt:options': { isRealMobile: false, tunnel: true } }, }); - const ltOptions = capabilities['lt:options'] as Record; - assert.equal(ltOptions.isRealMobile, false); - assert.equal(ltOptions.tunnel, true); + const realOptions = real['lt:options'] as Record; + assert.equal(realOptions.isRealMobile, true); + assert.equal(realOptions.tunnel, true); + assert.equal(realOptions.platformVersion, '18'); + + const virtual = buildTestMuCapabilities({ + ...base, + deviceType: 'virtual', + configured: { 'lt:options': { isRealMobile: true } }, + }); + assert.equal((virtual['lt:options'] as Record).isRealMobile, false); + + const unset = buildTestMuCapabilities(base); + assert.equal((unset['lt:options'] as Record).isRealMobile, false); }); -test('TestMu uploads go to the virtual-device upload API unless an endpoint is configured', async () => { - const tempDir = await mkdtempForTest('agent-device-testmu-upload-endpoint-'); - const appPath = path.join(tempDir, 'MyApp.apk'); +test('TestMu uploads go to the upload API of the selected device pool', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-upload-pool-'); + const appPath = path.join(tempDir, 'MyApp.ipa'); const endpoints: string[] = []; try { await fs.writeFile(appPath, 'placeholder'); @@ -133,19 +148,48 @@ test('TestMu uploads go to the virtual-device upload API unless an endpoint is c endpoints.push(String(input)); return jsonResponse({ app_url: 'lt://APP1' }); }; + await uploadTestMuApp(appPath, { ...auth, deviceType: 'real' }); + await uploadTestMuAppFromUrl('https://example.test/App.apk', { ...auth, deviceType: 'real' }); await uploadTestMuApp(appPath, auth); - await uploadTestMuAppFromUrl('https://example.test/App.apk', auth); - await uploadTestMuApp(appPath, { ...auth, endpoint: 'https://upload.test/virtual' }); + await uploadTestMuApp(appPath, { ...auth, deviceType: 'virtual' }); + await uploadTestMuApp(appPath, { + ...auth, + deviceType: 'real', + endpoint: 'https://upload.test/real', + }); assert.deepEqual(endpoints, [ + 'https://manual-api.lambdatest.com/app/upload/realDevice', + 'https://manual-api.lambdatest.com/app/upload/realDevice', 'https://manual-api.lambdatest.com/app/upload/virtualDevice', 'https://manual-api.lambdatest.com/app/upload/virtualDevice', - 'https://upload.test/virtual', + 'https://upload.test/real', ]); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); +test('a real-device upload of an .app directory asks for a signed .ipa', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-real-app-dir-'); + const appPath = path.join(tempDir, 'Demo.app'); + try { + await fs.mkdir(appPath); + const fetchMock = vi.fn(); + globalThis.fetch = fetchMock; + await assert.rejects( + uploadTestMuApp(appPath, { ...auth, deviceType: 'real' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.match(String(error.details?.hint), /signed \.ipa/); + return true; + }, + ); + assert.equal(fetchMock.mock.calls.length, 0); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + test('TestMu upload reads the lt:// reference and aborts while the request is in flight', async () => { const tempDir = await mkdtempForTest('agent-device-testmu-upload-'); const appPath = path.join(tempDir, 'App.apk'); diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-webdriver/src/testmu.ts index a06b14a0dd..a632cba50e 100644 --- a/packages/provider-webdriver/src/testmu.ts +++ b/packages/provider-webdriver/src/testmu.ts @@ -1,6 +1,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; +import type { ProviderDeviceType } from '@agent-device/contracts/remote'; import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime.ts'; import { AppError } from '@agent-device/kernel/errors'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; @@ -16,16 +17,21 @@ import { /** * TestMu session, upload, and artifact mechanics. Loaded on demand by the provider definition; - * `isRealMobile: false` in `lt:options` is what routes a session to the virtual-device pool, and + * `isRealMobile` in `lt:options` is what routes a session to the real or virtual device pool, and * the hostnames still carry the lambdatest.com brand. */ -const TESTMU_APP_UPLOAD_ENDPOINT = 'https://manual-api.lambdatest.com/app/upload/virtualDevice'; +const TESTMU_APP_UPLOAD_ENDPOINTS: Record = { + real: 'https://manual-api.lambdatest.com/app/upload/realDevice', + virtual: 'https://manual-api.lambdatest.com/app/upload/virtualDevice', +}; export const TESTMU_APPS_ENDPOINT = 'https://manual-api.lambdatest.com/app/data'; export const TESTMU_API_ENDPOINT = 'https://mobile-api.lambdatest.com/mobile-automation/api/v1'; const TESTMU_DASHBOARD_TEST_URL = 'https://appautomation.lambdatest.com/test?testID='; export type TestMuCapabilitiesOptions = { platform: CloudWebDriverPlatform; + /** Defaults to `virtual`. */ + deviceType?: ProviderDeviceType; deviceName: string; osVersion: string; app?: string; @@ -65,6 +71,8 @@ export async function listTestMuCloudArtifacts( export type TestMuUploadOptions = TestMuAuth & { clientVersion: string; + /** Selects the pool's upload API when no endpoint override is given; defaults to `virtual`. */ + deviceType?: ProviderDeviceType; endpoint?: string | URL; }; @@ -78,7 +86,10 @@ export async function uploadTestMuApp( if (!(await fs.stat(appPath)).isFile()) { throw new AppError('INVALID_ARGS', `TestMu AI can only upload an app file: ${appPath}`, { appPath, - hint: 'Zip the .app bundle of an iOS simulator build and pass the .zip.', + hint: + options.deviceType === 'real' + ? 'Real iOS devices install a signed .ipa; pass the .ipa file.' + : 'Zip the .app bundle of an iOS simulator build and pass the .zip.', }); } const form = await appFileUploadForm(appPath, 'appFile'); @@ -109,7 +120,7 @@ async function postTestMuUpload( form, { service: 'TestMu AI', - endpoint: options.endpoint ?? TESTMU_APP_UPLOAD_ENDPOINT, + endpoint: options.endpoint ?? TESTMU_APP_UPLOAD_ENDPOINTS[options.deviceType ?? 'virtual'], clientVersion: options.clientVersion, auth: options, readAppReference: readTestMuAppReference, @@ -142,11 +153,11 @@ export async function resolveTestMuAppReference( } /** - * Builds the W3C `alwaysMatch` capabilities for a TestMu virtual-device session. + * Builds the W3C `alwaysMatch` capabilities for a TestMu session. * * Standard Appium keys stay `appium:`-prefixed at the top level; everything TestMu-specific lives - * in `lt:options`. `isRealMobile: false` selects an emulator or simulator, and `w3c: true` keeps - * the hub on the W3C dialect agent-device speaks. `appiumVersion` is sent only when the caller + * in `lt:options`. `isRealMobile` selects a real device or an emulator/simulator, and `w3c: true` + * keeps the hub on the W3C dialect agent-device speaks. `appiumVersion` is sent only when the caller * pins one; otherwise TestMu AI starts its default server for the device. */ export function buildTestMuCapabilities( @@ -173,7 +184,7 @@ export function buildTestMuCapabilities( ...deviceFeatures, ...(asRecord(configuredLtOptions) ?? {}), // A configured value cannot switch the device pool or drop the W3C dialect agent-device speaks. - isRealMobile: false, + isRealMobile: options.deviceType === 'real', w3c: true, }, }; diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 3a1be8a0b8..2d9eb9a809 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -254,6 +254,7 @@ function summarizeProviderScenarioFlagExclusions() { 'providerSessionId', 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/src/__tests__/client-leases.test.ts b/src/__tests__/client-leases.test.ts new file mode 100644 index 0000000000..02ea6d0d58 --- /dev/null +++ b/src/__tests__/client-leases.test.ts @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAgentDeviceClient } from '../agent-device-client.ts'; +import { createTransport } from './client-transport-fixture.ts'; + +test('lease allocation carries the TestMu device type to the provider flags', async () => { + const setup = createTransport(async (req) => ({ + ok: true, + data: { + lease: { + leaseId: 'lease-new', + tenantId: req.meta?.tenantId, + runId: req.meta?.runId, + backend: req.meta?.leaseBackend, + }, + }, + })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.leases.allocate({ + tenant: 'testmu', + runId: 'remote-run', + leaseBackend: 'ios-instance', + leaseProvider: 'testmu', + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18', + providerDeviceType: 'real', + providerApp: 'lt://APP1', + }); + + assert.equal(setup.calls[0]?.command, 'lease_allocate'); + assert.equal(setup.calls[0]?.flags?.providerDeviceType, 'real'); + assert.equal(setup.calls[0]?.flags?.providerOsVersion, '18'); +}); diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index 1255a4af6a..1d882c96ff 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -3,6 +3,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { connectCommand } from '../cli/commands/connection.ts'; +import { runCliCapture } from './cli-capture.ts'; import { readActiveConnectionState, type RemoteConnectionState, @@ -150,6 +151,119 @@ test('connect testmu rejects BrowserStack network and re-sign flags before savin } }); +test('connect testmu stores and verifies the real-device pool', async () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-real-'); + const stateDir = path.join(tempRoot, '.state'); + vi.stubEnv('LT_USERNAME', 'lt-user'); + vi.stubEnv('LT_ACCESS_KEY', 'lt-key'); + + try { + await connectWithGeneratedProviderProfile({ + stateDir, + positionals: ['testmu'], + flags: { + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18', + providerDeviceType: 'real', + providerApp: 'lt://APP1', + }, + }); + + assert.deepEqual(mockedVerifyWebDriverConnection.mock.calls[0]?.[0], { + provider: 'testmu', + username: 'lt-user', + accessKey: 'lt-key', + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18', + app: 'lt://APP1', + deviceType: 'real', + }); + const state = readRequiredActiveState(stateDir); + const generated = readGeneratedConfig(state.remoteConfigPath); + assert.equal(generated.providerDeviceType, 'real'); + assert.equal(generated.providerOsVersion, '18'); + + // The saved profile reproduces the same verification when it is loaded again. + mockedVerifyWebDriverConnection.mockClear(); + await connectWithGeneratedProviderProfile({ + stateDir, + positionals: [], + flags: { remoteConfig: state.remoteConfigPath, force: true }, + }); + const reloaded = mockedVerifyWebDriverConnection.mock.calls[0]?.[0]; + assert.equal(reloaded?.provider, 'testmu'); + assert.equal(reloaded.deviceType, 'real'); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test('providers other than TestMu refuse --provider-device-type before saving a profile', () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-device-type-reject-'); + const base = { json: false, help: false, version: false, platform: 'android' as const }; + + try { + for (const [provider, flags, env] of [ + [ + 'browserstack', + { + ...base, + device: 'Google Pixel 8', + providerOsVersion: '14.0', + providerApp: 'bs://app-id', + }, + { BROWSERSTACK_USERNAME: 'u', BROWSERSTACK_ACCESS_KEY: 'k' }, + ], + [ + 'aws-device-farm', + { + ...base, + awsProjectArn: 'arn:aws:devicefarm:us-west-2:123:project/p', + awsDeviceArn: 'arn:aws:devicefarm:us-west-2::device/d', + }, + {}, + ], + ] as const) { + assert.throws( + () => + resolveCloudWebDriverConnectProfile({ + provider, + stateDir: path.join(tempRoot, '.state'), + cwd: tempRoot, + env, + flags: { ...flags, providerDeviceType: 'real' }, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.match( + error.message, + new RegExp(`--provider-device-type is only supported by TestMu AI, not ${provider}`), + ); + return true; + }, + ); + } + assert.equal(fs.existsSync(path.join(tempRoot, '.state')), false); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test('connect limrun refuses the TestMu device type', async () => { + const result = await runCliCapture( + ['connect', 'limrun', '--platform', 'ios', '--provider-device-type', 'real', '--json'], + { + env: { LIMRUN_API_KEY: 'lim_test_key' }, + stateDirPrefix: 'agent-device-connect-limrun-device-type-', + }, + ); + assert.equal(result.code, 1); + assert.match(result.stdout, /--provider-device-type is only supported by TestMu AI, not limrun/); +}); + async function connectWithGeneratedProviderProfile(options: { stateDir: string; positionals: string[]; @@ -176,11 +290,13 @@ async function connectWithGeneratedProviderProfile(options: { function readGeneratedConfig(configPath: string): { providerApp?: string; providerOsVersion?: string; + providerDeviceType?: string; providerBuild?: string; } { return JSON.parse(fs.readFileSync(configPath, 'utf8')) as { providerApp?: string; providerOsVersion?: string; + providerDeviceType?: string; providerBuild?: string; }; } diff --git a/src/cli.ts b/src/cli.ts index 67246cd596..5ace00beb7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -456,6 +456,7 @@ function buildClientConfig(ctx: CliRunContext): AgentDeviceClientConfig { deviceKey: connection?.deviceKey, providerApp: currentFlags.providerApp, providerOsVersion: currentFlags.providerOsVersion, + providerDeviceType: currentFlags.providerDeviceType, providerProject: currentFlags.providerProject, providerBuild: currentFlags.providerBuild, providerSessionName: currentFlags.providerSessionName, diff --git a/src/cli/commands/connection-runtime.ts b/src/cli/commands/connection-runtime.ts index db52353600..b645609807 100644 --- a/src/cli/commands/connection-runtime.ts +++ b/src/cli/commands/connection-runtime.ts @@ -883,6 +883,7 @@ async function allocateOrReuseLease( serial: flags.serial, providerApp: initialApp ?? flags.providerApp, providerOsVersion: flags.providerOsVersion, + providerDeviceType: flags.providerDeviceType, providerProject: flags.providerProject, providerBuild: flags.providerBuild, providerSessionName: flags.providerSessionName, diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index 031d48c8c7..c6ed0ebbb9 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -4,7 +4,10 @@ import { rejectBrowserStackOnlyDeviceFeatures, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; -import { rejectUnsupportedTestMuDeviceFeatures } from '@agent-device/provider-webdriver/testmu-device-features'; +import { + rejectTestMuOnlyProviderFlags, + rejectUnsupportedTestMuDeviceFeatures, +} from '@agent-device/provider-webdriver/testmu-device-features'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { PlatformSelector } from '@agent-device/kernel/device'; @@ -118,6 +121,7 @@ function browserStackProfileFields(options: { env?: EnvMap; cwd: string; }): RemoteConfigProfile { + rejectTestMuOnlyProviderFlags(options.flags, CLOUD_WEBDRIVER_PROVIDERS.browserStack); return hubProviderProfileFields(BROWSERSTACK_HUB_PROFILE, options); } @@ -127,7 +131,10 @@ function testMuProfileFields(options: { cwd: string; }): RemoteConfigProfile { rejectUnsupportedTestMuDeviceFeatures(options.flags); - return hubProviderProfileFields(TESTMU_HUB_PROFILE, options); + return { + ...hubProviderProfileFields(TESTMU_HUB_PROFILE, options), + providerDeviceType: options.flags.providerDeviceType, + }; } function hubProviderProfileFields( @@ -181,6 +188,7 @@ function awsDeviceFarmProfileFields(options: { }): RemoteConfigProfile { const { env, flags } = options; rejectBrowserStackOnlyDeviceFeatures(flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm); + rejectTestMuOnlyProviderFlags(flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm); const platform = requireCloudWebDriverPlatform( flags.platform, 'connect aws-device-farm requires --platform ios|android.', diff --git a/src/cli/connection/connect-provider-adapters.ts b/src/cli/connection/connect-provider-adapters.ts index a83c548365..7fcfabd34b 100644 --- a/src/cli/connection/connect-provider-adapters.ts +++ b/src/cli/connection/connect-provider-adapters.ts @@ -172,6 +172,7 @@ async function verifyTestMu( 'TestMu AI profile missed OS version.', ), app: requiredResolvedValue(flags.providerApp, 'TestMu AI profile missed app.'), + ...(flags.providerDeviceType ? { deviceType: flags.providerDeviceType } : {}), ...(env.TESTMU_API_ENDPOINT ? { apiEndpoint: env.TESTMU_API_ENDPOINT } : {}), }); } diff --git a/src/cli/connection/limrun-profile.ts b/src/cli/connection/limrun-profile.ts index b3025a78da..2033dec412 100644 --- a/src/cli/connection/limrun-profile.ts +++ b/src/cli/connection/limrun-profile.ts @@ -3,6 +3,7 @@ import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { CliFlags } from '@agent-device/contracts/command'; import { type EnvMap } from '@agent-device/kernel/source-value'; +import { rejectTestMuOnlyProviderFlags } from '@agent-device/provider-webdriver/testmu-device-features'; import { readMetroProfileFields } from './profile-fields.ts'; import { persistAndResolveGeneratedProfile } from './generated-config.ts'; import { resolveRequestedLeaseBackend } from '../commands/connection-runtime.ts'; @@ -53,6 +54,7 @@ function buildLimrunRemoteProfile(options: { flags: CliFlags }): RemoteConfigPro } function validateLimrunConnectFlags(flags: CliFlags): 'android-instance' | 'ios-instance' { + rejectTestMuOnlyProviderFlags(flags, 'limrun'); if (flags.platform !== 'android' && flags.platform !== 'ios') { throw new AppError('INVALID_ARGS', 'connect limrun requires --platform ios or android.'); } diff --git a/src/cli/parser/__tests__/args-parse-provider-device-type.test.ts b/src/cli/parser/__tests__/args-parse-provider-device-type.test.ts new file mode 100644 index 0000000000..534d2d6a3d --- /dev/null +++ b/src/cli/parser/__tests__/args-parse-provider-device-type.test.ts @@ -0,0 +1,17 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { parseArgs } from '../args.ts'; + +test('parseArgs reads the TestMu device type and rejects an unknown pool', () => { + const parsed = parseArgs(['connect', 'testmu', '--provider-device-type', 'real'], { + strictFlags: true, + }); + assert.equal(parsed.flags.providerDeviceType, 'real'); + assert.throws( + () => + parseArgs(['connect', 'testmu', '--provider-device-type', 'physical'], { + strictFlags: true, + }), + /Invalid provider-device-type: physical/, + ); +}); diff --git a/src/commands/command-flags.ts b/src/commands/command-flags.ts index f7b44a4618..fa04c2d2ef 100644 --- a/src/commands/command-flags.ts +++ b/src/commands/command-flags.ts @@ -28,6 +28,7 @@ function buildFlags(options: InternalRequestOptions): CommandFlags { providerSessionId: options.providerSessionId, providerApp: options.providerApp, providerOsVersion: options.providerOsVersion, + providerDeviceType: options.providerDeviceType, providerProject: options.providerProject, providerBuild: options.providerBuild, providerSessionName: options.providerSessionName, diff --git a/src/commands/schema/cli-help-topics.test.ts b/src/commands/schema/cli-help-topics.test.ts index cc92b8b6e7..c7f47c5157 100644 --- a/src/commands/schema/cli-help-topics.test.ts +++ b/src/commands/schema/cli-help-topics.test.ts @@ -446,6 +446,11 @@ test('usageForCommand resolves remote help topic', async () => { help.indexOf('BrowserStack hosted-device flow'), ); assert.match(testMuFlow, /--device "iPhone 16" --provider-os-version 18\.0/); + assert.match( + testMuFlow, + /connect testmu --provider-device-type real .*--provider-os-version 18 --provider-app \.\/MyApp\.ipa/, + ); + assert.match(testMuFlow, /major OS version \(18, not 18\.0\)/); assert.match(testMuFlow, /agent-device disconnect/); assert.match(help, /Limrun uses LIMRUN_API_KEY/); assert.match(help, /BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY/); diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 1211c54f88..b4d396fe9f 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -577,7 +577,7 @@ Providers: Direct proxy: agent-device connect proxy --daemon-base-url stores the shared proxy profile and client identity. BrowserStack: agent-device connect browserstack verifies credentials, the exact device, and a bs:// app reference, then stores a local provider profile. It does not create an App Automate session. AWS Device Farm: agent-device connect aws-device-farm verifies credentials and the exact project, device, and optional app upload, then stores a local provider profile. It does not create a remote access session. - TestMu AI: agent-device connect testmu verifies credentials, the exact virtual device (emulator or simulator) and OS version, and an lt:// app reference, then stores a local provider profile. It does not create a hub session. + TestMu AI: agent-device connect testmu verifies credentials, the exact virtual device (emulator or simulator) or, with --provider-device-type real, real device and OS version, and an lt:// app reference, then stores a local provider profile. It does not create a hub session. Limrun: agent-device connect limrun verifies access to the selected iOS or Android instance service, then stores a local provider profile. It does not create an instance. After direct-provider connect: @@ -627,6 +627,10 @@ TestMu AI virtual-device flow (emulators and simulators): agent-device artifacts --json agent-device disconnect +TestMu AI real-device flow: + agent-device connect testmu --provider-device-type real --platform ios --device "iPhone 16" --provider-os-version 18 --provider-app ./MyApp.ipa + Real iOS devices are listed by major OS version (18, not 18.0) and install a signed .ipa. Real and virtual devices have separate upload APIs, so pass an lt:// id uploaded for the pool you connect to. + BrowserStack hosted-device flow: BROWSERSTACK_USERNAME=... BROWSERSTACK_ACCESS_KEY=... agent-device connect browserstack --platform android --device "Google Pixel 8" --provider-os-version 14.0 --provider-app bs://app-id diff --git a/src/commands/schema/command-overrides.ts b/src/commands/schema/command-overrides.ts index f69ccda125..fcd1464609 100644 --- a/src/commands/schema/command-overrides.ts +++ b/src/commands/schema/command-overrides.ts @@ -77,6 +77,7 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { 'leaseBackend', 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/src/daemon-client/daemon-client-rpc.test.ts b/src/daemon-client/daemon-client-rpc.test.ts index faa8c6e835..a8656d63f2 100644 --- a/src/daemon-client/daemon-client-rpc.test.ts +++ b/src/daemon-client/daemon-client-rpc.test.ts @@ -45,6 +45,7 @@ test('lease allocation transports the provider configuration the session needs ( device: 'iPhone 15', providerApp: 'bs://app-id', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-2026-09-11', providerSessionName: 'smoke — iOS', @@ -72,6 +73,7 @@ test('lease allocation transports the provider configuration the session needs ( device: 'iPhone 15', providerApp: 'bs://app-id', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-2026-09-11', providerSessionName: 'smoke — iOS', diff --git a/src/daemon/handlers/session-doctor-options.ts b/src/daemon/handlers/session-doctor-options.ts index 4c9f6eb1de..7481ac986d 100644 --- a/src/daemon/handlers/session-doctor-options.ts +++ b/src/daemon/handlers/session-doctor-options.ts @@ -20,6 +20,7 @@ const REMOTE_PROVIDER_FLAG_KEYS = [ 'providerSessionId', 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/src/remote/remote-config-schema.ts b/src/remote/remote-config-schema.ts index d28e68a892..24a9d7e272 100644 --- a/src/remote/remote-config-schema.ts +++ b/src/remote/remote-config-schema.ts @@ -1,5 +1,6 @@ import { PROVIDER_DEVICE_ORIENTATIONS, + PROVIDER_DEVICE_TYPES, type CloudProviderProfileFields, type RemoteConfigMetroOptions, type RemoteConnectionProfileFields, @@ -78,6 +79,7 @@ export const REMOTE_CONFIG_FIELD_SPECS = [ { key: 'session', type: 'string' }, { key: 'providerApp', type: 'string' }, { key: 'providerOsVersion', type: 'string' }, + { key: 'providerDeviceType', type: 'enum', enumValues: PROVIDER_DEVICE_TYPES }, { key: 'providerProject', type: 'string' }, { key: 'providerBuild', type: 'string' }, { key: 'providerSessionName', type: 'string' }, diff --git a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts index d98a17f151..0fae68261d 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts @@ -185,6 +185,83 @@ test('AWS Device Farm facade rejects BrowserStack-owned device features at sessi }); }, 15_000); +test('TestMu facade routes a real-device session to the real pool and its upload API', async () => { + await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { + const provider = createProviderWebDriver({ + clientVersion: CLIENT_VERSION, + runHostCommand: unexpectedHostCommand, + }); + const runtime = runtimeFor( + provider.createDefaultRuntimes({ + LT_USERNAME: 'user', + LT_ACCESS_KEY: 'key', + TESTMU_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, + TESTMU_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/virtualDevice`, + TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/realDevice`, + }), + CLOUD_WEBDRIVER_PROVIDERS.testMu, + ); + const lease = makeLease(CLOUD_WEBDRIVER_PROVIDERS.testMu); + try { + await runtime.leaseLifecycle.allocate?.(lease, { + flags: { + platform: 'android', + device: 'Pixel 6', + providerOsVersion: '14', + providerDeviceType: 'real', + providerApp: 'https://builds.example/app.apk', + }, + }); + } finally { + await runtime.shutdown(); + } + + assert.deepEqual( + server.calls.filter((call) => call.path.startsWith('/lt/upload/')).map((call) => call.path), + ['/lt/upload/realDevice'], + ); + const session = server.calls.find((call) => call.path === '/wd/hub/session'); + const alwaysMatch = ( + session?.body as { capabilities?: { alwaysMatch?: Record } } | undefined + )?.capabilities?.alwaysMatch; + const ltOptions = alwaysMatch?.['lt:options'] as Record | undefined; + assert.equal(ltOptions?.isRealMobile, true); + assert.equal(ltOptions?.app, 'lt://REAL1'); + assert.equal(ltOptions?.platformVersion, '14'); + }); +}, 15_000); + +test('BrowserStack facade rejects the TestMu device type at session preparation', async () => { + await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { + const provider = createProviderWebDriver({ + clientVersion: CLIENT_VERSION, + runHostCommand: unexpectedHostCommand, + }); + const runtime = runtimeFor( + provider.createDefaultRuntimes({ + BROWSERSTACK_USERNAME: 'user', + BROWSERSTACK_ACCESS_KEY: 'key', + BROWSERSTACK_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, + }), + CLOUD_WEBDRIVER_PROVIDERS.browserStack, + ); + const lease = makeLease(CLOUD_WEBDRIVER_PROVIDERS.browserStack); + const context = browserStackContext(lease); + try { + await assert.rejects( + async () => + await runtime.leaseLifecycle.allocate?.(lease, { + flags: { ...context.flags, providerDeviceType: 'real' }, + }), + /--provider-device-type is only supported by TestMu AI, not browserstack/, + ); + assert.deepEqual(server.calls, []); + } finally { + await runtime.shutdown(); + } + }); +}, 15_000); + test('AWS Device Farm facade uses the injected host-command capability for its full lifecycle', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const host = new FakeAwsHostCommand(`${server.url}/wd/hub/`); @@ -518,6 +595,10 @@ class FakeCloudProviderServer extends CloudWebDriverTestServer { }); case 'POST /app-automate/upload': return cloudWebDriverTestJson({ app_url: 'bs://uploaded-app' }); + case 'POST /lt/upload/realDevice': + return cloudWebDriverTestJson({ app_url: 'lt://REAL1' }); + case 'POST /lt/upload/virtualDevice': + return cloudWebDriverTestJson({ app_url: 'lt://VIRTUAL1' }); case 'GET /app-automate/sessions/wd-1.json': return cloudWebDriverTestJson({ automation_session: { diff --git a/test/integration/provider-scenarios/daemon-http-lease-allocate.test.ts b/test/integration/provider-scenarios/daemon-http-lease-allocate.test.ts index fd615dd7fc..275c09b659 100644 --- a/test/integration/provider-scenarios/daemon-http-lease-allocate.test.ts +++ b/test/integration/provider-scenarios/daemon-http-lease-allocate.test.ts @@ -47,6 +47,7 @@ test('Provider-backed integration daemon HTTP lease allocate forwards provider a device: 'iPhone 15', providerApp: 'bs://app-id', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-1', providerSessionName: 'smoke', @@ -64,6 +65,7 @@ test('Provider-backed integration daemon HTTP lease allocate forwards provider a device: 'iPhone 15', providerApp: 'bs://app-id', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-1', providerSessionName: 'smoke', From add8062e2693d5b1d8c88168c30629041468f040 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 5/6] fix(provider-webdriver): upload the archive of materialized iOS builds Install from a remote source materializes an iOS build by extracting the `.app` bundle from a zipped simulator build or an .ipa. The WebDriver deployment runtime handed that extracted `.app` directory to the provider's uploader. Hosted upload APIs take a file, not a directory, so the upload could not succeed. When the materialized artifact is an iOS `.app` extracted from a `.zip` or `.ipa` archive, upload the archive it came from. Every other case uploads the installable path as before: no archive, an archive of another type, or an Android build. A provider without an uploader still installs the extracted bundle path. The bundle id and launch target hints are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/runtime-deployment.test.ts | 113 +++++++++++++++++- .../src/runtime-deployment.ts | 34 +++++- 2 files changed, 140 insertions(+), 7 deletions(-) diff --git a/packages/provider-webdriver/src/runtime-deployment.test.ts b/packages/provider-webdriver/src/runtime-deployment.test.ts index 67b06e14a9..559f3c37c9 100644 --- a/packages/provider-webdriver/src/runtime-deployment.test.ts +++ b/packages/provider-webdriver/src/runtime-deployment.test.ts @@ -1,6 +1,10 @@ -import { expect, test, vi } from 'vitest'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { afterEach, expect, test, vi } from 'vitest'; import { createCloudWebDriverCapabilities } from './capabilities.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createTestMuUploadApp } from './testmu.ts'; +import { mkdtempForTest } from './tmp-dir.fixtures.ts'; import { createWebDriverDeploymentRuntime } from './runtime-deployment.ts'; import type { WebDriverProviderSession } from './runtime-session.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; @@ -14,6 +18,13 @@ const device: DeviceInfo = { booted: true, }; +const iosDevice: DeviceInfo = { ...device, platform: 'apple', id: 'webdriver:ios' }; +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); + test('keeps a stale WebDriver owner unavailable before any deployment attempt', () => { const deployment = createWebDriverDeploymentRuntime({ provider: 'webdriver-test', @@ -75,6 +86,106 @@ test('aborts a WebDriver provider deployment while its install is in flight', as expect(installApp).toHaveBeenCalledWith('bs://uploaded-app', controller.signal); }); +// Materialization extracts `App.app.zip` (or an .ipa) to a `.app` directory, which no hosted +// upload API accepts; the archive it came from is the uploadable build. +test('a hosted upload of a materialized iOS bundle sends the archive it was extracted from', async () => { + const uploaded: string[] = []; + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'webdriver-test', + uploadApp: async ({ appPath }) => { + uploaded.push(appPath); + return { appReference: `lt://${uploaded.length}` }; + }, + findSessionForDevice: () => activeSession(installApp), + }); + const deploy = async (selected: DeviceInfo, artifact: Record) => + await deployment.deployMaterializedApp( + selected, + { artifact: { installablePath: '', ...artifact, cleanup: async () => {} } }, + new AbortController().signal, + ); + + const result = await deploy(iosDevice, { + archivePath: '/m/App.app.zip', + installablePath: '/m/extracted/App.app', + bundleId: 'com.example.app', + }); + await deploy(iosDevice, { archivePath: '/m/App.ipa', installablePath: '/m/x/Payload/App.app' }); + await deploy(iosDevice, { installablePath: '/m/App.app' }); + await deploy(iosDevice, { archivePath: '/m/App.tar.gz', installablePath: '/m/x/App.app' }); + await deploy(device, { archivePath: '/m/build.zip', installablePath: '/m/x/app.apk' }); + + expect(uploaded).toEqual([ + '/m/App.app.zip', + '/m/App.ipa', + '/m/App.app', + '/m/x/App.app', + '/m/x/app.apk', + ]); + expect(result).toEqual({ bundleId: 'com.example.app', launchTarget: 'com.example.app' }); + expect(installApp).toHaveBeenNthCalledWith(1, 'lt://1', expect.any(AbortSignal)); +}); + +test('a provider without an uploader still installs the materialized bundle path', async () => { + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'webdriver-test', + findSessionForDevice: () => activeSession(installApp), + }); + await deployment.deployMaterializedApp( + iosDevice, + { + artifact: { + archivePath: '/m/App.app.zip', + installablePath: '/m/extracted/App.app', + bundleId: 'com.example.app', + cleanup: async () => {}, + }, + }, + new AbortController().signal, + ); + expect(installApp).toHaveBeenCalledWith('/m/extracted/App.app', expect.any(AbortSignal)); +}); + +test('TestMu uploads the zipped simulator build that install-from-source extracted', async () => { + const tempDir = await mkdtempForTest('agent-device-materialized-upload-'); + try { + const archivePath = path.join(tempDir, 'App.app.zip'); + const installablePath = path.join(tempDir, 'extracted', 'App.app'); + await fs.writeFile(archivePath, 'zip bytes'); + await fs.mkdir(installablePath, { recursive: true }); + const uploadedNames: unknown[] = []; + globalThis.fetch = async (_input, init) => { + const body = init?.body; + if (!(body instanceof FormData)) throw new Error('expected a multipart upload'); + uploadedNames.push((body.get('appFile') as File).name); + return new Response(JSON.stringify({ app_id: 'APP42' }), { status: 200 }); + }; + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'testmu', + uploadApp: createTestMuUploadApp({ + clientVersion: '0.0.0-test', + username: 'user', + accessKey: 'key', + }), + findSessionForDevice: () => activeSession(installApp), + }); + + await deployment.deployMaterializedApp( + iosDevice, + { artifact: { archivePath, installablePath, cleanup: async () => {} } }, + new AbortController().signal, + ); + + expect(uploadedNames).toEqual(['App.app.zip']); + expect(installApp).toHaveBeenCalledWith('lt://APP42', expect.any(AbortSignal)); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + function activeSession( installApp: (appPath: string, signal?: AbortSignal) => Promise, ): WebDriverProviderSession { diff --git a/packages/provider-webdriver/src/runtime-deployment.ts b/packages/provider-webdriver/src/runtime-deployment.ts index 07e3ecf16d..f88fe4f304 100644 --- a/packages/provider-webdriver/src/runtime-deployment.ts +++ b/packages/provider-webdriver/src/runtime-deployment.ts @@ -6,6 +6,7 @@ import type { AppDeploymentInput, AppDeploymentResult, DeployMaterializedAppInput, + MaterializedAppSource, } from '@agent-device/contracts/app-deployment-runtime'; import type { RuntimeOperationFact } from '@agent-device/contracts/platform-runtime'; import { publicPlatformString, type DeviceInfo } from '@agent-device/kernel/device'; @@ -49,10 +50,10 @@ export function createWebDriverDeploymentRuntime( findSessionForDevice(device: DeviceInfo): WebDriverProviderSession | undefined; }>, ): WebDriverDeploymentRuntime { - const installApp = async ( + const install = async ( device: DeviceInfo, app: string, - appPath: string, + paths: Readonly<{ appPath: string; uploadPath: string }>, installOptions?: ProviderDeviceInstallOptions, signal?: AbortSignal, ): Promise => { @@ -63,13 +64,20 @@ export function createWebDriverDeploymentRuntime( session, device, app, - appPath, + paths.uploadPath, installOptions, signal, ); - await session.client.installApp(upload?.appReference ?? appPath, signal); + await session.client.installApp(upload?.appReference ?? paths.appPath, signal); return providerInstallResult(upload, installOptions); }; + const installApp = async ( + device: DeviceInfo, + app: string, + appPath: string, + installOptions?: ProviderDeviceInstallOptions, + signal?: AbortSignal, + ) => await install(device, app, { appPath, uploadPath: appPath }, installOptions, signal); return Object.freeze({ fact: (device) => deploymentFact(options.findSessionForDevice(device)), installApp, @@ -92,10 +100,13 @@ export function createWebDriverDeploymentRuntime( ), deployMaterializedApp: async (device, input, signal) => deploymentResult( - await installApp( + await install( device, '', - input.artifact.installablePath, + { + appPath: input.artifact.installablePath, + uploadPath: materializedUploadPath(input.artifact), + }, { appIdentifierHint: input.artifact.bundleId, packageNameHint: input.artifact.packageName, @@ -106,6 +117,17 @@ export function createWebDriverDeploymentRuntime( }); } +/** + * Materialization extracts an iOS `.app` bundle out of a zipped simulator build or an .ipa, and no + * hosted upload API takes a directory, so the uploader gets the archive the bundle came from. + */ +function materializedUploadPath(artifact: MaterializedAppSource): string { + const { archivePath, installablePath } = artifact; + return archivePath && /\.(zip|ipa)$/i.test(archivePath) && /\.app\/?$/i.test(installablePath) + ? archivePath + : installablePath; +} + function deploymentFact(session: WebDriverProviderSession | undefined): RuntimeOperationFact { if (!session) { return Object.freeze({ From abf7a0a25a1dad2fe6451dfa2beaedbb84f1ae16 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 6/6] docs: add the TestMu AI device cloud guide Add a TestMu AI guide next to the BrowserStack and AWS Device Farm ones and link it from the sidebar. It covers credentials and `connect`, the exact device and OS version match, app references and uploads, device features, real devices with `--provider-device-type real`, the CLI and Node.js client workflows, artifacts, and endpoint overrides. List TestMu AI among the device clouds in the README, the device clouds overview, the client API page, and the command reference. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- website/docs/docs/_meta.json | 5 + website/docs/docs/client-api.md | 6 +- website/docs/docs/commands.md | 4 +- website/docs/docs/device-clouds.md | 5 +- website/docs/docs/testmu.md | 200 +++++++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 9 deletions(-) create mode 100644 website/docs/docs/testmu.md diff --git a/README.md b/README.md index a5f9cb069c..9c056a81cb 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ The same session and evidence model works at every step: the agent explores the | --- | --- | --- | | Local | Trying commands and debugging apps on simulators, emulators, physical devices, macOS, and Linux. | Follow the Quick Start. | | CI/CD | Automated pull request and merge validation with replay scripts and captured artifacts. | Try the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml). | -| Cloud / remote | Linux runners, managed devices, and remote jobs. | Set up a [remote proxy](https://oss.callstack.com/agent-device/docs/remote-proxy), connect a [device cloud](https://oss.callstack.com/agent-device/docs/device-clouds) (BrowserStack, AWS Device Farm, Limrun), or [contact Callstack](mailto:hello@callstack.com) for team QA. | +| Cloud / remote | Linux runners, managed devices, and remote jobs. | Set up a [remote proxy](https://oss.callstack.com/agent-device/docs/remote-proxy), connect a [device cloud](https://oss.callstack.com/agent-device/docs/device-clouds) (BrowserStack, AWS Device Farm, TestMu AI, Limrun), or [contact Callstack](mailto:hello@callstack.com) for team QA. | ## How it works @@ -145,7 +145,7 @@ The same session and evidence model works at every step: the agent explores the Support depth varies by target. Newer backends such as HarmonyOS and Vega OS cover a subset of commands; run `agent-device capabilities --platform ` to see what a target supports. -Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. Inspect ownership without a daemon via `agent-device device status`, and settle provably dead owners with `agent-device device release --stale`. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds). +Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. Inspect ownership without a daemon via `agent-device device status`, and settle provably dead owners with `agent-device device release --stale`. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, TestMu AI, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds). `agent-device` uses the inspect-act-verify process from Vercel's [agent-browser](https://github.com/vercel-labs/agent-browser) for mobile, TV, and desktop apps. Basic `--platform web` support runs `agent-browser` in the same session and replay system. diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index 68655a6001..b7d84f29fa 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -113,6 +113,11 @@ "label": "AWS Device Farm", "link": "/docs/aws-device-farm" }, + { + "type": "custom-link", + "label": "TestMu AI", + "link": "/docs/testmu" + }, { "type": "custom-link", "label": "Limrun", diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 9265e32296..7c3563fe35 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -117,7 +117,7 @@ stdout/stderr. The option mirrors `open --launch-console` and is not valid for U or contacts the daemon. Pass `{ stateDir }` to resolve an explicit override the same way the CLI resolves `--state-dir`. `client.sessions.artifacts({ provider, providerSessionId })` mirrors `artifacts --provider ... --provider-session ...` and returns provider-hosted `cloudArtifacts`. -Use it for BrowserStack or AWS Device Farm session videos/logs after a cloud session has stopped, or omit `providerSessionId` when an embedding host has registered a provider runtime that can infer the active lease. Limrun does not currently expose provider artifacts through this command. +Use it for BrowserStack, AWS Device Farm, or TestMu AI session videos/logs after a cloud session has stopped, or omit `providerSessionId` when an embedding host has registered a provider runtime that can infer the active lease. Limrun does not currently expose provider artifacts through this command. ```ts const result = await client.sessions.artifacts({ @@ -134,7 +134,7 @@ if ('cloudArtifacts' in result) { ## Device cloud sessions -Limrun, BrowserStack, and AWS Device Farm can be driven through the normal typed client methods. Use the corresponding CLI `connect` flow when you want persisted local connection state. Use direct client config when a Node integration already owns credentials and provider selectors. +Limrun, BrowserStack, AWS Device Farm, and TestMu AI can be driven through the normal typed client methods. Use the corresponding CLI `connect` flow when you want persisted local connection state. Use direct client config when a Node integration already owns credentials and provider selectors. ```ts import { createAgentDeviceClient } from 'agent-device'; @@ -158,7 +158,7 @@ from an explicit selector, an existing session, one local booted/bootable candid simulator with the app installed, or one provider-owned candidate. Ambiguous requests fail with structured retry selectors instead of silently retargeting. -Use `client.sessions.artifacts({ provider, providerSessionId })` with `closed.provider?.providerSessionId` to fetch provider-hosted video and log URLs after close. See the [BrowserStack](/docs/browserstack), [AWS Device Farm](/docs/aws-device-farm), and [Limrun](/docs/limrun) guides for provider-specific setup. +Use `client.sessions.artifacts({ provider, providerSessionId })` with `closed.provider?.providerSessionId` to fetch provider-hosted video and log URLs after close. See the [BrowserStack](/docs/browserstack), [AWS Device Farm](/docs/aws-device-farm), [TestMu AI](/docs/testmu), and [Limrun](/docs/limrun) guides for provider-specific setup. ## Web sessions diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 5af97672f6..44cffce0d2 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -114,7 +114,7 @@ agent-device fold open - Remote daemon clients can pass `--daemon-base-url http(s)://host:port[/base-path]` to skip local daemon discovery/startup and call a remote HTTP daemon directly. - Use `--daemon-auth-token ` (or `AGENT_DEVICE_DAEMON_AUTH_TOKEN`) for explicit service/API-token automation against non-loopback remote daemon URLs; the client sends it in both the JSON-RPC request token and HTTP auth headers. - Use [Remote Proxy](/docs/remote-proxy) when you need to run `agent-device proxy` on a Mac with simulator/device access and drive it from another machine through cloudflared, ngrok, or another HTTP tunnel. -- Use [BrowserStack](/docs/browserstack) or [AWS Device Farm](/docs/aws-device-farm) when a CI agent needs a hosted device session without interactive login. +- Use [BrowserStack](/docs/browserstack), [AWS Device Farm](/docs/aws-device-farm), or [TestMu AI](/docs/testmu) when a CI agent needs a hosted device session without interactive login. - For human cloud access, `connect` can discover a cloud connection profile, while `connect --remote-config ...` uses a local profile. Both refresh a stored CLI session into a short-lived `adc_agent_...` token when needed. If no CLI session exists, interactive shells start login automatically; CI and non-interactive shells fail with API-token setup instructions. Use `--no-login` to disable implicit login. `AGENT_DEVICE_CLOUD_BASE_URL` is the bridge/control-plane API origin; its `/api-keys` route may redirect to the dashboard for token creation. - For remote `connect` and `connect --remote-config` flows, see [Remote Metro workflow](#remote-metro-workflow). - Android React Native relaunch flows require an installed package name for `open --relaunch`; install/reinstall the APK first, then relaunch by package. `open --relaunch` is rejected because runtime hints are written through the installed app sandbox. @@ -1274,7 +1274,7 @@ agent-device artifacts --provider aws-device-farm --provider-session ` and `--provider `. BrowserStack uses `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY`. AWS Device Farm uses the AWS CLI credential chain and infers the region from the session ARN when possible. See [BrowserStack](/docs/browserstack) and [AWS Device Farm](/docs/aws-device-farm) for CI credential setup. +- Historical lookup requires `--provider-session ` and `--provider `. BrowserStack uses `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY`. TestMu AI uses `LT_USERNAME` and `LT_ACCESS_KEY`. AWS Device Farm uses the AWS CLI credential chain and infers the region from the session ARN when possible. See [BrowserStack](/docs/browserstack), [AWS Device Farm](/docs/aws-device-farm), and [TestMu AI](/docs/testmu) for CI credential setup. - When a cloud runtime is registered in-process by an embedding host, `artifacts` can infer the active provider session from the current lease before disconnect. - `disconnect --json` and `close --json` include provider release data when the runtime returns final cloud artifacts after session teardown. Some providers only finalize video/log URLs after the remote session is stopped, so retry `agent-device artifacts --provider --json` if the first response is `pending`. diff --git a/website/docs/docs/device-clouds.md b/website/docs/docs/device-clouds.md index b7f165dd5f..0e4333c012 100644 --- a/website/docs/docs/device-clouds.md +++ b/website/docs/docs/device-clouds.md @@ -9,9 +9,10 @@ Use a device cloud or farm when an agent needs to automate a hosted mobile devic - [BrowserStack](/docs/browserstack): Android and iOS App Automate sessions over WebDriver. - [AWS Device Farm](/docs/aws-device-farm): Android and iOS remote-access sessions through AWS. +- [TestMu AI](/docs/testmu): Android emulator, iOS simulator, and real-device sessions over WebDriver. - [Limrun](/docs/limrun): direct iOS simulator and Android emulator instances. -All three integrations run through the local `agent-device` daemon. `connect` checks the credentials and configuration, then saves non-secret connection state. It does not allocate a device. BrowserStack and AWS Device Farm allocate a hosted session on `open`. Limrun allocates an instance on the first device command, such as `install` or `open`. +All four integrations run through the local `agent-device` daemon. `connect` checks the credentials and configuration, then saves non-secret connection state. It does not allocate a device. BrowserStack, AWS Device Farm, and TestMu AI allocate a hosted session on `open`. Limrun allocates an instance on the first device command, such as `install` or `open`. For each provider, the standard lifecycle is: @@ -20,4 +21,4 @@ For each provider, the standard lifecycle is: 3. Follow the printed next command to install or open the app. 4. Run normal device commands, then `agent-device close` and `agent-device disconnect`. -Each provider guide covers its connection selectors, client configuration, MCP setup, artifacts, and troubleshooting. Generated remote profiles are safe to store as non-secret configuration. They may include app IDs, ARNs, device names, OS versions, and labels, but never provider API keys or AWS secret keys. +Each provider guide covers its connection selectors, client configuration, MCP setup, artifacts, and troubleshooting. Generated remote profiles are safe to store as non-secret configuration. They may include app IDs, ARNs, device names, OS versions, and labels, but never provider API keys, access keys, or AWS secret keys. diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md new file mode 100644 index 0000000000..c2adc8266d --- /dev/null +++ b/website/docs/docs/testmu.md @@ -0,0 +1,200 @@ +--- +title: TestMu AI +description: Drive TestMu AI (formerly LambdaTest) virtual devices, Android emulators and iOS simulators, and real devices with agent-device. +--- + +# TestMu AI + +TestMu AI (formerly LambdaTest) hosts virtual devices for Android emulator and iOS simulator +WebDriver sessions, and real devices you select with `--provider-device-type real`. One Appium hub +fronts both pools; agent-device selects the pool with `isRealMobile` and defaults to the +virtual-device pool. + +## Credentials and connection + +Set TestMu AI credentials in a non-interactive environment. These are the same variables every +TestMu AI SDK reads: + +```bash +export LT_USERNAME=... +export LT_ACCESS_KEY=... +``` + +Connect with the platform, exact device name and OS version, and the app to test: + +```bash +agent-device connect testmu \ + --platform android \ + --device "Galaxy S22 Ultra 5G" \ + --provider-os-version 14 \ + --provider-app lt://APP-id +``` + +`--device` and `--provider-os-version` must match the virtual-device catalog spelling exactly. The +hub rejects `--provider-os-version 18` for a device listed with `18.0`, so `connect` does too and +lists the versions the device offers. + +`--provider-app` accepts a TestMu AI app reference such as `lt://APP...`, an HTTP(S) app URL, or +an existing local app path (`.apk`, or a zipped simulator `.app` for iOS). TestMu AI uploads a local +path or fetches a URL when it creates the hosted session, through the virtual-device upload API. + +During `connect`, agent-device checks the device/OS pair against TestMu AI's virtual-device +catalog (`/capability/generator?isVirtualDevice=true`), verifies the credentials against your +uploaded-app listing, matches an `lt://` reference against that listing, and confirms that a local +artifact exists before saving its absolute path. `open` still needs the app's installed package or +bundle identifier, not the upload name or `lt://` id. + +Optional labels: + +```bash +--provider-project agent-device +--provider-build "$GITHUB_RUN_ID" +--provider-session-name "$GITHUB_JOB" +``` + +Optional device features: + +```bash +--provider-device-orientation portrait # or landscape (alias --device-orientation) +--provider-geo-location US # (alias --geo-location) +--provider-timezone UTC+05:30 # (alias --timezone) +--provider-appium-version 2.16.2 # (alias --appium-version) +--provider-language fr # (alias --language) +--provider-locale fr_FR # (alias --locale) +``` + +TestMu AI receives these values in `lt:options` when it creates the hosted session. + +- Without `--provider-appium-version`, agent-device sends no Appium version and TestMu AI starts + its default server for the device. Pin a version when a suite depends on one. +- `--provider-network-profile`, `--provider-custom-network`, and `--provider-no-resign-app` are + BrowserStack capabilities; `connect testmu` and TestMu AI session creation refuse them by flag + name rather than ignoring them. +- Session video and device logs are requested on every session so `artifacts` has something to + return. + +## Real devices + +Pass `--provider-device-type real` to run on a physical device. Everything else works as for +virtual devices: `connect` checks the device/OS pair against the real-device catalog +(`/capability/generator?isVirtualDevice=false`), and a local path or URL is uploaded through the +real-device upload API. + +```bash +agent-device connect testmu \ + --provider-device-type real \ + --platform ios \ + --device "iPhone 16" \ + --provider-os-version 18 \ + --provider-app ./MyApp.ipa + +agent-device connect testmu \ + --provider-device-type real \ + --platform android \ + --device "Pixel 6" \ + --provider-os-version 14 \ + --provider-app https://example.com/builds/app.apk +``` + +- Real iOS devices are listed by major OS version only: use `--provider-os-version 18`, not `18.0`. + The exact-spelling check still applies, so `connect` rejects `18.0` for a real iPhone 16 and lists + the versions it offers. +- Real iOS devices install a signed `.ipa`; a zipped simulator `.app` only runs on simulators. + Android takes an `.apk` or `.aab`. +- Real and virtual devices have separate upload APIs. Pass an `lt://` id that was uploaded for the + pool you connect to; when in doubt, pass the local path or URL and let agent-device upload it. +- `TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT` redirects real-device uploads, as + `TESTMU_APP_UPLOAD_ENDPOINT` does for virtual-device uploads. +- `--provider-device-type` applies only to TestMu AI; other providers refuse it. + +## CLI workflow + +```bash +export LT_USERNAME=... +export LT_ACCESS_KEY=... + +agent-device connect testmu \ + --platform ios \ + --device "iPhone 16" \ + --provider-os-version 18.0 \ + --provider-app ./MyApp.app.zip \ + --provider-build "$GITHUB_RUN_ID" + +agent-device open com.example.app +agent-device snapshot -i +agent-device click 'label="Continue"' +agent-device close +agent-device artifacts --json +agent-device disconnect +``` + +For MCP-only use, run `connect` in the same effective state directory before starting +`agent-device mcp`. MCP exposes `open`, `snapshot`, `click`, `close`, and `artifacts`, but not +provider `connect` commands. + +## Node.js client + +The typed client reaches TestMu AI through a lease. Allocate one with the provider selectors, then +scope a client to it for normal commands. `sessions.close()` ends the hosted session and releases +the lease; `leases.release()` in `finally` is then a no-op, and still releases the lease when a +command fails first. The daemon reads `LT_USERNAME` and `LT_ACCESS_KEY` from its environment. Add +`providerDeviceType: 'real'` to `leases.allocate` to run on a real device. + +```ts +import { createAgentDeviceClient } from 'agent-device'; + +const scope = { + tenant: 'testmu', + runId: process.env.GITHUB_RUN_ID ?? 'local-run', + leaseBackend: 'ios-instance', + leaseProvider: 'testmu', +} as const; + +const lease = await createAgentDeviceClient().leases.allocate({ + ...scope, + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18.0', + providerApp: 'lt://APP-id', + providerProject: 'agent-device', + providerBuild: process.env.GITHUB_RUN_ID, +}); +const client = createAgentDeviceClient({ ...scope, leaseId: lease.leaseId }); + +let providerSessionId: string | undefined; +try { + await client.apps.open({ app: 'com.example.app' }); + await client.capture.snapshot({ interactiveOnly: true }); + await client.interactions.click({ selector: 'label="Continue"' }); + const closed = await client.sessions.close(); + providerSessionId = closed.provider?.providerSessionId; +} finally { + await client.leases.release({ ...scope, leaseId: lease.leaseId }); +} + +if (providerSessionId) { + const artifacts = await client.sessions.artifacts({ provider: 'testmu', providerSessionId }); + if ('cloudArtifacts' in artifacts) console.log(artifacts.cloudArtifacts); +} +``` + +## Artifacts and troubleshooting + +After `close`, TestMu AI can return session video, Appium logs, device logs, network and command +logs, a screenshot archive, and the App Automation dashboard link. Run `agent-device artifacts +--json`, or look up a previous session explicitly: + +```bash +agent-device artifacts --provider testmu --json +``` + +The TestMu AI session id is the WebDriver session id. If artifact lookup is pending immediately +after `close`, retry it; TestMu AI finalizes video and log URLs after the session ends. + +Endpoints can be redirected for a staging or private TestMu AI deployment with +`TESTMU_WEBDRIVER_ENDPOINT`, `TESTMU_APP_UPLOAD_ENDPOINT` (virtual devices), +`TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT` (real devices), and `TESTMU_API_ENDPOINT`. + +On hosted WebDriver sessions, `fill` checks that the field received focus before it sends keys. If +it cannot confirm focus, it fails without typing. Use `snapshot -i` to confirm the target, or +`press ` followed by `type `.