diff --git a/packages/serve-sim/README.md b/packages/serve-sim/README.md index ee9bf66e..c3d37d68 100644 --- a/packages/serve-sim/README.md +++ b/packages/serve-sim/README.md @@ -38,6 +38,7 @@ Requires macOS with Xcode command line tools (`xcrun simctl`) and Node.js 18+. ` ``` serve-sim [device...] Start preview server (default: localhost:3200) serve-sim --no-preview [device...] Stream in foreground without a preview server +serve-sim device [device] Stream a physical iPhone/iPad (default: localhost:3300) serve-sim gesture '' [-d udid] Send a touch gesture serve-sim button [name] [-d udid] Send a button press (default: home) serve-sim type [-d udid] Type text via the simulator keyboard @@ -90,6 +91,11 @@ serve-sim --detach # start a background helper, return JSON serve-sim --list # show running streams serve-sim --kill # stop all helpers +# Stream a physical device (see “Physical devices” below for one-time setup) +serve-sim device # first connected iPhone/iPad +serve-sim device "iPhone" # target a device by name +serve-sim device --host 0.0.0.0 # expose the viewer on your LAN + # Type text into the focused field serve-sim type "Hello, world!" echo "from stdin" | serve-sim type --stdin @@ -128,6 +134,47 @@ Sources: - **file** — image (PNG/JPEG/HEIC/…) or video (mp4/mov/m4v/webm/…). The CLI sniffs the kind from the extension and falls back to magic bytes for files without an extension. - **webcam** — live `AVCaptureDevice` (built-in, Continuity, external). +## Physical devices + +`serve-sim device` streams a real iPhone or iPad to the browser — useful on Intel +Macs that can't boot Apple-silicon simulators, or any time you want to drive a +physical device. Real devices have no simulator framebuffer or synthetic-touch +APIs, so device mode drives [WebDriverAgent](https://github.com/appium/WebDriverAgent) +(an XCUITest runner) for the live screen plus tap/swipe/button input. + +```sh +serve-sim device # first connected device → http://localhost:3300 +serve-sim device "iPhone" # target a device by name or udid +serve-sim device --port 4000 # custom viewer port +serve-sim device --host 0.0.0.0 # expose on the LAN (viewer is unauthenticated) +``` + +The viewer at `http://localhost:3300` shows the live screen (MJPEG) and forwards +pointer events: a press is a tap, a press-and-drag is a swipe, and the **Home** +button presses the hardware home. The first run builds and code-signs +WebDriverAgent onto the device with your Apple ID — subsequent runs reuse it. + +### One-time setup + +Requires Xcode (not just the command line tools) and a free or paid Apple +Developer account for signing: + +```sh +pipx install pymobiledevice3 +git clone --depth 1 https://github.com/appium/WebDriverAgent.git \ + ~/.serve-sim-device/WebDriverAgent +``` + +Plug in the device and tap **Trust** when prompted, then run `serve-sim device`. + +### Environment overrides + +| Variable | Purpose | +| --- | --- | +| `SERVE_SIM_WDA_DIR` | Path to the WebDriverAgent checkout (default `~/.serve-sim-device/WebDriverAgent`). | +| `SERVE_SIM_WDA_TEAM` | Apple Developer Team ID for signing (auto-detected from your signing identity when omitted). | +| `SERVE_SIM_WDA_BUNDLE_ID` | Bundle id for the WDA runner (default `com.serve-sim.wda.runner`). | + ## Connectors `serve-sim` can be used with dev servers, browser, and AI editors for more seamless integration. diff --git a/packages/serve-sim/src/debug.ts b/packages/serve-sim/src/debug.ts index 773ea05c..786f6dfc 100644 --- a/packages/serve-sim/src/debug.ts +++ b/packages/serve-sim/src/debug.ts @@ -10,3 +10,6 @@ export const debugCli = createDebug("serve-sim:cli"); export const debugHelper = createDebug("serve-sim:helper"); export const debugState = createDebug("serve-sim:state"); export const debugMw = createDebug("serve-sim:mw"); +// Real-device (WebDriverAgent) mode: device detection, WDA runner lifecycle, +// usbmux port forwarding, and the MJPEG/input relay server. +export const debugDevice = createDebug("serve-sim:device"); diff --git a/packages/serve-sim/src/device-server.ts b/packages/serve-sim/src/device-server.ts new file mode 100644 index 00000000..c529491a --- /dev/null +++ b/packages/serve-sim/src/device-server.ts @@ -0,0 +1,317 @@ +/** + * Device-mode relay server. + * + * Bridges a browser to a physical iOS device that's being driven by + * WebDriverAgent (see `wda.ts`). It deliberately mirrors the shape of the + * native simulator helper's HTTP surface (`/stream.mjpeg`, `/config`, + * `/health`) so the mental model is the same, but the simulator client UI is + * heavily `simctl`-coupled (device lists, exec-on-host, WebKit devtools) and + * none of that applies to a real device — so device mode ships its own focused, + * self-contained viewer page at `/` instead. + * + * GET / → minimal viewer (MJPEG + pointer→/input) + * GET /stream.mjpeg → reverse-proxy WDA's on-device MJPEG server + * POST /input → JSON pointer event → WDA tap/drag/button + * GET /config → device logical window size + name + * GET /health → readiness probe + * + * Touch model: the viewer reports normalized 0..1 coordinates; we scale by the + * device's logical window size (points) before handing off to WDA. A press that + * doesn't move becomes a tap; a press that moves becomes a drag (so swipes and + * scrolls work). + */ +import { createServer, type IncomingMessage, type ServerResponse } from "http"; +import { get as httpGet } from "http"; +import { debugDevice } from "./debug"; +import type { WdaSession } from "./wda"; + +interface PointerEventBody { + type: "tap" | "drag" | "button"; + /** Normalized 0..1 (tap + drag start). */ + x?: number; + y?: number; + /** Normalized 0..1 (drag end). */ + x2?: number; + y2?: number; + /** For type: "button" — e.g. "home". */ + name?: string; +} + +export interface DeviceServer { + url: string; + stop(): Promise; +} + +/** + * Start the relay server bound to `host:port`, driving `session`. + * + * `host` defaults to 127.0.0.1; pass "0.0.0.0" to expose on the LAN (the viewer + * has no auth, so only do that on a trusted network). + */ +export async function startDeviceServer(opts: { + session: WdaSession; + port: number; + host?: string; +}): Promise { + const { session, port } = opts; + const host = opts.host ?? "127.0.0.1"; + + const server = createServer((req, res) => handle(req, res, session)); + // MJPEG is long-lived; disable the default socket/keep-alive timeouts. + server.keepAliveTimeout = 0; + server.headersTimeout = 0; + server.requestTimeout = 0; + server.timeout = 0; + + await new Promise((resolve, reject) => { + const onError = (err: Error) => { + server.removeListener("listening", onListening); + reject(err); + }; + const onListening = () => { + server.removeListener("error", onError); + resolve(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen(port, host); + }); + + return { + url: `http://${host === "0.0.0.0" ? "localhost" : host}:${port}`, + stop: () => + new Promise((resolve) => { + server.close(() => resolve()); + }), + }; +} + +function handle(req: IncomingMessage, res: ServerResponse, session: WdaSession): void { + const url = (req.url ?? "/").split("?")[0]; + + if (url === "/health") { + return json(res, 200, { status: "ok", device: session.device.udid }); + } + if (url === "/config") { + const size = session.getWindowSize(); + return json(res, 200, { + width: size?.width ?? 0, + height: size?.height ?? 0, + orientation: "portrait", + device: session.device.name, + productType: session.device.productType, + productVersion: session.device.productVersion, + }); + } + if (url === "/stream.mjpeg") { + return proxyMjpeg(res, session); + } + if (url === "/input" && req.method === "POST") { + return handleInput(req, res, session); + } + if (url === "/" || url === "/index.html") { + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + return void res.end(viewerHtml(session)); + } + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("Not found"); +} + +/** Reverse-proxy WDA's on-device MJPEG stream straight to the browser. */ +function proxyMjpeg(res: ServerResponse, session: WdaSession): void { + const upstream = httpGet(session.mjpegUrl, (up) => { + const contentType = + up.headers["content-type"] ?? "multipart/x-mixed-replace; boundary=--BoundaryString"; + res.writeHead(200, { + "Content-Type": contentType, + "Cache-Control": "no-cache, no-store", + "Connection": "keep-alive", + "Access-Control-Allow-Origin": "*", + }); + up.pipe(res); + res.on("close", () => up.destroy()); + }); + upstream.on("error", (err) => { + debugDevice("mjpeg upstream error: %s", err.message); + if (!res.headersSent) res.writeHead(502, { "Content-Type": "text/plain" }); + res.end("MJPEG upstream unavailable"); + }); +} + +function handleInput(req: IncomingMessage, res: ServerResponse, session: WdaSession): void { + let raw = ""; + let tooLarge = false; + req.on("data", (chunk) => { + if (tooLarge) return; + raw += chunk; + if (raw.length > 4096) { + // Pointer payloads are tiny; reject anything larger with a clear 413 + // before tearing down so the client gets a response, not a reset. + tooLarge = true; + json(res, 413, { error: "payload_too_large" }); + req.destroy(); + } + }); + req.on("end", () => { + if (tooLarge) return; + let body: PointerEventBody; + try { + body = JSON.parse(raw) as PointerEventBody; + } catch { + return json(res, 400, { error: "invalid_json" }); + } + const size = session.getWindowSize(); + if (!size) return json(res, 503, { error: "no_window_size" }); + + const toPoints = (n: number | undefined, axis: "x" | "y") => + clamp01(n ?? 0) * (axis === "x" ? size.width : size.height); + + void (async () => { + try { + if (body.type === "button") { + await session.pressButton(body.name ?? "home"); + } else if (body.type === "drag") { + await session.drag( + toPoints(body.x, "x"), + toPoints(body.y, "y"), + toPoints(body.x2, "x"), + toPoints(body.y2, "y"), + ); + } else { + await session.tap(toPoints(body.x, "x"), toPoints(body.y, "y")); + } + json(res, 200, { ok: true }); + } catch (err) { + json(res, 500, { error: (err as Error).message }); + } + })(); + }); +} + +function clamp01(n: number): number { + if (Number.isNaN(n)) return 0; + return n < 0 ? 0 : n > 1 ? 1 : n; +} + +function json(res: ServerResponse, status: number, obj: unknown): void { + const body = JSON.stringify(obj); + res.writeHead(status, { + "Content-Type": "application/json", + "Cache-Control": "no-cache, no-store", + "Access-Control-Allow-Origin": "*", + }); + res.end(body); +} + +/** Minimal self-contained viewer: MJPEG + pointer→/input bridge. */ +function viewerHtml(session: WdaSession): string { + const name = escapeHtml(session.device.name); + const subtitle = escapeHtml( + `${session.device.productType} · iOS ${session.device.productVersion}`, + ); + return ` + + + + +${name} · serve-sim + + + +
+

${name}

+

${subtitle}

+
+
+ device screen +
+
+ +
+
+ + +`; +} + +function escapeHtml(s: string): string { + return s + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} diff --git a/packages/serve-sim/src/index.ts b/packages/serve-sim/src/index.ts index f3d67fe7..be539cb8 100755 --- a/packages/serve-sim/src/index.ts +++ b/packages/serve-sim/src/index.ts @@ -1864,6 +1864,88 @@ function bindPreviewServer(port: number, middleware: ReturnType d.udid === deviceArg || d.name === deviceArg) + : detectRealDevice(); + if (!target) { + console.error(`Device not found: ${deviceArg}`); + console.error("Connected devices:"); + for (const d of devices) console.error(` ${d.name} (${d.udid}) — iOS ${d.productVersion}`); + process.exit(1); + } + + const config = defaultWdaConfig(); + const session = new WdaSession(target, config); + + console.log(`Starting device stream for ${target.name} (iOS ${target.productVersion})…`); + console.log("This launches WebDriverAgent on the device — first run builds + signs it."); + + try { + await session.start(); + } catch (err) { + if (err instanceof WdaSetupError) { + console.error(`\n${err.message}`); + if (err.instructions) console.error(`\n${err.instructions}`); + } else { + console.error(`Failed to start WebDriverAgent: ${(err as Error).message}`); + } + await session.stop(); + process.exit(1); + } + + const host = opts.host ?? "127.0.0.1"; + const server = await startDeviceServer({ session, port: opts.port ?? 3300, host }); + + const networkIP = getLocalNetworkIP(); + const exposedToLan = host !== "127.0.0.1" && host !== "localhost" && host !== "::1"; + console.log(""); + console.log(` - Local: ${server.url}`); + if (exposedToLan && networkIP) { + console.log(` - Network: http://${networkIP}:${opts.port ?? 3300}`); + } else if (networkIP) { + console.log(` - Network: \x1b[2muse --host 0.0.0.0 to expose on http://${networkIP}:${opts.port ?? 3300}\x1b[0m`); + } + console.log(""); + + const shutdown = async () => { + await server.stop(); + await session.stop(); + process.exit(0); + }; + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); + await new Promise(() => {}); +} + // ─── Main ─── const program = new Command(); @@ -2001,4 +2083,36 @@ program .argument("[args...]") .action((args: string[]) => permissions(args)); +program + .command("device") + .description("Stream a physical iPhone/iPad to the browser via WebDriverAgent") + .argument("[device]", "Device to target (udid or name; default: first connected)") + .option("-p, --port ", "Port for the device viewer (default: 3300)", (v) => parseInt(v, 10)) + .option( + "--host ", + "Interface to bind the viewer to. Use 0.0.0.0 to expose on the LAN " + + "(the viewer is unauthenticated — only on trusted networks).", + "127.0.0.1", + ) + .addHelpText( + "after", + ` +Real devices have no simulator framebuffer or synthetic-touch APIs, so this +drives WebDriverAgent (an XCUITest runner) for the live screen + tap/swipe. + +One-time setup: + pipx install pymobiledevice3 + git clone --depth 1 https://github.com/appium/WebDriverAgent.git ~/.serve-sim-device/WebDriverAgent + +Then: + serve-sim device Stream the first connected device + serve-sim device "iPhone" Stream a device by name + serve-sim device --host 0.0.0.0 Expose the viewer on your LAN + +Env overrides: SERVE_SIM_WDA_DIR, SERVE_SIM_WDA_TEAM, SERVE_SIM_WDA_BUNDLE_ID`, + ) + .action((deviceArg: string | undefined, opts: { port?: number; host?: string }) => + device(deviceArg, opts), + ); + await program.parseAsync(process.argv); diff --git a/packages/serve-sim/src/wda.ts b/packages/serve-sim/src/wda.ts new file mode 100644 index 00000000..6f46e765 --- /dev/null +++ b/packages/serve-sim/src/wda.ts @@ -0,0 +1,479 @@ +/** + * Real-device (physical iPhone/iPad) support via WebDriverAgent (WDA). + * + * serve-sim's native helper streams the *simulator* framebuffer over private + * CoreSimulator/SimulatorKit APIs and injects input over the simulator's + * synthetic-touch socket. None of that exists for a physical device. The only + * mechanism Apple ships that gives us BOTH a live screen and synthetic input on + * a real device is an XCUITest runner — i.e. WebDriverAgent, the same thing + * Appium drives. + * + * This module owns the WDA lifecycle for a single connected device: + * 1. detect the device over usbmux (`pymobiledevice3 usbmux list`), + * 2. launch the prebuilt WebDriverAgentRunner via `xcodebuild + * test-without-building` (it stays alive and hosts an HTTP control server + * on device port 8100 + an MJPEG server on 9100), + * 3. forward those two ports to localhost over usbmux, + * 4. health-check `:8100/status`, open a WDA session, and read the device's + * logical window size (used to map normalized 0..1 touch coords to points). + * + * The actual screen relay + input translation lives in `device-server.ts`; this + * file is purely "make WDA reachable and give me a typed client for it". + * + * Building/signing WebDriverAgentRunner onto the device is an inherently + * per-user Xcode operation (it needs your Apple Development identity and a + * 7-day free-provisioning cert), so it's a one-time prerequisite rather than + * something we do silently on every run. If the build product is missing we + * attempt a build when a team id is resolvable, otherwise we print the exact + * command to run. + */ +import { spawn, execFileSync, type ChildProcess } from "child_process"; +import { existsSync } from "fs"; +import { homedir } from "os"; +import { join } from "path"; +import { debugDevice } from "./debug"; + +/** A physical iOS device visible over usbmux. */ +export interface RealDevice { + udid: string; + name: string; + /** Marketing-ish product type, e.g. "iPhone17,1". */ + productType: string; + /** iOS version string, e.g. "26.5". */ + productVersion: string; +} + +export interface WdaConfig { + /** Checkout of appium/WebDriverAgent. */ + wdaDir: string; + /** Apple Developer Team ID (10-char) used to sign the runner. */ + teamId?: string; + /** Bundle id for the runner. Must be unique to the signing team. */ + bundleId: string; + /** Local port mapped to WDA's on-device control server (8100). */ + controlPort: number; + /** Local port mapped to WDA's on-device MJPEG server (9100). */ + mjpegPort: number; +} + +const DEFAULT_WDA_DIR = join(homedir(), ".serve-sim-device", "WebDriverAgent"); +const WDA_DEVICE_CONTROL_PORT = 8100; +const WDA_DEVICE_MJPEG_PORT = 9100; + +/** Locate the `pymobiledevice3` CLI (pipx installs it under ~/.local/bin). */ +export function findPymobiledevice3(): string | null { + const candidates = [ + join(homedir(), ".local", "bin", "pymobiledevice3"), + "/opt/homebrew/bin/pymobiledevice3", + "/usr/local/bin/pymobiledevice3", + ]; + for (const c of candidates) { + if (existsSync(c)) return c; + } + // Fall back to PATH lookup. + try { + const found = execFileSync("command", ["-v", "pymobiledevice3"], { + encoding: "utf-8", + shell: "/bin/bash", + }).trim(); + if (found) return found; + } catch {} + return null; +} + +/** List physical iOS devices connected over USB. */ +export function listRealDevices(): RealDevice[] { + const pmd = findPymobiledevice3(); + if (!pmd) return []; + try { + const out = execFileSync(pmd, ["usbmux", "list"], { + encoding: "utf-8", + timeout: 8_000, + stdio: ["ignore", "pipe", "ignore"], + }); + const data = JSON.parse(out) as Array<{ + ConnectionType?: string; + DeviceName?: string; + ProductType?: string; + ProductVersion?: string; + UniqueDeviceID?: string; + Identifier?: string; + }>; + return data + .filter((d) => (d.ConnectionType ?? "USB") === "USB") + .map((d) => ({ + udid: d.UniqueDeviceID ?? d.Identifier ?? "", + name: d.DeviceName ?? "iPhone", + productType: d.ProductType ?? "", + productVersion: d.ProductVersion ?? "", + })) + .filter((d) => d.udid.length > 0); + } catch (err) { + debugDevice("listRealDevices failed: %s", (err as Error).message); + return []; + } +} + +/** First connected physical device, or null. */ +export function detectRealDevice(): RealDevice | null { + return listRealDevices()[0] ?? null; +} + +/** + * Resolve the Apple Developer Team ID from the first "Apple Development" + * code-signing identity (the team id is the cert's OU field). Returns null when + * no identity is configured — the caller then asks the user to set + * SERVE_SIM_WDA_TEAM explicitly. + */ +export function resolveTeamId(): string | null { + if (process.env.SERVE_SIM_WDA_TEAM) return process.env.SERVE_SIM_WDA_TEAM; + try { + const list = execFileSync("security", ["find-identity", "-v", "-p", "codesigning"], { + encoding: "utf-8", + timeout: 5_000, + }); + const match = /"((?:Apple Development|iPhone Developer):[^"]+)"/.exec(list); + if (!match) return null; + const identity = match[1]!; + const pem = execFileSync("security", ["find-certificate", "-c", identity, "-p"], { + encoding: "utf-8", + timeout: 5_000, + }); + const subject = execFileSync("openssl", ["x509", "-noout", "-subject"], { + input: pem, + encoding: "utf-8", + timeout: 5_000, + }); + const ou = /OU\s*=\s*([A-Z0-9]{10})/.exec(subject); + return ou ? ou[1]! : null; + } catch (err) { + debugDevice("resolveTeamId failed: %s", (err as Error).message); + return null; + } +} + +export function defaultWdaConfig(): WdaConfig { + return { + wdaDir: process.env.SERVE_SIM_WDA_DIR ?? DEFAULT_WDA_DIR, + teamId: resolveTeamId() ?? undefined, + bundleId: process.env.SERVE_SIM_WDA_BUNDLE_ID ?? "com.serve-sim.wda.runner", + controlPort: WDA_DEVICE_CONTROL_PORT, + mjpegPort: WDA_DEVICE_MJPEG_PORT, + }; +} + +/** Logical screen size in points, as reported by WDA's /window/size. */ +export interface WindowSize { + width: number; + height: number; +} + +/** Raised when WDA can't be launched; `instructions` is user-facing setup help. */ +export class WdaSetupError extends Error { + constructor(message: string, readonly instructions?: string) { + super(message); + this.name = "WdaSetupError"; + } +} + +/** + * Owns a running WebDriverAgent for one device: the xcodebuild runner process, + * the two usbmux forwards, and an open WDA session. Provides a small typed + * client (tap/drag/pressButton) used by the relay server. + */ +export class WdaSession { + private runner?: ChildProcess; + private forwards: ChildProcess[] = []; + private sessionId?: string; + private windowSize?: WindowSize; + private stopped = false; + + constructor( + readonly device: RealDevice, + readonly config: WdaConfig, + ) {} + + get controlBase(): string { + return `http://127.0.0.1:${this.config.controlPort}`; + } + + get mjpegUrl(): string { + return `http://127.0.0.1:${this.config.mjpegPort}/`; + } + + getWindowSize(): WindowSize | undefined { + return this.windowSize; + } + + /** Path to the prebuilt WebDriverAgentRunner-Runner.app, if it exists. */ + private get runnerProductPath(): string { + return join( + this.config.wdaDir, + "build", + "Build", + "Products", + "Debug-iphoneos", + "WebDriverAgentRunner-Runner.app", + ); + } + + private commonBuildArgs(): string[] { + const args = [ + "-project", + join(this.config.wdaDir, "WebDriverAgent.xcodeproj"), + "-scheme", + "WebDriverAgentRunner", + "-destination", + `id=${this.device.udid}`, + "-derivedDataPath", + join(this.config.wdaDir, "build"), + "-allowProvisioningUpdates", + `PRODUCT_BUNDLE_IDENTIFIER=${this.config.bundleId}`, + "CODE_SIGN_STYLE=Automatic", + ]; + if (this.config.teamId) args.push(`DEVELOPMENT_TEAM=${this.config.teamId}`); + return args; + } + + /** Build + sign the runner if its product isn't already present. */ + private async ensureBuilt(): Promise { + if (existsSync(this.runnerProductPath)) { + debugDevice("WDA runner product already built at %s", this.runnerProductPath); + return; + } + if (!existsSync(join(this.config.wdaDir, "WebDriverAgent.xcodeproj"))) { + throw new WdaSetupError( + `WebDriverAgent checkout not found at ${this.config.wdaDir}`, + [ + "Set up WebDriverAgent once:", + ` git clone --depth 1 https://github.com/appium/WebDriverAgent.git "${this.config.wdaDir}"`, + "Then re-run serve-sim device (it will build + sign the runner).", + "Override the location with SERVE_SIM_WDA_DIR.", + ].join("\n"), + ); + } + if (!this.config.teamId) { + throw new WdaSetupError( + "No Apple Developer Team ID found to sign WebDriverAgent.", + [ + "Open Xcode once and sign in with your Apple ID (Settings → Accounts),", + "then set your team id explicitly:", + " export SERVE_SIM_WDA_TEAM=XXXXXXXXXX", + "(find it under Apple Developer → Membership, or via your signing cert).", + ].join("\n"), + ); + } + debugDevice("building WDA runner (team=%s bundle=%s)", this.config.teamId, this.config.bundleId); + await new Promise((resolve, reject) => { + const child = spawn("xcodebuild", ["build-for-testing", ...this.commonBuildArgs()], { + stdio: ["ignore", "pipe", "pipe"], + }); + let tail = ""; + const collect = (d: Buffer) => { + tail = (tail + d.toString()).slice(-4000); + }; + child.stdout?.on("data", collect); + child.stderr?.on("data", collect); + child.once("exit", (code) => { + if (code === 0 && existsSync(this.runnerProductPath)) resolve(); + else reject(new WdaSetupError(`WebDriverAgent build failed (exit ${code}).\n${tail}`)); + }); + child.once("error", reject); + }); + } + + /** Launch the runner; resolves once it logs its on-device server URL. */ + private launchRunner(): Promise { + return new Promise((resolve, reject) => { + const child = spawn( + "xcodebuild", + ["test-without-building", ...this.commonBuildArgs()], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + this.runner = child; + + let settled = false; + const onLine = (buf: Buffer) => { + const text = buf.toString(); + if (!settled && text.includes("ServerURLHere->")) { + settled = true; + debugDevice("WDA runner reported server URL on device"); + resolve(); + } + if (/Test Suite '.*' (failed|did not run)|TEST EXECUTE FAILED|Testing failed/.test(text)) { + if (!settled) { + settled = true; + reject(new WdaSetupError(`WebDriverAgent runner failed to launch.\n${text.slice(-2000)}`)); + } + } + }; + child.stdout?.on("data", onLine); + child.stderr?.on("data", onLine); + child.once("exit", (code) => { + if (!settled) { + settled = true; + reject(new WdaSetupError(`WebDriverAgent runner exited early (code ${code}).`)); + } + }); + child.once("error", (err) => { + if (!settled) { + settled = true; + reject(err); + } + }); + // Cap the wait so a hung runner doesn't block forever. + setTimeout(() => { + if (!settled) { + settled = true; + reject(new WdaSetupError("Timed out waiting for WebDriverAgent to start on the device.")); + } + }, 90_000); + }); + } + + /** usbmux-forward a single device port to the same local port. */ + private startForward(localPort: number, devicePort: number): ChildProcess { + const pmd = findPymobiledevice3(); + if (!pmd) throw new WdaSetupError("pymobiledevice3 not found (needed for usbmux port forwarding)."); + const child = spawn( + pmd, + ["usbmux", "forward", String(localPort), String(devicePort), "--serial", this.device.udid], + { stdio: ["ignore", "ignore", "ignore"] }, + ); + this.forwards.push(child); + return child; + } + + private async pollStatusReady(timeoutMs = 30_000): Promise { + const deadline = Date.now() + timeoutMs; + let lastErr = ""; + while (Date.now() < deadline) { + try { + const res = await wdaFetch(`${this.controlBase}/status`, {}, 2_000); + if (res.ok) { + const body = (await res.json()) as { value?: { ready?: boolean } }; + if (body.value?.ready) return; + } + } catch (err) { + lastErr = (err as Error).message; + } + await delay(300); + } + throw new WdaSetupError(`WebDriverAgent /status never became ready. ${lastErr}`); + } + + private async openSession(): Promise { + const res = await wdaFetch(`${this.controlBase}/session`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ capabilities: { alwaysMatch: {} } }), + }); + const body = (await res.json()) as { value?: { sessionId?: string }; sessionId?: string }; + this.sessionId = body.value?.sessionId ?? body.sessionId; + if (!this.sessionId) throw new WdaSetupError("Failed to create a WebDriverAgent session."); + await this.refreshWindowSize(); + } + + async refreshWindowSize(): Promise { + if (!this.sessionId) return undefined; + try { + const res = await wdaFetch(`${this.controlBase}/session/${this.sessionId}/window/size`); + const body = (await res.json()) as { value?: WindowSize }; + if (body.value) this.windowSize = body.value; + } catch (err) { + debugDevice("window/size failed: %s", (err as Error).message); + } + return this.windowSize; + } + + /** Full startup: build (if needed) → launch runner → forwards → session. */ + async start(): Promise { + await this.ensureBuilt(); + await this.launchRunner(); + this.startForward(this.config.controlPort, WDA_DEVICE_CONTROL_PORT); + this.startForward(this.config.mjpegPort, WDA_DEVICE_MJPEG_PORT); + await this.pollStatusReady(); + await this.openSession(); + debugDevice( + "WDA session ready: device=%s window=%o", + this.device.udid, + this.windowSize, + ); + } + + // ── Input (point coordinates are in logical points) ── + + async tap(xPoints: number, yPoints: number): Promise { + if (!this.sessionId) return; + await wdaFetch(`${this.controlBase}/session/${this.sessionId}/wda/tap`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ x: xPoints, y: yPoints }), + }).catch((err) => debugDevice("tap failed: %s", (err as Error).message)); + } + + async drag( + fromX: number, + fromY: number, + toX: number, + toY: number, + durationSec = 0.2, + ): Promise { + if (!this.sessionId) return; + await wdaFetch(`${this.controlBase}/session/${this.sessionId}/wda/dragfromtoforduration`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + fromX, + fromY, + toX, + toY, + duration: durationSec, + }), + }).catch((err) => debugDevice("drag failed: %s", (err as Error).message)); + } + + /** Press a hardware button. WDA supports "home", "volumeUp", "volumeDown". */ + async pressButton(name: string): Promise { + if (!this.sessionId) return; + await wdaFetch(`${this.controlBase}/session/${this.sessionId}/wda/pressButton`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name }), + }).catch((err) => debugDevice("pressButton failed: %s", (err as Error).message)); + } + + async stop(): Promise { + if (this.stopped) return; + this.stopped = true; + for (const f of this.forwards) { + try { f.kill("SIGTERM"); } catch {} + } + this.forwards = []; + if (this.runner) { + try { this.runner.kill("SIGTERM"); } catch {} + this.runner = undefined; + } + } +} + +function delay(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +/** + * Fetch against WDA over the usbmux forward. + * + * `pymobiledevice3 usbmux forward` is connection-per-request: it closes the TCP + * connection after each response. Node's global `fetch` (undici) pools sockets + * with keep-alive and will happily reuse one the forwarder has already closed, + * then block forever waiting for a response that never comes. Forcing + * `Connection: close` makes undici open a fresh socket per request, and the + * abort timeout is a belt-and-suspenders guard so a stuck call surfaces as an + * error instead of hanging the input handler. + */ +function wdaFetch(url: string, init: RequestInit = {}, timeoutMs = 8_000): Promise { + const headers = new Headers(init.headers); + headers.set("Connection", "close"); + return fetch(url, { ...init, headers, signal: AbortSignal.timeout(timeoutMs) }); +} diff --git a/skills/serve-sim/SKILL.md b/skills/serve-sim/SKILL.md index aba8f519..3a356e87 100644 --- a/skills/serve-sim/SKILL.md +++ b/skills/serve-sim/SKILL.md @@ -17,13 +17,13 @@ Drive an Apple Simulator (iOS, iPad, Apple Watch) from an agent using the [serve - The user wants to **simulate a memory warning** or **rotate the device** programmatically. - The user wants to **read the simulator's accessibility tree** to find UI elements without pixel hunting. - The user wants to **grant, revoke, or reset an app's privacy permissions** — camera, photos, location, contacts, or **push notifications**. +- The user wants to **stream a physical iPhone/iPad** to a browser with tap/swipe input — use `serve-sim device` (drives WebDriverAgent). Handy on Intel Macs that can't boot Apple-silicon simulators. ## When NOT to use - Android emulators → use `adb shell` tooling. - Building or installing an iOS app → use `xcodebuild` or `xcrun simctl install`. - React Native in-app runtime debugging (Redux state, network inspection, component tree) → use rn-debugger tooling. -- Real iOS hardware devices → use `xcrun devicectl` or Xcode. ## Prerequisites @@ -78,6 +78,7 @@ Key invariants the agent must respect: | Rotate device | `npx serve-sim rotate ` | `portrait`, `portrait_upside_down`, `landscape_left`, `landscape_right`. | | Simulate memory warning | `npx serve-sim memory-warning` | Equivalent to Debug → Simulate Memory Warning. | | CoreAnimation debug | `npx serve-sim ca-debug