From 250682ad636300a4425e0f515c09f402004242ab Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:22:38 +0100 Subject: [PATCH 01/24] test: touches, buttons, scrolls and keys on an iOS simulator reach the HID helper (#1019) --- src/drivers/ios/input/device-input.test.ts | 84 ++++++++++++++++++++++ src/drivers/ios/input/device-input.ts | 17 +++++ src/drivers/ios/input/index.ts | 1 + src/ports/byte-pipe.ts | 10 +++ 4 files changed, 112 insertions(+) create mode 100644 src/drivers/ios/input/device-input.test.ts create mode 100644 src/drivers/ios/input/device-input.ts create mode 100644 src/drivers/ios/input/index.ts diff --git a/src/drivers/ios/input/device-input.test.ts b/src/drivers/ios/input/device-input.test.ts new file mode 100644 index 000000000..f90d04d44 --- /dev/null +++ b/src/drivers/ios/input/device-input.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; + +import { + encodeScreenFrame, + INPUT_TAG, + isUnavailableInput, + type DeviceInput, + type InputScreenConfig, + type VirtualDriverDevice, +} from "../../../core/index.js"; +import { FakeBytePipeLauncher, FakeClock, ScriptedProcessRunner } from "../../../ports/index.js"; +import { createIosDeviceInput } from "./index.js"; + +const COMMAND = "/opt/simlock/simlock-screen-capture"; +const DEVICE_SET = "/home/.simlock/devices/ios"; + +function simulator(runtimeId = "com.apple.CoreSimulator.SimRuntime.iOS-26-5"): VirtualDriverDevice { + return { + address: "UDID-1", + deviceId: "UDID-1", + driverData: { + deviceTypeId: "com.apple.CoreSimulator.SimDeviceType.iPhone-16", + name: "simlock-1", + runtimeId, + udid: "UDID-1", + }, + physical: false, + }; +} + +function setup() { + const launcher = new FakeBytePipeLauncher(); + const clock = new FakeClock(); + const processRunner = new ScriptedProcessRunner([]); + const factory = createIosDeviceInput({ + baseEnvironment: { PATH: "/usr/bin" }, + clock, + command: COMMAND, + deviceSet: DEVICE_SET, + launcher, + processRunner, + }); + return { clock, factory, launcher, processRunner }; +} + +function envelope(tag: number, body: unknown): Uint8Array { + return encodeScreenFrame(tag, new TextEncoder().encode(JSON.stringify(body))); +} + +function decoded(writes: readonly Uint8Array[]): { tag: number; body: unknown }[] { + return writes.map((bytes) => ({ + body: JSON.parse(new TextDecoder().decode(bytes.subarray(5))) as unknown, + tag: bytes[4] ?? -1, + })); +} + +describe("iOS device input", () => { + it("touches, buttons, scrolls and keys on an iOS simulator reach the HID helper", async () => { + const { factory, launcher } = setup(); + const opening = factory.open(simulator(), new AbortController().signal); + expect(launcher.starts).toEqual([ + { + args: ["UDID-1", "--input"], + command: COMMAND, + env: { PATH: "/usr/bin", SIMLOCK_IOS_DEVICE_SET: DEVICE_SET }, + }, + ]); + launcher.last.write(envelope(INPUT_TAG.config, { height: 2556, width: 1179 })); + const input = (await opening) as DeviceInput; + const configs: InputScreenConfig[] = []; + input.onConfig((config) => configs.push(config)); + expect(configs).toEqual([{ height: 2556, orientation: "portrait", width: 1179 }]); + + input.send({ kind: "touch", type: "begin", x: 0.5, y: 0.25 }); + input.send({ kind: "touch", type: "move", x: 0.5, y: 0.5 }); + input.send({ kind: "touch", type: "end", x: 0.5, y: 0.5 }); + expect(decoded(launcher.last.stdinWrites)).toEqual([ + { body: { type: "begin", x: 0.5, y: 0.25 }, tag: INPUT_TAG.touch }, + { body: { type: "move", x: 0.5, y: 0.5 }, tag: INPUT_TAG.touch }, + { body: { type: "end", x: 0.5, y: 0.5 }, tag: INPUT_TAG.touch }, + ]); + expect(isUnavailableInput(input)).toBe(false); + }); +}); diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts new file mode 100644 index 000000000..fc8e14258 --- /dev/null +++ b/src/drivers/ios/input/device-input.ts @@ -0,0 +1,17 @@ +import type { DeviceInputFactory } from "../../../core/index.js"; +import type { BytePipeLauncher, Clock, Logger, ProcessRunner } from "../../../ports/index.js"; + +export interface IosDeviceInputOptions { + readonly command: string; + readonly launcher: BytePipeLauncher; + readonly clock: Clock; + readonly processRunner: ProcessRunner; + readonly baseEnvironment: NodeJS.ProcessEnv; + /** why: absent when the simulators live in Xcode's own set, which the helper searches by default. */ + readonly deviceSet?: string; + readonly logger?: Logger; +} + +export function createIosDeviceInput(_options: IosDeviceInputOptions): DeviceInputFactory { + return { open: () => Promise.resolve({ unavailable: "" }) }; +} diff --git a/src/drivers/ios/input/index.ts b/src/drivers/ios/input/index.ts new file mode 100644 index 000000000..d0ebe8063 --- /dev/null +++ b/src/drivers/ios/input/index.ts @@ -0,0 +1 @@ +export { createIosDeviceInput, type IosDeviceInputOptions } from "./device-input.js"; diff --git a/src/ports/byte-pipe.ts b/src/ports/byte-pipe.ts index 4b3c7e746..f9783229b 100644 --- a/src/ports/byte-pipe.ts +++ b/src/ports/byte-pipe.ts @@ -17,6 +17,9 @@ export interface BytePipeProcess { // why: never rejects -- a program the OS refused to start resolves with `error` set, so a caller // has one place to learn how it ended. readonly exited: Promise; + // why: a capture program in input mode reads its commands on stdin; rejects when the write fails, + // so a caller learns the program is gone. + writeStdin(bytes: Uint8Array): Promise; closeStdin(): void; kill(signal: "SIGTERM" | "SIGKILL"): void; } @@ -52,6 +55,7 @@ export class NodeBytePipeLauncher implements BytePipeLauncher { child.stdin.end(); }, exited, + writeStdin: () => Promise.resolve(), kill: (signal) => { if (child.exitCode === null && child.signalCode === null) child.kill(signal); }, @@ -72,6 +76,7 @@ export class FakeBytePipeProcess implements BytePipeProcess { readonly exited: Promise; stdinClosed = false; stdinCloses = 0; + readonly stdinWrites: Uint8Array[] = []; readonly kills: string[] = []; exitOnStdinClose = true; readonly #onStderrLine: ((line: string) => void) | undefined; @@ -110,6 +115,11 @@ export class FakeBytePipeProcess implements BytePipeProcess { this.#resolveExit(exit); } + writeStdin(bytes: Uint8Array): Promise { + this.stdinWrites.push(bytes); + return Promise.resolve(); + } + closeStdin(): void { this.stdinClosed = true; this.stdinCloses += 1; From c832b39831192b65cf8ab1873cd9e9594e8fd9e4 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:23:47 +0100 Subject: [PATCH 02/24] feat(input): an iOS simulator's device input writes touches to the capture program's input mode (#1019) --- src/daemon/main.ts | 1 + src/drivers/ios/index.ts | 19 ++++++ src/drivers/ios/input/device-input.ts | 96 +++++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 919415417..81a43585d 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -1325,6 +1325,7 @@ async function discoverIosDriver( filesystem: options.filesystem, idGenerator: options.idGenerator, instanceId: options.instanceId, + logger: logger.child("ios"), onSlimmed: emitSlimDiagnostic(options.eventBus), screenCapture: { baseEnvironment: process.env, diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 89853681d..15bd1ea3a 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -6,6 +6,7 @@ import { type ComponentInstallResult, type ComponentReceipt, type ComponentRemoval, + type DeviceInputFactory, COMPONENT_REMOVAL_TIMEOUT_MS, type ExactDeviceRequest, type Driver, @@ -50,6 +51,7 @@ import type { Clock, Filesystem, IdGenerator, + Logger, ProcessResult, ProcessRunner, } from "../../ports/index.js"; @@ -58,6 +60,7 @@ import { refuseOwnSetLeftovers, refuseXcodeSetLeftovers } from "./device-set-lef import { devicectlExecCommand } from "./devicectl-exec.js"; import { fillSimctlDevice } from "./simctl-device.js"; import { IosPhysicalHandler } from "./physical.js"; +import { createIosDeviceInput } from "./input/index.js"; import { createIosScreenCapture } from "./screen-capture.js"; import { labelsFor, resolveSlimCategories, slimSignature } from "./slim-labels.js"; @@ -216,6 +219,8 @@ export interface IosSimctlDriverOptions { readonly bootTimeoutMs: number; }; readonly slimByDefault?: boolean; + // why: the device input logs a Home press that did nothing; nothing else in the driver logs through it. + readonly logger?: Logger; // why: absent when this host has no way to run the capture program; the driver then offers no // screen at all rather than one that fails on every open. readonly screenCapture?: { @@ -349,9 +354,23 @@ export class IosSimctlDriver implements Driver { ? {} : { onLog: options.screenCapture.onLog }), }); + // why: the input is offered exactly when the screen capture is, because both are the one program. + this.input = + options.screenCapture === undefined + ? undefined + : createIosDeviceInput({ + baseEnvironment: options.screenCapture.baseEnvironment, + clock: options.clock, + command: configuredScreenCaptureCommand(options) ?? options.screenCapture.command, + ...(rootOptions === undefined ? {} : { deviceSet: deviceRoot }), + ...(options.logger === undefined ? {} : { logger: options.logger }), + launcher: options.screenCapture.launcher, + processRunner: options.processRunner, + }); } readonly screenCapture: ScreenCapture | undefined; + readonly input: DeviceInputFactory | undefined; /** * why: Establishes the device set this driver owns before it can be asked to do anything, diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts index fc8e14258..a9fb73075 100644 --- a/src/drivers/ios/input/device-input.ts +++ b/src/drivers/ios/input/device-input.ts @@ -1,5 +1,20 @@ -import type { DeviceInputFactory } from "../../../core/index.js"; -import type { BytePipeLauncher, Clock, Logger, ProcessRunner } from "../../../ports/index.js"; +import { + type DeviceInput, + type DeviceInputFactory, + encodeScreenFrame, + INPUT_TAG, + type InputMessage, + type InputScreenConfig, + type PhysicalDriverDevice, + type VirtualDriverDevice, +} from "../../../core/index.js"; +import type { + BytePipeLauncher, + BytePipeProcess, + Clock, + Logger, + ProcessRunner, +} from "../../../ports/index.js"; export interface IosDeviceInputOptions { readonly command: string; @@ -12,6 +27,79 @@ export interface IosDeviceInputOptions { readonly logger?: Logger; } -export function createIosDeviceInput(_options: IosDeviceInputOptions): DeviceInputFactory { - return { open: () => Promise.resolve({ unavailable: "" }) }; +const FRAME_HEADER_BYTES = 4; + +export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInputFactory { + return { + open: async (device: VirtualDriverDevice | PhysicalDriverDevice) => { + const child = options.launcher.start(options.command, [device.deviceId, "--input"], { + env: environmentFor(options), + }); + const config = await readConfig(child); + return new IosDeviceInput(child, config); + }, + }; +} + +// why: the daemon's own device set must not reach the helper, which would search it instead of the +// one this driver owns. +function environmentFor(options: IosDeviceInputOptions): NodeJS.ProcessEnv { + const { SIMLOCK_IOS_DEVICE_SET: _daemonDeviceSet, ...environment } = options.baseEnvironment; + return options.deviceSet === undefined + ? environment + : { ...environment, SIMLOCK_IOS_DEVICE_SET: options.deviceSet }; +} + +// why: the helper's stdout is the stream's envelope (a 4-byte length counting the tag, a tag, the +// payload); the first 0x82 is its size, written once its HID client is set up. +async function readConfig(child: BytePipeProcess): Promise { + const reader = child.stdout.getReader(); + let buffered = new Uint8Array(); + for (;;) { + const step = await reader.read(); + if (step.done) throw new Error("The input helper ended before it reported the screen size"); + buffered = Uint8Array.from([...buffered, ...step.value]); + if (buffered.length < FRAME_HEADER_BYTES) continue; + const length = new DataView(buffered.buffer).getUint32(0); + if (buffered.length < FRAME_HEADER_BYTES + length) continue; + const size = JSON.parse( + new TextDecoder().decode( + buffered.subarray(FRAME_HEADER_BYTES + 1, FRAME_HEADER_BYTES + length), + ), + ) as { width: number; height: number }; + return { height: size.height, orientation: "portrait", width: size.width }; + } +} + +function frameOf(tag: number, body: unknown): Uint8Array { + return encodeScreenFrame(tag, new TextEncoder().encode(JSON.stringify(body))); +} + +class IosDeviceInput implements DeviceInput { + readonly #child: BytePipeProcess; + readonly #config: InputScreenConfig; + + constructor(child: BytePipeProcess, config: InputScreenConfig) { + this.#child = child; + this.#config = config; + } + + send(message: InputMessage): void { + if (message.kind !== "touch") return; + const { type, x, y } = message; + void this.#child.writeStdin(frameOf(INPUT_TAG.touch, { type, x, y })); + } + + onConfig(listener: (config: InputScreenConfig) => void): () => void { + listener(this.#config); + return () => undefined; + } + + onFailed(): () => void { + return () => undefined; + } + + close(): Promise { + return Promise.resolve(); + } } From f13a4d5028135dd06fe76cc135cc30a1d768a890 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:24:30 +0100 Subject: [PATCH 03/24] test: writeStdin delivers bytes to a child's stdin and rejects when the write fails (#1019) --- .agents/scratch/pr-1019.md | 16 ++++++++++++++++ src/ports/byte-pipe.test.ts | 13 +++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 .agents/scratch/pr-1019.md diff --git a/.agents/scratch/pr-1019.md b/.agents/scratch/pr-1019.md new file mode 100644 index 000000000..8d7970924 --- /dev/null +++ b/.agents/scratch/pr-1019.md @@ -0,0 +1,16 @@ +Closes #1019 + +### Status + +Build in progress: cycle 1 (touches reach the helper's stdin). + +- [ ] The two Tests lines and the `byte-pipe` test pass in `pnpm test`, and `src/source-hygiene.test.ts` passes. +- [ ] `pnpm build:capture` builds `simlock-screen-capture` with the input mode where Xcode is selected. +- [ ] On a simulator, `e2e/slow-agent-device.test.ts` opens a lease and the Appduct fixture through agent-device, opens `WS /v1/leases/{id}/input`, gets 0x83 and 0x82, taps the centre of `tab-status` as agent-device's snapshot places it, and the next snapshot shows the Status tab's screen (slow lane, iOS simulator). +- [ ] `watching-the-screen.mdx` lists what input does on iOS: touch, scroll, Home, keys on iOS 26 runtimes; not the app switcher, Lock, rotation or keys on iOS 27 (#1024). + +## Assumptions + +- Assumption: where the iOS device input lives — `src/drivers/ios/input/` (an `index.ts` and `device-input.ts`, as Android's `input/`) instead of the spec's single `input.ts`, so its test goes through an `index.ts` (testing rule 6). + +_Written by an agent._ diff --git a/src/ports/byte-pipe.test.ts b/src/ports/byte-pipe.test.ts index 382109fbe..e16c2c58d 100644 --- a/src/ports/byte-pipe.test.ts +++ b/src/ports/byte-pipe.test.ts @@ -29,6 +29,19 @@ describe("NodeBytePipeLauncher", () => { expect(await child.exited).toEqual({ code: 7, signal: null }); }); + it("writeStdin delivers the exact bytes to the child's stdin and rejects once stdin is closed", async () => { + const child = new NodeBytePipeLauncher().start( + process.execPath, + ["-e", "process.stdin.pipe(process.stdout)"], + { env: process.env }, + ); + await child.writeStdin(Uint8Array.from([0, 255, 10])); + await child.writeStdin(Uint8Array.from([128, 1])); + child.closeStdin(); + expect([...(await readAll(child.stdout))]).toEqual([0, 255, 10, 128, 1]); + await expect(child.writeStdin(Uint8Array.from([7]))).rejects.toThrow(/write after end/i); + }); + it("reports each stderr line once it is complete", async () => { const lines: string[] = []; const child = new NodeBytePipeLauncher().start( From e6d03413524eb72aaa55dacd5675201faf0fea34 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:24:38 +0100 Subject: [PATCH 04/24] feat(ports): a byte pipe's stdin can be written, and a failed write rejects (#1019) --- src/ports/byte-pipe.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ports/byte-pipe.ts b/src/ports/byte-pipe.ts index f9783229b..ed1d2d9e2 100644 --- a/src/ports/byte-pipe.ts +++ b/src/ports/byte-pipe.ts @@ -55,7 +55,15 @@ export class NodeBytePipeLauncher implements BytePipeLauncher { child.stdin.end(); }, exited, - writeStdin: () => Promise.resolve(), + // why: unlike the stop, a write that fails is the caller's to learn about, so the error + // reaches the promise (the listener above only keeps it from being thrown). + writeStdin: (bytes) => + new Promise((resolve, reject) => { + child.stdin.write(bytes, (error) => { + if (error === null || error === undefined) resolve(); + else reject(error); + }); + }), kill: (signal) => { if (child.exitCode === null && child.signalCode === null) child.kill(signal); }, From c2fc99673c09da4d9808ec8b8976c586ca33e54d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:24:54 +0100 Subject: [PATCH 05/24] test: home on an iOS simulator relaunches SpringBoard and lock, app switcher and back are dropped (#1019) --- src/drivers/ios/input/device-input.test.ts | 28 ++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/drivers/ios/input/device-input.test.ts b/src/drivers/ios/input/device-input.test.ts index f90d04d44..f8514e008 100644 --- a/src/drivers/ios/input/device-input.test.ts +++ b/src/drivers/ios/input/device-input.test.ts @@ -31,7 +31,15 @@ function simulator(runtimeId = "com.apple.CoreSimulator.SimRuntime.iOS-26-5"): V function setup() { const launcher = new FakeBytePipeLauncher(); const clock = new FakeClock(); - const processRunner = new ScriptedProcessRunner([]); + const processRunner = new ScriptedProcessRunner([ + { + match: { + args: ["simctl", "--set", DEVICE_SET, "launch", "UDID-1", "com.apple.springboard"], + command: "xcrun", + }, + result: { code: 0, stderr: "", stdout: "" }, + }, + ]); const factory = createIosDeviceInput({ baseEnvironment: { PATH: "/usr/bin" }, clock, @@ -56,7 +64,7 @@ function decoded(writes: readonly Uint8Array[]): { tag: number; body: unknown }[ describe("iOS device input", () => { it("touches, buttons, scrolls and keys on an iOS simulator reach the HID helper", async () => { - const { factory, launcher } = setup(); + const { factory, launcher, processRunner } = setup(); const opening = factory.open(simulator(), new AbortController().signal); expect(launcher.starts).toEqual([ { @@ -79,6 +87,22 @@ describe("iOS device input", () => { { body: { type: "move", x: 0.5, y: 0.5 }, tag: INPUT_TAG.touch }, { body: { type: "end", x: 0.5, y: 0.5 }, tag: INPUT_TAG.touch }, ]); + const writtenBefore = launcher.last.stdinWrites.length; + + input.send({ button: "app_switcher", kind: "button" }); + input.send({ button: "lock", kind: "button" }); + input.send({ button: "back", kind: "button" }); + expect(processRunner.calls).toEqual([]); + expect(launcher.last.stdinWrites).toHaveLength(writtenBefore); + input.send({ button: "home", kind: "button" }); + expect(processRunner.calls).toEqual([ + { + args: ["simctl", "--set", DEVICE_SET, "launch", "UDID-1", "com.apple.springboard"], + command: "xcrun", + options: { timeoutMs: 10_000 }, + }, + ]); + expect(launcher.last.stdinWrites).toHaveLength(writtenBefore); expect(isUnavailableInput(input)).toBe(false); }); }); From 5298249dfdbb06ae65432fbbca7e2ad1d5bb8a33 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:25:16 +0100 Subject: [PATCH 06/24] feat(input): Home on an iOS simulator relaunches SpringBoard with simctl (#1019) --- src/drivers/ios/input/device-input.ts | 41 +++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts index a9fb73075..8377e5017 100644 --- a/src/drivers/ios/input/device-input.ts +++ b/src/drivers/ios/input/device-input.ts @@ -28,6 +28,8 @@ export interface IosDeviceInputOptions { } const FRAME_HEADER_BYTES = 4; +const HOME_TIMEOUT_MS = 10_000; +const SPRINGBOARD = "com.apple.springboard"; export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInputFactory { return { @@ -36,7 +38,7 @@ export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInpu env: environmentFor(options), }); const config = await readConfig(child); - return new IosDeviceInput(child, config); + return new IosDeviceInput(options, device.deviceId, child, config); }, }; } @@ -76,20 +78,55 @@ function frameOf(tag: number, body: unknown): Uint8Array { } class IosDeviceInput implements DeviceInput { + readonly #options: IosDeviceInputOptions; + readonly #udid: string; readonly #child: BytePipeProcess; readonly #config: InputScreenConfig; - constructor(child: BytePipeProcess, config: InputScreenConfig) { + constructor( + options: IosDeviceInputOptions, + udid: string, + child: BytePipeProcess, + config: InputScreenConfig, + ) { + this.#options = options; + this.#udid = udid; this.#child = child; this.#config = config; } send(message: InputMessage): void { + if (message.kind === "button") { + if (message.button === "home") this.#home(); + return; + } if (message.kind !== "touch") return; const { type, x, y } = message; void this.#child.writeStdin(frameOf(INPUT_TAG.touch, { type, x, y })); } + // why: Xcode 26 and later silently drop the HID home-button press, so Home brings SpringBoard + // forward the way serve-sim does; the app switcher and Lock have no such way and are dropped. + #home(): void { + const { deviceSet, logger, processRunner } = this.#options; + const args = [ + "simctl", + ...(deviceSet === undefined ? [] : ["--set", deviceSet]), + "launch", + this.#udid, + SPRINGBOARD, + ]; + void processRunner.run("xcrun", args, { timeoutMs: HOME_TIMEOUT_MS }).then( + (result) => { + if (result.code !== 0) + logger?.warn("Home did nothing", { stderr: result.stderr, udid: this.#udid }); + }, + (error: unknown) => { + logger?.warn("Home did nothing", { reason: String(error), udid: this.#udid }); + }, + ); + } + onConfig(listener: (config: InputScreenConfig) => void): () => void { listener(this.#config); return () => undefined; From 6c10eeb98a2892a1f4f1680d694a7d1f1d7f42fc Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:25:32 +0100 Subject: [PATCH 07/24] test: scrolls and keys on an iOS 26 simulator are written to the HID helper (#1019) --- src/drivers/ios/input/device-input.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/drivers/ios/input/device-input.test.ts b/src/drivers/ios/input/device-input.test.ts index f8514e008..89e4bd0d7 100644 --- a/src/drivers/ios/input/device-input.test.ts +++ b/src/drivers/ios/input/device-input.test.ts @@ -103,6 +103,17 @@ describe("iOS device input", () => { }, ]); expect(launcher.last.stdinWrites).toHaveLength(writtenBefore); + + input.send({ dx: 0, dy: 0.5, kind: "scroll", x: 0.5, y: 0.25 }); + input.send({ dx: -0.25, dy: 0, kind: "scroll" }); + input.send({ kind: "key", type: "down", usage: 0x04 }); + input.send({ kind: "key", type: "up", usage: 0x04 }); + expect(decoded(launcher.last.stdinWrites.slice(writtenBefore))).toEqual([ + { body: { dx: 0, dy: 0.5, x: 0.5, y: 0.25 }, tag: INPUT_TAG.scroll }, + { body: { dx: -0.25, dy: 0 }, tag: INPUT_TAG.scroll }, + { body: { type: "down", usage: 0x04 }, tag: INPUT_TAG.key }, + { body: { type: "up", usage: 0x04 }, tag: INPUT_TAG.key }, + ]); expect(isUnavailableInput(input)).toBe(false); }); }); From 6400082640f17ffb58b5dbd20381161e848f04b5 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:25:46 +0100 Subject: [PATCH 08/24] feat(input): scrolls and keys reach the iOS HID helper (#1019) --- src/drivers/ios/input/device-input.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts index 8377e5017..5abf4b861 100644 --- a/src/drivers/ios/input/device-input.ts +++ b/src/drivers/ios/input/device-input.ts @@ -73,6 +73,24 @@ async function readConfig(child: BytePipeProcess): Promise { } } +// why: the helper reads serve-sim's JSON for each tag; a scroll's position is left out when the +// client sent none, so the helper scrolls under the centre. +function frameOfMessage(message: Exclude): Uint8Array { + switch (message.kind) { + case "touch": + return frameOf(INPUT_TAG.touch, { type: message.type, x: message.x, y: message.y }); + case "scroll": + return frameOf(INPUT_TAG.scroll, { + dx: message.dx, + dy: message.dy, + ...(message.x === undefined ? {} : { x: message.x }), + ...(message.y === undefined ? {} : { y: message.y }), + }); + case "key": + return frameOf(INPUT_TAG.key, { type: message.type, usage: message.usage }); + } +} + function frameOf(tag: number, body: unknown): Uint8Array { return encodeScreenFrame(tag, new TextEncoder().encode(JSON.stringify(body))); } @@ -100,9 +118,7 @@ class IosDeviceInput implements DeviceInput { if (message.button === "home") this.#home(); return; } - if (message.kind !== "touch") return; - const { type, x, y } = message; - void this.#child.writeStdin(frameOf(INPUT_TAG.touch, { type, x, y })); + void this.#child.writeStdin(frameOfMessage(message)); } // why: Xcode 26 and later silently drop the HID home-button press, so Home brings SpringBoard From 5f9dca589814621d8052bf60e991dec3e8af5cb7 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:26:01 +0100 Subject: [PATCH 09/24] test: a key on an iOS 27 or unknown runtime is dropped and the config says keyboardUnavailable (#1019) --- src/drivers/ios/input/device-input.test.ts | 25 ++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/drivers/ios/input/device-input.test.ts b/src/drivers/ios/input/device-input.test.ts index 89e4bd0d7..7ba6b0d9d 100644 --- a/src/drivers/ios/input/device-input.test.ts +++ b/src/drivers/ios/input/device-input.test.ts @@ -14,14 +14,14 @@ import { createIosDeviceInput } from "./index.js"; const COMMAND = "/opt/simlock/simlock-screen-capture"; const DEVICE_SET = "/home/.simlock/devices/ios"; -function simulator(runtimeId = "com.apple.CoreSimulator.SimRuntime.iOS-26-5"): VirtualDriverDevice { +function simulator(runtimeId?: string): VirtualDriverDevice { return { address: "UDID-1", deviceId: "UDID-1", driverData: { deviceTypeId: "com.apple.CoreSimulator.SimDeviceType.iPhone-16", name: "simlock-1", - runtimeId, + runtimeId: runtimeId ?? "com.apple.CoreSimulator.SimRuntime.iOS-26-5", udid: "UDID-1", }, physical: false, @@ -115,5 +115,26 @@ describe("iOS device input", () => { { body: { type: "up", usage: 0x04 }, tag: INPUT_TAG.key }, ]); expect(isUnavailableInput(input)).toBe(false); + + for (const runtimeId of [ + "com.apple.CoreSimulator.SimRuntime.iOS-27-0", + "com.apple.CoreSimulator.SimRuntime.iOS-28-1", + "garbage", + ]) { + const other = setup(); + const opened = other.factory.open(simulator(runtimeId), new AbortController().signal); + other.launcher.last.write(envelope(INPUT_TAG.config, { height: 2556, width: 1179 })); + const limited = (await opened) as DeviceInput; + const limitedConfigs: InputScreenConfig[] = []; + limited.onConfig((config) => limitedConfigs.push(config)); + expect(limitedConfigs).toEqual([ + { height: 2556, keyboardUnavailable: true, orientation: "portrait", width: 1179 }, + ]); + limited.send({ kind: "key", type: "down", usage: 0x04 }); + limited.send({ kind: "touch", type: "begin", x: 0.5, y: 0.25 }); + expect(decoded(other.launcher.last.stdinWrites)).toEqual([ + { body: { type: "begin", x: 0.5, y: 0.25 }, tag: INPUT_TAG.touch }, + ]); + } }); }); From 2984ad9ee2a9e456df9eb8034279013355537977 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:26:20 +0100 Subject: [PATCH 10/24] feat(input): keys on an iOS 27 or unknown runtime are dropped and the config says so (#1019) --- src/drivers/ios/index.ts | 14 ++------------ src/drivers/ios/input/device-input.ts | 20 ++++++++++++++++++-- src/drivers/ios/runtime-version.ts | 13 +++++++++++++ 3 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 src/drivers/ios/runtime-version.ts diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 15bd1ea3a..52b8f3c1a 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -61,6 +61,7 @@ import { devicectlExecCommand } from "./devicectl-exec.js"; import { fillSimctlDevice } from "./simctl-device.js"; import { IosPhysicalHandler } from "./physical.js"; import { createIosDeviceInput } from "./input/index.js"; +import { iosRuntimeVersionFromId } from "./runtime-version.js"; import { createIosScreenCapture } from "./screen-capture.js"; import { labelsFor, resolveSlimCategories, slimSignature } from "./slim-labels.js"; @@ -2276,18 +2277,7 @@ function iosDriverData(device: DriverDevice): IosDriverData { }; } -export function iosRuntimeVersionFromId(runtimeId: string): readonly [number, number] | undefined { - const match = /iOS-(\d+)(?:-(\d+))?(?:-(\d+))?$/.exec(runtimeId); - if (match === null) { - return undefined; - } - const major = Number.parseInt(match[1] ?? "", 10); - const minor = match[2] === undefined ? 0 : Number.parseInt(match[2], 10); - if (Number.isNaN(major) || Number.isNaN(minor)) { - return undefined; - } - return [major, minor]; -} +export { iosRuntimeVersionFromId }; export function supportsPersistentSlim(version: readonly [number, number] | undefined): boolean { if (version === undefined) { diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts index 5abf4b861..2ed471122 100644 --- a/src/drivers/ios/input/device-input.ts +++ b/src/drivers/ios/input/device-input.ts @@ -15,6 +15,7 @@ import type { Logger, ProcessRunner, } from "../../../ports/index.js"; +import { iosRuntimeVersionFromId } from "../runtime-version.js"; export interface IosDeviceInputOptions { readonly command: string; @@ -37,8 +38,10 @@ export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInpu const child = options.launcher.start(options.command, [device.deviceId, "--input"], { env: environmentFor(options), }); - const config = await readConfig(child); - return new IosDeviceInput(options, device.deviceId, child, config); + const size = await readConfig(child); + const keys = keysAvailable(device); + const config: InputScreenConfig = keys ? size : { ...size, keyboardUnavailable: true }; + return new IosDeviceInput(options, device.deviceId, child, config, keys); }, }; } @@ -52,6 +55,15 @@ function environmentFor(options: IosDeviceInputOptions): NodeJS.ProcessEnv { : { ...environment, SIMLOCK_IOS_DEVICE_SET: options.deviceSet }; } +// why: the iOS 27 guest ignores injected keys (F21), so keys are dropped there; a runtime that cannot +// be read is treated the same, because a key that does nothing is worse than a key refused. +function keysAvailable(device: VirtualDriverDevice | PhysicalDriverDevice): boolean { + const data = device.driverData as { runtimeId?: unknown } | null; + const runtimeId = typeof data?.runtimeId === "string" ? data.runtimeId : ""; + const version = iosRuntimeVersionFromId(runtimeId); + return version !== undefined && version[0] < 27; +} + // why: the helper's stdout is the stream's envelope (a 4-byte length counting the tag, a tag, the // payload); the first 0x82 is its size, written once its HID client is set up. async function readConfig(child: BytePipeProcess): Promise { @@ -100,13 +112,16 @@ class IosDeviceInput implements DeviceInput { readonly #udid: string; readonly #child: BytePipeProcess; readonly #config: InputScreenConfig; + readonly #keys: boolean; constructor( options: IosDeviceInputOptions, udid: string, child: BytePipeProcess, config: InputScreenConfig, + keys: boolean, ) { + this.#keys = keys; this.#options = options; this.#udid = udid; this.#child = child; @@ -118,6 +133,7 @@ class IosDeviceInput implements DeviceInput { if (message.button === "home") this.#home(); return; } + if (message.kind === "key" && !this.#keys) return; void this.#child.writeStdin(frameOfMessage(message)); } diff --git a/src/drivers/ios/runtime-version.ts b/src/drivers/ios/runtime-version.ts new file mode 100644 index 000000000..10b28e566 --- /dev/null +++ b/src/drivers/ios/runtime-version.ts @@ -0,0 +1,13 @@ +// why: the iOS major and minor a simulator's runtime id names; the slim pass and the device input both decide by it. +export function iosRuntimeVersionFromId(runtimeId: string): readonly [number, number] | undefined { + const match = /iOS-(\d+)(?:-(\d+))?(?:-(\d+))?$/.exec(runtimeId); + if (match === null) { + return undefined; + } + const major = Number.parseInt(match[1] ?? "", 10); + const minor = match[2] === undefined ? 0 : Number.parseInt(match[2], 10); + if (Number.isNaN(major) || Number.isNaN(minor)) { + return undefined; + } + return [major, minor]; +} From dec2af81c38f47a2ea7684d158db78e3fb599c3c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:26:49 +0100 Subject: [PATCH 11/24] test: an iOS open rejects for a missing helper, an early exit or no size in 10 s, and a later exit fails the input (#1019) --- src/drivers/ios/input/device-input.test.ts | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/drivers/ios/input/device-input.test.ts b/src/drivers/ios/input/device-input.test.ts index 7ba6b0d9d..260074f87 100644 --- a/src/drivers/ios/input/device-input.test.ts +++ b/src/drivers/ios/input/device-input.test.ts @@ -137,4 +137,46 @@ describe("iOS device input", () => { ]); } }); + + it("an open rejects when the helper is missing, exits first or reports no size within 10 s, and a helper that exits later fails the input", async () => { + const missing = setup(); + const missingOpen = missing.factory.open(simulator(), new AbortController().signal); + missing.launcher.last.exit({ code: null, error: "spawn ENOENT", signal: null }); + await expect(missingOpen).rejects.toThrow( + "The input helper could not be started: spawn ENOENT", + ); + + const early = setup(); + const earlyOpen = early.factory.open(simulator(), new AbortController().signal); + early.launcher.last.exit({ code: 1, signal: null }); + await expect(earlyOpen).rejects.toThrow( + "The input helper exited with code 1 before it reported the screen size", + ); + + const silent = setup(); + const silentOpen = silent.factory.open(simulator(), new AbortController().signal).then( + () => "opened", + (error: unknown) => (error instanceof Error ? error.message : "not an error"), + ); + let settled = false; + void silentOpen.then(() => { + settled = true; + }); + silent.clock.advance(9_900); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + silent.clock.advance(100); + expect(await silentOpen).toBe("The input helper reported no screen size within 10 s"); + expect(silent.launcher.last.stdinClosed).toBe(true); + + const later = setup(); + const laterOpen = later.factory.open(simulator(), new AbortController().signal); + later.launcher.last.write(envelope(INPUT_TAG.config, { height: 2556, width: 1179 })); + const input = (await laterOpen) as DeviceInput; + const failures: string[] = []; + input.onFailed((error) => failures.push(error.message)); + later.launcher.last.exit({ code: 1, signal: null }); + await new Promise((resolve) => setImmediate(resolve)); + expect(failures).toEqual(["The input helper exited with code 1"]); + }); }); From cd26646e46a2b34a423e90abe4613f5463b2a2ce Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:27:25 +0100 Subject: [PATCH 12/24] feat(input): an iOS open fails for a missing, early-exiting or silent helper, and a later exit fails the input (#1019) --- src/drivers/ios/input/device-input.ts | 51 ++++++++++++++++++++++++--- src/drivers/ios/screen-capture.ts | 4 +-- src/ports/index.ts | 1 + 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts index 2ed471122..774cde56e 100644 --- a/src/drivers/ios/input/device-input.ts +++ b/src/drivers/ios/input/device-input.ts @@ -9,6 +9,7 @@ import { type VirtualDriverDevice, } from "../../../core/index.js"; import type { + BytePipeExit, BytePipeLauncher, BytePipeProcess, Clock, @@ -16,6 +17,7 @@ import type { ProcessRunner, } from "../../../ports/index.js"; import { iosRuntimeVersionFromId } from "../runtime-version.js"; +import { stopProgram } from "../screen-capture.js"; export interface IosDeviceInputOptions { readonly command: string; @@ -30,6 +32,9 @@ export interface IosDeviceInputOptions { const FRAME_HEADER_BYTES = 4; const HOME_TIMEOUT_MS = 10_000; +// why: the helper sets up its HID client before it reports a size; one that never does is stuck, and +// the open must not wait on it (architecture rule 11). +const READY_DEADLINE_MS = 10_000; const SPRINGBOARD = "com.apple.springboard"; export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInputFactory { @@ -38,7 +43,13 @@ export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInpu const child = options.launcher.start(options.command, [device.deviceId, "--input"], { env: environmentFor(options), }); - const size = await readConfig(child); + let size: InputScreenConfig; + try { + size = await within(READY_DEADLINE_MS, options.clock, readConfig(child)); + } catch (error) { + await stopProgram(child, options.clock); + throw error; + } const keys = keysAvailable(device); const config: InputScreenConfig = keys ? size : { ...size, keyboardUnavailable: true }; return new IosDeviceInput(options, device.deviceId, child, config, keys); @@ -71,7 +82,12 @@ async function readConfig(child: BytePipeProcess): Promise { let buffered = new Uint8Array(); for (;;) { const step = await reader.read(); - if (step.done) throw new Error("The input helper ended before it reported the screen size"); + if (step.done) { + const exit = await child.exited; + throw new Error( + `The input helper ${describeExit(exit)}${exit.error === undefined ? " before it reported the screen size" : ""}`, + ); + } buffered = Uint8Array.from([...buffered, ...step.value]); if (buffered.length < FRAME_HEADER_BYTES) continue; const length = new DataView(buffered.buffer).getUint32(0); @@ -85,6 +101,22 @@ async function readConfig(child: BytePipeProcess): Promise { } } +function describeExit(exit: BytePipeExit): string { + if (exit.error !== undefined) return `could not be started: ${exit.error}`; + return exit.signal === null + ? `exited with code ${String(exit.code)}` + : `was killed by ${exit.signal}`; +} + +function within(milliseconds: number, clock: Clock, work: Promise): Promise { + return new Promise((resolve, reject) => { + const timer = clock.setTimer(milliseconds, () => { + reject(new Error("The input helper reported no screen size within 10 s")); + }); + void work.then(resolve, reject).finally(() => clock.cancel(timer)); + }); +} + // why: the helper reads serve-sim's JSON for each tag; a scroll's position is left out when the // client sent none, so the helper scrolls under the centre. function frameOfMessage(message: Exclude): Uint8Array { @@ -113,6 +145,8 @@ class IosDeviceInput implements DeviceInput { readonly #child: BytePipeProcess; readonly #config: InputScreenConfig; readonly #keys: boolean; + readonly #failedListeners = new Set<(error: Error) => void>(); + #closing: Promise | undefined; constructor( options: IosDeviceInputOptions, @@ -122,6 +156,11 @@ class IosDeviceInput implements DeviceInput { keys: boolean, ) { this.#keys = keys; + void child.exited.then((exit) => { + if (this.#closing !== undefined) return; + const error = new Error(`The input helper ${describeExit(exit)}`); + for (const listener of this.#failedListeners) listener(error); + }); this.#options = options; this.#udid = udid; this.#child = child; @@ -164,11 +203,13 @@ class IosDeviceInput implements DeviceInput { return () => undefined; } - onFailed(): () => void { - return () => undefined; + onFailed(listener: (error: Error) => void): () => void { + this.#failedListeners.add(listener); + return () => this.#failedListeners.delete(listener); } close(): Promise { - return Promise.resolve(); + this.#closing ??= stopProgram(this.#child, this.#options.clock); + return this.#closing; } } diff --git a/src/drivers/ios/screen-capture.ts b/src/drivers/ios/screen-capture.ts index 45efd81be..813cf3683 100644 --- a/src/drivers/ios/screen-capture.ts +++ b/src/drivers/ios/screen-capture.ts @@ -114,7 +114,7 @@ function startCapture( }, }, ); - const close = once(() => stop(child, options.clock)); + const close = once(() => stopProgram(child, options.clock)); return { body: bodyOf(child, tail, close), close }; } @@ -172,7 +172,7 @@ function bodyOf( }); } -async function stop(child: BytePipeProcess, clock: Clock): Promise { +export async function stopProgram(child: BytePipeProcess, clock: Clock): Promise { child.closeStdin(); if (await exitsWithin(child, clock, STOP_GRACE_MS)) return; child.kill("SIGKILL"); diff --git a/src/ports/index.ts b/src/ports/index.ts index 4d1a09b13..ae9e3fd0c 100644 --- a/src/ports/index.ts +++ b/src/ports/index.ts @@ -110,6 +110,7 @@ export { type TcpStreamConnector, } from "./tcp-stream.js"; export { + type BytePipeExit, type BytePipeLauncher, type BytePipeProcess, FakeBytePipeLauncher, From 02aad417469d2ea9a3df67d85cabd3cafc3ed087 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:27:36 +0100 Subject: [PATCH 13/24] test: a physical iPhone's device input is unavailable and no process is started (#1019) --- src/drivers/ios/input/device-input.test.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/drivers/ios/input/device-input.test.ts b/src/drivers/ios/input/device-input.test.ts index 260074f87..0fce402ce 100644 --- a/src/drivers/ios/input/device-input.test.ts +++ b/src/drivers/ios/input/device-input.test.ts @@ -179,4 +179,24 @@ describe("iOS device input", () => { await new Promise((resolve) => setImmediate(resolve)); expect(failures).toEqual(["The input helper exited with code 1"]); }); + + it("a physical iPhone's device input is unavailable and no process is started", async () => { + const { clock, factory, launcher } = setup(); + const pending = factory + .open( + { + address: "UDID-1", + deviceId: "UDID-1", + driverData: {}, + enrolledApps: [], + physical: true, + }, + new AbortController().signal, + ) + .catch((error: unknown) => (error instanceof Error ? error.message : "not an error")); + clock.advance(10_000); + const opening = await pending; + expect(opening).toEqual({ unavailable: "Input is not supported on physical iPhones" }); + expect(launcher.starts).toEqual([]); + }); }); From cd2035d28512ebab174bcb3233cc9fb803b3d993 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:28:33 +0100 Subject: [PATCH 14/24] feat(input): a physical iPhone's device input is unavailable (#1019) --- .agents/scratch/green7.patch | 23 +++++++++++++++++++++++ src/drivers/ios/input/device-input.ts | 5 +++++ 2 files changed, 28 insertions(+) create mode 100644 .agents/scratch/green7.patch diff --git a/.agents/scratch/green7.patch b/.agents/scratch/green7.patch new file mode 100644 index 000000000..8dc8cddb5 --- /dev/null +++ b/.agents/scratch/green7.patch @@ -0,0 +1,23 @@ +diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts +index 774cde56e..7a6d2ff47 100644 +--- a/src/drivers/ios/input/device-input.ts ++++ b/src/drivers/ios/input/device-input.ts +@@ -3,6 +3,7 @@ import { + type DeviceInputFactory, + encodeScreenFrame, + INPUT_TAG, ++ isPhysicalDriverDevice, + type InputMessage, + type InputScreenConfig, + type PhysicalDriverDevice, +@@ -40,6 +41,10 @@ const SPRINGBOARD = "com.apple.springboard"; + export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInputFactory { + return { + open: async (device: VirtualDriverDevice | PhysicalDriverDevice) => { ++ // why: nothing on a physical iPhone injects input (F23); only a simulator has a HID helper. ++ if (isPhysicalDriverDevice(device)) { ++ return { unavailable: "Input is not supported on physical iPhones" }; ++ } + const child = options.launcher.start(options.command, [device.deviceId, "--input"], { + env: environmentFor(options), + }); diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts index 774cde56e..7a6d2ff47 100644 --- a/src/drivers/ios/input/device-input.ts +++ b/src/drivers/ios/input/device-input.ts @@ -3,6 +3,7 @@ import { type DeviceInputFactory, encodeScreenFrame, INPUT_TAG, + isPhysicalDriverDevice, type InputMessage, type InputScreenConfig, type PhysicalDriverDevice, @@ -40,6 +41,10 @@ const SPRINGBOARD = "com.apple.springboard"; export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInputFactory { return { open: async (device: VirtualDriverDevice | PhysicalDriverDevice) => { + // why: nothing on a physical iPhone injects input (F23); only a simulator has a HID helper. + if (isPhysicalDriverDevice(device)) { + return { unavailable: "Input is not supported on physical iPhones" }; + } const child = options.launcher.start(options.command, [device.deviceId, "--input"], { env: environmentFor(options), }); From 8b4b14223c26db2cb8e348500061eb770a62625d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:28:49 +0100 Subject: [PATCH 15/24] test: closing stops the iOS helper without failing the input, and a failed write fails it (#1019) --- src/drivers/ios/input/device-input.test.ts | 25 ++++++++++++++++++++++ src/ports/byte-pipe.ts | 3 +++ 2 files changed, 28 insertions(+) diff --git a/src/drivers/ios/input/device-input.test.ts b/src/drivers/ios/input/device-input.test.ts index 0fce402ce..23b04ecb1 100644 --- a/src/drivers/ios/input/device-input.test.ts +++ b/src/drivers/ios/input/device-input.test.ts @@ -180,6 +180,31 @@ describe("iOS device input", () => { expect(failures).toEqual(["The input helper exited with code 1"]); }); + it("closing stops the helper without failing the input, and a failed write fails the input and stops the helper", async () => { + const closing = setup(); + const closingOpen = closing.factory.open(simulator(), new AbortController().signal); + closing.launcher.last.write(envelope(INPUT_TAG.config, { height: 2556, width: 1179 })); + const closable = (await closingOpen) as DeviceInput; + const closeFailures: string[] = []; + closable.onFailed((error) => closeFailures.push(error.message)); + await closable.close(); + expect(closing.launcher.last.stdinClosed).toBe(true); + await new Promise((resolve) => setImmediate(resolve)); + expect(closeFailures).toEqual([]); + + const broken = setup(); + const brokenOpen = broken.factory.open(simulator(), new AbortController().signal); + broken.launcher.last.write(envelope(INPUT_TAG.config, { height: 2556, width: 1179 })); + const input = (await brokenOpen) as DeviceInput; + const failures: string[] = []; + input.onFailed((error) => failures.push(error.message)); + broken.launcher.last.writeError = new Error("write EPIPE"); + input.send({ kind: "touch", type: "begin", x: 0.5, y: 0.5 }); + await new Promise((resolve) => setImmediate(resolve)); + expect(failures).toEqual(["The input helper stopped taking input: write EPIPE"]); + expect(broken.launcher.last.stdinClosed).toBe(true); + }); + it("a physical iPhone's device input is unavailable and no process is started", async () => { const { clock, factory, launcher } = setup(); const pending = factory diff --git a/src/ports/byte-pipe.ts b/src/ports/byte-pipe.ts index ed1d2d9e2..24e027f9b 100644 --- a/src/ports/byte-pipe.ts +++ b/src/ports/byte-pipe.ts @@ -87,6 +87,8 @@ export class FakeBytePipeProcess implements BytePipeProcess { readonly stdinWrites: Uint8Array[] = []; readonly kills: string[] = []; exitOnStdinClose = true; + // why: a test sets this to make every later write fail, as a pipe whose reader has gone does. + writeError: Error | undefined; readonly #onStderrLine: ((line: string) => void) | undefined; #controller!: ReadableStreamDefaultController; #resolveExit!: (exit: BytePipeExit) => void; @@ -124,6 +126,7 @@ export class FakeBytePipeProcess implements BytePipeProcess { } writeStdin(bytes: Uint8Array): Promise { + if (this.writeError !== undefined) return Promise.reject(this.writeError); this.stdinWrites.push(bytes); return Promise.resolve(); } From 01bc334cc2dd250995b8032005300c1fb4694bd4 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:29:04 +0100 Subject: [PATCH 16/24] feat(input): a failed write fails the iOS input and stops the helper (#1019) --- src/drivers/ios/input/device-input.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts index 7a6d2ff47..de8e45c89 100644 --- a/src/drivers/ios/input/device-input.ts +++ b/src/drivers/ios/input/device-input.ts @@ -162,9 +162,7 @@ class IosDeviceInput implements DeviceInput { ) { this.#keys = keys; void child.exited.then((exit) => { - if (this.#closing !== undefined) return; - const error = new Error(`The input helper ${describeExit(exit)}`); - for (const listener of this.#failedListeners) listener(error); + this.#fail(new Error(`The input helper ${describeExit(exit)}`)); }); this.#options = options; this.#udid = udid; @@ -178,7 +176,13 @@ class IosDeviceInput implements DeviceInput { return; } if (message.kind === "key" && !this.#keys) return; - void this.#child.writeStdin(frameOfMessage(message)); + this.#child.writeStdin(frameOfMessage(message)).catch((error: unknown) => { + this.#fail( + new Error( + `The input helper stopped taking input: ${error instanceof Error ? error.message : String(error)}`, + ), + ); + }); } // why: Xcode 26 and later silently drop the HID home-button press, so Home brings SpringBoard @@ -208,6 +212,14 @@ class IosDeviceInput implements DeviceInput { return () => undefined; } + // why: a helper that is closing, or already failed, has told its listeners; a second word would + // close a socket that is already closed. + #fail(error: Error): void { + if (this.#closing !== undefined) return; + void this.close(); + for (const listener of this.#failedListeners) listener(error); + } + onFailed(listener: (error: Error) => void): () => void { this.#failedListeners.add(listener); return () => this.#failedListeners.delete(listener); From aa4e6721cbb1a82849a7e9f2e531974a61c48c0b Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:31:53 +0100 Subject: [PATCH 17/24] feat(input): the capture program's input mode, the vendored HID injector, and the docs for iOS input (#1019) --- docs/internal/VENDORED-CODE.md | 2 +- e2e/slow-agent-device.test.ts | 3 +- src/drivers/ios/AGENTS.md | 12 + src/ports/AGENTS.md | 2 +- vendor/README.md | 2 +- vendor/screen-capture/NOTICE.md | 57 +- vendor/screen-capture/Package.swift | 3 + .../simlock-screen-capture/FrameCapture.swift | 6 +- .../simlock-screen-capture/HIDInjector.swift | 613 ++++++++++++++++++ .../simlock-screen-capture/InputMode.swift | 91 +++ .../Sources/simlock-screen-capture/main.swift | 13 +- .../using-devices/watching-the-screen.mdx | 4 +- 12 files changed, 783 insertions(+), 25 deletions(-) create mode 100644 vendor/screen-capture/Sources/simlock-screen-capture/HIDInjector.swift create mode 100644 vendor/screen-capture/Sources/simlock-screen-capture/InputMode.swift diff --git a/docs/internal/VENDORED-CODE.md b/docs/internal/VENDORED-CODE.md index 0d4a11f74..4a8ec7951 100644 --- a/docs/internal/VENDORED-CODE.md +++ b/docs/internal/VENDORED-CODE.md @@ -14,7 +14,7 @@ license text(s) they require under `LICENSES/`, and a `NOTICE.md`. | What | Upstream | Version / commit | License | Where it lives | Consumed by | Re-verify with | |---|---|---|---|---|---|---| -| serve-sim's capture and encode pipeline — CoreSimulator framebuffer capture and the H.264 encode path, patched by us, with four files rewritten by us that stay serve-sim's modified files | `EvanBacon/serve-sim` | commit `6d009686ed08f7d8bd19bca9f9f67a9237931f44` | Apache-2.0 for serve-sim's files and our rewrites of four of them; MIT for the files that are ours entirely | `vendor/screen-capture/Sources/`, licenses at `vendor/screen-capture/LICENSES/serve-sim.APACHE-2.0.txt` and `…/idevice.MIT.txt` | the `simlock-screen-capture` and `simlock-device-capture` programs `pnpm build` puts in dist/bin, run by `src/drivers/ios/screen-capture.ts` | the `curl … \| diff -u -` command in [`vendor/screen-capture/NOTICE.md`](../../vendor/screen-capture/NOTICE.md), per file | +| serve-sim's capture, encode and HID-input pipeline — CoreSimulator framebuffer capture, the H.264 encode path and the `HIDInjector` that injects touches and keys, patched by us, with four files rewritten by us that stay serve-sim's modified files | `EvanBacon/serve-sim` | commit `6d009686ed08f7d8bd19bca9f9f67a9237931f44` | Apache-2.0 for serve-sim's files and our rewrites of four of them; MIT for the files that are ours entirely | `vendor/screen-capture/Sources/`, licenses at `vendor/screen-capture/LICENSES/serve-sim.APACHE-2.0.txt` and `…/idevice.MIT.txt` | the `simlock-screen-capture` and `simlock-device-capture` programs `pnpm build` puts in dist/bin, run by `src/drivers/ios/screen-capture.ts` and, in its `--input` mode, `src/drivers/ios/input/` | the `curl … \| diff -u -` command in [`vendor/screen-capture/NOTICE.md`](../../vendor/screen-capture/NOTICE.md), per file | | the HEVC capability string the iPhone's media-stream offer advertises for payload 100 — one string inside one of our own files | `jkcoxson/idevice` | `master` at the time of taking; the string is what its receiver documents | MIT | `vendor/screen-capture/LICENSES/idevice.MIT.txt`, the string in `vendor/screen-capture/Sources/SimlockMediaBridge/SimlockMediaBridge.m` | `simlock-device-capture`, whose offer a physical iPhone accepts without painting torn pictures | the string in `Sources/SimlockMediaBridge/SimlockMediaBridge.m` against idevice's receiver, and its `LICENSE.txt` at `https://raw.githubusercontent.com/jkcoxson/idevice/master/LICENSE.txt` | | `scrcpy-server` — the Android screen-streaming server artifact, byte-identical to upstream's release asset | `Genymobile/scrcpy` | release `v5.0.1`; revision built `dc8a55a24c27ae78ae202b5a3288da681d0f141a` | Apache-2.0; the Kotlin standard library inside it adds BSD-3-Clause (ThreeTen-BP) and BSL-1.0 (Boost) code | `vendor/scrcpy/scrcpy-server-v5.0.1`, licenses at `vendor/scrcpy/LICENSES/scrcpy.APACHE-2.0.txt` and `vendor/scrcpy/LICENSES/threetenbp.BSD-3-Clause.txt`, digest in `vendor/scrcpy/SHA256SUMS.txt` | the Android driver's screen capture, which pushes it unmodified to a leased emulator; the client that drives it is ours, in `src/drivers/android/`, and is not vendored | `cd vendor/scrcpy && shasum -a 256 -c SHA256SUMS.txt`, and the same against upstream's own release `SHA256SUMS.txt`, whose GPG signature `vendor/scrcpy/NOTICE.md` says how to check | diff --git a/e2e/slow-agent-device.test.ts b/e2e/slow-agent-device.test.ts index 64951a107..0e4896643 100644 --- a/e2e/slow-agent-device.test.ts +++ b/e2e/slow-agent-device.test.ts @@ -236,8 +236,7 @@ function defineSuite(target: Target): void { }, ); - // why: the input socket's iOS device input arrives with #1019, so the case waits for it there. - it.skipIf(platform === "ios")( + it( "an agent taps the centre of tab-status through the lease's input socket, as agent-device's snapshot places it, and the next snapshot shows the Status tab's screen", { timeout: 600_000 }, async () => { diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index 3eba13e83..36d0b0bef 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -16,6 +16,8 @@ Parent: [drivers](../AGENTS.md). Flows: [device selection](../../../docs/interna | `devicectl-exec.ts` | `devicectlExecCommand`: the reviewed `devicectl` list | | `slim-labels.ts` | `SLIM_CATEGORIES`, `resolveSlimCategories`, `labelsFor`, `slimSignature` | | `prerequisites.ts` | `iosPrerequisites`: Xcode selected, license accepted, first launch done | +| `input/` | `createIosDeviceInput`: the lease's input socket on a simulator, through `simlock-screen-capture --input`; a physical iPhone answers unavailable | +| `runtime-version.ts` | `iosRuntimeVersionFromId`: the iOS major and minor a runtime id names | | `screen-capture.ts` | `createIosScreenCapture`: runs `vendor/screen-capture/` on a simulator or a physical iPhone | ## How it works @@ -93,6 +95,15 @@ Programs: dist/bin from `pnpm build`, or `drivers.ios.screenCaptureCommand` and program enforces its pin: a CoreDevice major other than 642 exits 69 with the reason, a 503 for the viewer. Asked to stop while still starting, a program stops and goes inside its 2 s. +Input — `input` (offered with `screenCapture`, same launcher): `simlock-screen-capture --input` +with the same `SIMLOCK_IOS_DEVICE_SET` rule. stdout: one `0x82` `{width,height}` (the device type's native +size) within 10 s of spawn, else the open rejects (also for a missing program or an early exit; the helper is +stopped). stdin: envelopes `[len][tag]{json}` for touch `0x03`, key `0x06` and scroll `0x0b`; a failed write or a later +exit calls `onFailed`. Home runs `xcrun simctl [--set ] launch com.apple.springboard` (10 s; a failure is +logged); the app switcher, Lock and Back are dropped. The config is always `portrait`, with `keyboardUnavailable: true` +when the runtime is iOS 27 or later or unreadable, and keys are then dropped. Close: stdin, 2 s, SIGKILL. A physical iPhone answers +`{unavailable}` and starts nothing. + Advisories — `runtime-cache-unreclaimable` (asset bundles no installed runtime mounts); `slim-runtime-unsupported` (installed runtimes below 18.5, only when `ios.defaultMode` is `slim`). @@ -122,6 +133,7 @@ record of a device's mode; `driverData` holds the slim signature. ## Known gaps +- **Input:** no app switcher, Lock, rotation or keys on iOS 27 runtimes (#1024). The input follows the device's native frame, so after a rotation taps and picture stay in the same frame. After the health monitor reboots a crashed simulator the helper's HID client is bound to the old boot, so taps may vanish until the lease's input reopens. - **A restart between `simctl create` and the durable mark under `default` (#968)** leaves one unlisted `simlock-` simulator in Xcode's set. Delete it with `xcrun simctl delete `. - **A runtime download outlives the runtime (#79).** `simctl runtime delete` leaves its ~8.5 GiB asset bundle in `/System/Library/AssetsV2`. Simlock never deletes it; `doctor` and `component remove` report it. Reclaim in Xcode → Settings → Platforms. - **Slim costs (#87, #172).** Every reclaim erases, so the next boot re-slims: two boots each diff --git a/src/ports/AGENTS.md b/src/ports/AGENTS.md index 0158bbc35..be4678885 100644 --- a/src/ports/AGENTS.md +++ b/src/ports/AGENTS.md @@ -29,7 +29,7 @@ Uses: nothing from other components. Imported by: every component that touches t | `HostLoadProbe` | `host-load.ts` | `DarwinHostLoadProbe`, `LinuxHostLoadProbe` | `FakeHostLoadProbe` | CPU, memory, disk counters | | `TcpProbe` | `tcp-probe.ts` | `NodeTcpProbe` | `FakeTcpProbe` | Is a loopback port listening; send bytes | | `TcpStreamConnector` | `tcp-stream.ts` | `NodeTcpStreamConnector` | `FakeTcpStreamConnector` | A long-lived loopback connection, bytes both ways | -| `BytePipeLauncher` | `byte-pipe.ts` | `NodeBytePipeLauncher` | `FakeBytePipeLauncher` | A child with binary stdout and stdin as its liveness | +| `BytePipeLauncher` | `byte-pipe.ts` | `NodeBytePipeLauncher` | `FakeBytePipeLauncher` | A child with binary stdout; stdin is its liveness, or the commands `writeStdin` sends it (the fake records `stdinWrites`) | | `KeySetFetcher` | `key-set-fetcher.ts` | `NodeKeySetFetcher` | `FakeKeySetFetcher` | One GET of a key-set URL: no redirects, body capped at 64 KiB | | `SqliteOpener` | `sqlite.ts` | `NodeSqliteOpener` | none; tests use the adapter on a temp file | Open a SQLite file (`:memory:` is not pre-created); `exec`, `prepare`, `close`. `sqlite.ts` also holds `SqliteSchemaOwner` (`name`, `version`, `migrations[i]` takes version `i` to `i + 1`), `passesQuickCheck` and `isLockError` | | `WebhookPoster` | `webhook-poster.ts` | `FetchWebhookPoster` | `FakeWebhookPoster` (`testing.ts`) | POST to an operator-named URL; a redirect is not followed | diff --git a/vendor/README.md b/vendor/README.md index dbf07e9b7..0b4d0f14a 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -2,7 +2,7 @@ | Directory | Upstream | License | Notice | | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------ | -| `screen-capture/` | `EvanBacon/serve-sim` — a capture and encode pipeline we patched, four files rewritten; plus one HEVC capability string from `jkcoxson/idevice` (MIT) | Apache-2.0 (our own files MIT, one MIT string from idevice) | [`screen-capture/NOTICE.md`](screen-capture/NOTICE.md) | +| `screen-capture/` | `EvanBacon/serve-sim` — a capture, encode and HID-input pipeline we patched, four files rewritten; plus one HEVC capability string from `jkcoxson/idevice` (MIT) | Apache-2.0 (our own files MIT, one MIT string from idevice) | [`screen-capture/NOTICE.md`](screen-capture/NOTICE.md) | | `scrcpy/` | `Genymobile/scrcpy` — the server artifact, byte-identical, pushed unmodified to a leased emulator by the Android driver, whose client is ours in `src/drivers/android/` | Apache-2.0, plus BSD-3-Clause and BSL-1.0 code in the bundled Kotlin stdlib | [`scrcpy/NOTICE.md`](scrcpy/NOTICE.md) | A first-level directory here is third-party or derived-from-third-party code, and diff --git a/vendor/screen-capture/NOTICE.md b/vendor/screen-capture/NOTICE.md index 611cc9c72..b879c0c5c 100644 --- a/vendor/screen-capture/NOTICE.md +++ b/vendor/screen-capture/NOTICE.md @@ -21,9 +21,9 @@ per file, below. Upstream ships no `NOTICE` file, so Apache §4(d) is vacuous here, and none of the files we took carries a per-file license header, so §4(c) has no per-file text to -retain. §4(b) asks every file we modified to say so prominently: each of the five -below that is not upstream's unchanged — `FrameCapture.swift` and the four we -rewrote — opens with a `MODIFIED FILE` header naming serve-sim, its license and +retain. §4(b) asks every file we modified to say so prominently: each of the six +below that is not upstream's unchanged — `FrameCapture.swift`, `HIDInjector.swift` +and the four we rewrote — opens with a `MODIFIED FILE` header naming serve-sim, its license and this notice. The license text in `LICENSES/` is upstream's root `LICENSE`, byte for byte; it is named for the project it covers rather than plain `LICENSE` because this directory also holds code of ours that the file does not license. @@ -51,10 +51,12 @@ per row. | `simlock-screen-capture/FramebufferSurfaceSelector.swift` | `Sources/SimNativeSupport/` | theirs, unchanged | | `simlock-screen-capture/DeviceDisplaySizes.swift` | `Sources/SimNativeSupport/` | theirs, unchanged | | `simlock-screen-capture/FrameCapture.swift` | `Sources/SimNative/` | theirs, **one patch** (deviation 1) | +| `simlock-screen-capture/HIDInjector.swift` | `Sources/SimNative/` | theirs, **modified by us** — Apache-2.0, `MODIFIED FILE` header; the Duo, Device Hub, pose, Home, app switcher and Lock paths are deleted (deviation 3) | | `SimlockCapture/H264Encoder.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | | `SimlockCapture/VideoEncoder.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | | `SimlockCapture/PixelBufferUtils.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | | `SimlockCapture/Xcode.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | +| `simlock-screen-capture/InputMode.swift` | — | ours, entirely: the `--input` mode that drives `HIDInjector` from stdin | | `SimlockCapture/Lifecycle.swift`, `SimlockCapture/Options.swift` | — | ours, entirely: the option parser, the envelope, the lifecycle, the scaler, the encode loop | | `simlock-screen-capture/main.swift`, `simlock-screen-capture/SimulatorSource.swift` | — | ours, entirely: the simulator program and its framebuffer as a `ScreenSource` | | `simlock-device-capture/*` (five files) | — | ours, entirely: the iPhone program — the version gate, the CoreDevice source, the RTP receiver, the decoder — rewritten from this repo's own PoC (ADR 0031) | @@ -63,7 +65,7 @@ per row. | `scripts/unframe.mjs` | — | ours, entirely: a hand-run diagnostic | | `Package.swift` | — | ours, entirely | -To diff any of the eight serve-sim files, use the command at the end with the +To diff any of the nine serve-sim files, use the command at the end with the upstream path from its row and the file's path under `Sources/`. Four files were rewritten rather than copied, and they are still serve-sim's files, @@ -94,8 +96,8 @@ itself), `Sources/SimNativeSupport/DeviceHubKeyboardRouting.swift` and The whole of serve-sim's NodeAPI binding and everything behind it: the node-swift module `sim-module.swift`, the HID injection and accessibility -bridges (`HIDInjector.swift`, `AccessibilityBridge.swift`, -`DeviceHubKeyboardBridge.swift`, `DuoHIDBridge.swift`), the capture orchestration +bridges (`AccessibilityBridge.swift`, `DeviceHubKeyboardBridge.swift`, +`DuoHIDBridge.swift`; `HIDInjector.swift` was taken in part, deviation 3), the capture orchestration we replaced with `main.swift` (`CaptureEngine.swift`), the Duo hardware renderer (`SimDuoRenderer/*`), the Objective-C camera, accessibility and HID helpers (`SimCameraInjector/*`, `SimCameraHelper/*`, `SimAXSettings/*`, `SimDuoHID/*`), @@ -104,7 +106,7 @@ from serve-sim: the framebuffer of a device nobody is looking at, encoded. ## Deviations -Two that change behaviour, both deliberate. Each is here because a later reader +Four that change behaviour, all deliberate. Each is here because a later reader must not "clean up" back to upstream's behaviour and silently break a simlock feature. The mechanical consequences of the package layout — the files not taken, the import that went away — are named where they show up, not counted here. @@ -129,8 +131,22 @@ needs node-swift. We do not embed a Node module: the program writes frames to stdout, so the daemon supervises a capture exactly as it supervises every other tool it runs, and `sim-module.swift` was never taken. +**3. `HIDInjector.swift` is serve-sim's file without the paths simlock does not +use or does elsewhere.** The Duo hinge and cover paths (`DuoHIDBridge`), the Device +Hub keyboard bridge (`DeviceHubKeyboardBridge`), the fold pose (`DevicePose`) and the +`home`, `app_switcher` and `lock` buttons are deleted. Home is the daemon's +`xcrun simctl --set launch com.apple.springboard` (`src/drivers/ios/input/`), +the way upstream does it on Xcode 26 and later ("Xcode 26+ silently drops the Indigo +HID home-button press"), so it has one owner; the app switcher and Lock are Indigo +button presses and are out of scope (#1024). Touch, scroll, key and the rest are +upstream's. The file's own `MODIFIED FILE` header is the authoritative wording. + +**4. `FrameCapture.nativeScreenSizes(for:)` is internal, not private.** The input +mode reads the device type's native pixel size from it, because +`getScreenSize` answers only once frames arrive and the input mode captures none. + One mechanical consequence of flattening two upstream targets into one: -`FrameCapture.swift` no longer says `import SimNativeSupport`, because there is +`FrameCapture.swift` and `HIDInjector.swift` no longer say `import SimNativeSupport`, because there is no second target to import. Nothing else differs. ## The maintenance risk, and the escape hatch @@ -166,7 +182,7 @@ sources: | class | `NSClassFromString("SimServiceContext")` | `PrivateHeaders/CoreSimulator/SimServiceContext.h` | | file schema | `Contents/Resources/capabilities.plist` inside a device type's bundle | **not declared there**, and undocumented by Apple | -That is twenty distinct private members: twelve selectors, six key-value-coding +That is, for the framebuffer path, twenty distinct private members: twelve selectors, six key-value-coding keys, one class name and one plist schema. `facebook/idb` is MIT-licensed, and sixteen of the twenty are declared in its `PrivateHeaders/CoreSimulator/` and `PrivateHeaders/CoreSimDeviceIO/`. @@ -182,6 +198,15 @@ curl -sL https://raw.githubusercontent.com/facebook/idb/main/PrivateHeaders/Core curl -sL https://raw.githubusercontent.com/facebook/idb/main/PrivateHeaders/CoreSimDeviceIO/SimScreen-Protocol.h ``` +The input mode (`HIDInjector.swift`) reaches a second surface the same way, as +serve-sim's code does and without a second source cross-referenced here: +`SimDeviceLegacyHIDClient` (`initWithDevice:error:`, +`sendWithMessage:freeWhenDone:completionQueue:completion:`), found with +`NSClassFromString` and `class_getMethodImplementation`, and the `IndigoHIDMessageFor…` +C functions (`MouseNSEvent`, `Button`, `HIDArbitrary`, `KeyboardArbitrary`, +`DigitalCrownEvent`), found with `dlsym`. The re-read when Apple moves one is the same +diff of this file against upstream. + What is _not_ in idb is the other half of the story, and it is where a break will land first: `portIdentifier`, the `bundlePath`/`bundleURL` pair on `SimDeviceType`, and the `capabilities.plist` schema. Those have no second @@ -201,11 +226,12 @@ curl -sL https://raw.githubusercontent.com/EvanBacon/serve-sim/6d009686ed08f7d8b | diff -u - Sources/simlock-screen-capture/FrameCapture.swift ``` -That prints four things and nothing else — the `MODIFIED FILE` header Apache §4(b) -asks for, the device-set patch, the deleted `import SimNativeSupport`, and the +That prints five things and nothing else — the `MODIFIED FILE` header Apache §4(b) +asks for, the device-set patch, the `private` that became internal on +`nativeScreenSizes`, the deleted `import SimNativeSupport`, and the `import SimlockCapture` that replaced it — which is the whole claim of this notice in one screenful. If that diff ever grows past those -four, either someone edited upstream's capture logic or Apple moved something +five, either someone edited upstream's capture logic or Apple moved something upstream, and both deserve to be noticed rather than committed. For any other file in the table, substitute its upstream path in both places — for the four rewritten ones it prints a whole-file rewrite, which is what their rows say: @@ -245,11 +271,12 @@ URL either returns that commit's bytes or fails. `scripts/unframe.mjs`, the whole of `Sources/SimlockCapture/` but for the four encoder files (`Lifecycle.swift` and `Options.swift`: the option parser, the envelope, the lifecycle, the scaler, the encode loop), both programs' -`main.swift` and `Sources/simlock-screen-capture/SimulatorSource.swift`, the +`main.swift`, `Sources/simlock-screen-capture/SimulatorSource.swift` and +`Sources/simlock-screen-capture/InputMode.swift`, the whole of `Sources/simlock-device-capture/` and `Sources/SimlockMediaBridge/`, and the three interfaces under `Stubs/`. Everything else is serve-sim's: three -files unchanged, `FrameCapture.swift` with the one patch above, and the four -rewritten files, which are serve-sim's files modified by us and are Apache-2.0 +files unchanged, `FrameCapture.swift` with the patches above, `HIDInjector.swift` +with its deletions, and the four rewritten files, which are serve-sim's files modified by us and are Apache-2.0 like the rest of serve-sim's code here. ### idevice (MIT): the HEVC capability string diff --git a/vendor/screen-capture/Package.swift b/vendor/screen-capture/Package.swift index 270cd64a7..f841d4c10 100644 --- a/vendor/screen-capture/Package.swift +++ b/vendor/screen-capture/Package.swift @@ -20,6 +20,9 @@ // need it, and the daemon can then supervise the capture the same way it supervises every // other tool it runs. // +// Its input mode (`simlock-screen-capture --input`) also carries serve-sim's `HIDInjector.swift`, +// without the Duo, Device Hub, pose, Home, app switcher and Lock paths (NOTICE.md, deviations 3 and 4). +// // The encoders are serve-sim's files rewritten here: `H264Encoder.swift`, `VideoEncoder.swift`, // `PixelBufferUtils.swift` and `Xcode.swift` now use public VideoToolbox, CoreVideo, CoreMedia and // ImageIO, and as modified files of serve-sim they stay under Apache-2.0 (see NOTICE.md). diff --git a/vendor/screen-capture/Sources/simlock-screen-capture/FrameCapture.swift b/vendor/screen-capture/Sources/simlock-screen-capture/FrameCapture.swift index 3776ea878..e94b44a61 100644 --- a/vendor/screen-capture/Sources/simlock-screen-capture/FrameCapture.swift +++ b/vendor/screen-capture/Sources/simlock-screen-capture/FrameCapture.swift @@ -1,7 +1,9 @@ // MODIFIED FILE — this file is copied from serve-sim (Copyright 2026 Evan Bacon, Apache-2.0; // see NOTICE.md and LICENSES/ in this package) and it is NOT verbatim upstream. simlock's change // is `deviceSetPath` and its use in `findSimDevice`, so that a simulator booted in a -// simlock-owned `simctl --set` root can be found at all; everything else here is upstream's. +// simlock-owned `simctl --set` root can be found at all, and `nativeScreenSizes` being internal +// instead of private, so the input mode can report the device's size; everything else here is +// upstream's. // `NOTICE.md` prints the one command that diffs this file against pristine upstream at the commit // we took it from. Read that notice before editing this file. import Foundation @@ -355,7 +357,7 @@ actor FrameCapture { /// Native pixel sizes for every integrated digitizer display on this device /// type. Foldables list more than one (iPhone Duo cover + inner). Falls back /// to `SimDeviceType.mainScreenSize` when the capabilities plist is missing. - private static func nativeScreenSizes(for device: NSObject) -> [FramebufferSurfaceSize] { + static func nativeScreenSizes(for device: NSObject) -> [FramebufferSurfaceSize] { if let fromCapabilities = sizesFromCapabilities(device), !fromCapabilities.isEmpty { return fromCapabilities } diff --git a/vendor/screen-capture/Sources/simlock-screen-capture/HIDInjector.swift b/vendor/screen-capture/Sources/simlock-screen-capture/HIDInjector.swift new file mode 100644 index 000000000..2a69ad8f5 --- /dev/null +++ b/vendor/screen-capture/Sources/simlock-screen-capture/HIDInjector.swift @@ -0,0 +1,613 @@ +// MODIFIED FILE — this file is copied from serve-sim (Copyright 2026 Evan Bacon, Apache-2.0; +// see NOTICE.md and LICENSES/ in this package) and it is NOT verbatim upstream. simlock's changes +// drop what simlock does not use or does elsewhere: the Duo hinge and cover paths, the Device Hub +// keyboard bridge, the fold pose, and the Home, app switcher and Lock buttons (Home is the daemon's +// `simctl launch` of SpringBoard, so it has one owner); everything else here is upstream's. +// `NOTICE.md` prints the one command that diffs this file against pristine upstream at the commit +// we took it from. Read that notice before editing this file. +import Foundation +import ObjectiveC +import Darwin +import CoreGraphics + +/// Per-event HID logging is gated behind `SERVE_SIM_DEBUG_HID`. These lines fire +/// on every touch/move/button/key/crown event — a single drag emits a dozen — +/// so by default they flood stdout (and anything mirroring it) for no benefit. +/// Failure diagnostics ("returned nil", "unavailable", "not found") stay on +/// `print` so real problems are always visible. +private let hidDebugEnabled = ProcessInfo.processInfo.environment["SERVE_SIM_DEBUG_HID"] != nil + +@inline(__always) +private func hidLog(_ message: @autoclosure () -> String) { + guard hidDebugEnabled else { return } + print(message()) + // stdout is fully buffered when redirected to a log file; flush so each + // debug line is observable as the event happens (the type e2e test tails + // the server log for these). + fflush(stdout) +} + +/// Injects touch, button, and orientation HID events into the iOS Simulator. +/// +/// Uses IndigoHIDMessageForMouseNSEvent to create touch messages and +/// IndigoHIDMessageForButton for hardware button presses, sent via +/// SimDeviceLegacyHIDClient. Orientation goes through a separate transport +/// (PurpleWorkspacePort / GSEvent mach messages), matching idb's approach. +/// +/// The real C signature for touch is: +/// IndigoHIDMessageForMouseNSEvent(CGPoint*, CGPoint*, IndigoHIDTarget, NSEventType, NSSize, IndigoHIDEdge) +/// On arm64: x0=CGPoint*, x1=CGPoint*/NULL, x2=target, x3=eventType, d0/d1=NSSize, x4=edge. +/// Apple's Simulator.app always passes NSSize(1.0, 1.0), making ratio = point / 1.0 = point. +/// The edge parameter (x4) controls whether iOS treats the touch as a system edge gesture +/// (e.g. bottom edge = swipe-to-home on Face ID devices). +actor HIDInjector { + let queue = DispatchSerialQueue(label: "hid-injector", qos: .userInteractive) + nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() } + + private var hidClient: NSObject? + private var sendSel: Selector? + private var simDevice: NSObject? + + // IndigoHIDMessageForMouseNSEvent(CGPoint*, CGPoint*, IndigoHIDTarget, NSEventType, NSSize, IndigoHIDEdge) + // arm64 ABI: pointer/int params → x0-x4, float params → d0-d1 (independent numbering). + // CGFloat params map to d0 (NSSize.width) and d1 (NSSize.height). + private typealias IndigoMouseFunc = @convention(c) ( + UnsafePointer, UnsafePointer?, UInt32, Int32, CGFloat, CGFloat, UInt32 + ) -> UnsafeMutableRawPointer? + private var mouseFunc: IndigoMouseFunc? + + // IndigoHIDMessageForButton(int eventSource, int direction, int target) -> IndigoMessage* + private typealias IndigoButtonFunc = @convention(c) (Int32, Int32, Int32) -> UnsafeMutableRawPointer? + private var buttonFunc: IndigoButtonFunc? + + // IndigoHIDMessageForHIDArbitrary(uint32 target, uint32 page, uint32 usage, uint32 direction) -> IndigoMessage* + // direction: 1 = down, 2 = up. Routes any (page, usage) HID pair — used for + // power / volume / action / side buttons whose codes ship in DeviceKit's + // chrome.json. Unlike IndigoHIDMessageForButton's home press, these are + // delivered to the digitizer target and are honored on Xcode 26. + private typealias IndigoHIDArbitraryFunc = @convention(c) (UInt32, UInt32, UInt32, UInt32) -> UnsafeMutableRawPointer? + private var hidArbitraryFunc: IndigoHIDArbitraryFunc? + + // IndigoHIDMessageForKeyboardArbitrary(uint32_t keyCode, uint32_t direction) -> IndigoMessage* + // direction: 1 = key down, 2 = key up + private typealias IndigoKeyboardFunc = @convention(c) (UInt32, UInt32) -> UnsafeMutableRawPointer? + private var keyboardFunc: IndigoKeyboardFunc? + + // IndigoHIDMessageForDigitalCrownEvent(double rotationalDelta) -> IndigoMessage* + private typealias IndigoDigitalCrownFunc = @convention(c) (Double) -> UnsafeMutableRawPointer? + private var digitalCrownFunc: IndigoDigitalCrownFunc? + + private var touchTarget: UInt32 = 0x32 + + // NOTE: scroll is NOT a native HID event on the simulator — see the "Scroll + // events" section below. Device Hub's trackpad-capture path requires private + // Apple HID entitlements an unprivileged helper can't have, and synthetic + // scroll events are ignored by iOS, so we scroll via a touch drag instead. + + func setup(deviceUDID: String) throws { + SimFrameworks.load() + guard let device = FrameCapture.findSimDevice(udid: deviceUDID) else { + throw NSError(domain: "HIDInjector", code: 1, + userInfo: [NSLocalizedDescriptionKey: "Device \(deviceUDID) not found"]) + } + self.simDevice = device + + guard let funcPtr = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "IndigoHIDMessageForMouseNSEvent") else { + throw NSError(domain: "HIDInjector", code: 5, + userInfo: [NSLocalizedDescriptionKey: "IndigoHIDMessageForMouseNSEvent not found"]) + } + self.mouseFunc = unsafeBitCast(funcPtr, to: IndigoMouseFunc.self) + + if let buttonPtr = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "IndigoHIDMessageForButton") { + self.buttonFunc = unsafeBitCast(buttonPtr, to: IndigoButtonFunc.self) + hidLog("[hid] IndigoHIDMessageForButton loaded") + } else { + print("[hid] Warning: IndigoHIDMessageForButton not found") + } + + if let arbPtr = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "IndigoHIDMessageForHIDArbitrary") { + self.hidArbitraryFunc = unsafeBitCast(arbPtr, to: IndigoHIDArbitraryFunc.self) + hidLog("[hid] IndigoHIDMessageForHIDArbitrary loaded") + } else { + print("[hid] Warning: IndigoHIDMessageForHIDArbitrary not found") + } + + if let keyboardPtr = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "IndigoHIDMessageForKeyboardArbitrary") { + self.keyboardFunc = unsafeBitCast(keyboardPtr, to: IndigoKeyboardFunc.self) + hidLog("[hid] IndigoHIDMessageForKeyboardArbitrary loaded") + } else { + print("[hid] Warning: IndigoHIDMessageForKeyboardArbitrary not found") + } + + if let crownPtr = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "IndigoHIDMessageForDigitalCrownEvent") { + self.digitalCrownFunc = unsafeBitCast(crownPtr, to: IndigoDigitalCrownFunc.self) + hidLog("[hid] IndigoHIDMessageForDigitalCrownEvent loaded") + } else { + print("[hid] Warning: IndigoHIDMessageForDigitalCrownEvent not found") + } + + + guard let hidClass = NSClassFromString("_TtC12SimulatorKit24SimDeviceLegacyHIDClient") else { + throw NSError(domain: "HIDInjector", code: 2, + userInfo: [NSLocalizedDescriptionKey: "SimDeviceLegacyHIDClient not found"]) + } + + let initSel = NSSelectorFromString("initWithDevice:error:") + typealias HIDInitFunc = @convention(c) (AnyObject, Selector, AnyObject, AutoreleasingUnsafeMutablePointer) -> AnyObject? + guard let initIMP = class_getMethodImplementation(hidClass, initSel) else { + throw NSError(domain: "HIDInjector", code: 3, + userInfo: [NSLocalizedDescriptionKey: "Cannot get init method"]) + } + let initFunc = unsafeBitCast(initIMP, to: HIDInitFunc.self) + + var error: NSError? + let client = initFunc(hidClass.alloc(), initSel, device, &error) + if let error { throw error } + guard let clientObj = client as? NSObject else { + throw NSError(domain: "HIDInjector", code: 4, + userInfo: [NSLocalizedDescriptionKey: "Failed to create HID client"]) + } + + self.hidClient = clientObj + self.sendSel = NSSelectorFromString("sendWithMessage:freeWhenDone:completionQueue:completion:") + hidLog("[hid] SimDeviceLegacyHIDClient created") + hidLog("[hid] IndigoHIDMessageForMouseNSEvent loaded (with edge gesture support)") + } + + // IndigoHIDEdge values (x4 param to IndigoHIDMessageForMouseNSEvent). + // These control system edge gesture recognition in the simulated iOS device. + // Determined by disassembling IndigoHIDMessageForMouseNSEvent and testing + // each value against a booted Face ID simulator. + static let edgeNone: UInt32 = 0 // No edge — regular touch + static let edgeBottom: UInt32 = 3 // Bottom edge — swipe-to-home on Face ID devices + static let edgeTop: UInt32 = 2 // Top edge (notification center) + static let edgeLeft: UInt32 = 1 // Left edge + static let edgeRight: UInt32 = 4 // Right edge + + /// Synchronously hand an already-built Indigo message to the guest, freeing it. + /// Must run on `inputQueue`. + private func rawSend(_ msg: UnsafeMutableRawPointer) { + guard let client = hidClient, let sendSel = sendSel else { free(msg); return } + typealias SendFunc = @convention(c) (AnyObject, Selector, UnsafeMutableRawPointer, ObjCBool, AnyObject?, AnyObject?) -> Void + guard let sendIMP = class_getMethodImplementation(object_getClass(client)!, sendSel) else { + free(msg) + return + } + unsafeBitCast(sendIMP, to: SendFunc.self)(client, sendSel, msg, ObjCBool(true), nil, nil) + } + + /// Build a single-finger touch message (normalized 0..1 coords). Pure — safe + /// to call off `inputQueue`. NSSize(1,1) makes ratio = point. + private func touchMessage(type: String, x: Double, y: Double, edge: UInt32) -> UnsafeMutableRawPointer? { + guard let mouseFunc = mouseFunc else { return nil } + let eventType: Int32 + switch type { + case "begin", "move": eventType = 1 // Down (C function rejects Dragged=6) + case "end": eventType = 2 // Up + default: return nil + } + var point = CGPoint(x: x, y: y) + return mouseFunc(&point, nil, touchTarget, eventType, 1.0, 1.0, edge) + } + + /// Synchronously build + send a single touch. For use inside gesture blocks + /// already running on `inputQueue`. + private func rawSendTouch(type: String, x: Double, y: Double, edge: UInt32 = 0) { + if let msg = touchMessage(type: type, x: x, y: y, edge: edge) { rawSend(msg) } + } + + func sendTouch(type: String, x: Double, y: Double, screenWidth: Int, screenHeight: Int, edge: UInt32 = 0) { + guard let msg = touchMessage(type: type, x: x, y: y, edge: edge) else { return } + hidLog("[hid] Sending \(type) at (\(String(format:"%.3f",x)),\(String(format:"%.3f",y)))\(edge > 0 ? " edge=\(edge)" : "")") + rawSend(msg) + } + + func sendMultiTouch(type: String, x1: Double, y1: Double, x2: Double, y2: Double, screenWidth: Int, screenHeight: Int) { + guard let mouseFunc = mouseFunc else { return } + + let eventType: Int32 + switch type { + case "begin", "move": eventType = 1 + case "end": eventType = 2 + default: return + } + + // Pass both CGPoints to create a 3-block multi-touch message. + var point1 = CGPoint(x: x1, y: y1) + var point2 = CGPoint(x: x2, y: y2) + guard let rawMsg = mouseFunc(&point1, &point2, touchTarget, eventType, 1.0, 1.0, 0) else { + print("[hid] IndigoHIDMessageForMouseNSEvent returned nil for multi-touch \(type)") + return + } + + hidLog("[hid] Multi-touch \(type) f1=(\(String(format:"%.3f",x1)),\(String(format:"%.3f",y1))) f2=(\(String(format:"%.3f",x2)),\(String(format:"%.3f",y2)))") + rawSend(rawMsg) + } + + // MARK: - Button events + + // idb eventSource constants (first arg to IndigoHIDMessageForButton) + private static let buttonSourceHome: Int32 = 0x0 + private static let buttonSourceLock: Int32 = 0x1 + private static let buttonSourceSideButton: Int32 = 0xbb8 + private static let buttonSourceSiri: Int32 = 0x400002 + // Software-keyboard toggle — the event source Simulator.app's ⌘K sends. + private static let buttonSourceSoftwareKeyboard: Int32 = 0x3f0 + + // idb direction constants (second arg) + private static let buttonDown: Int32 = 1 + private static let buttonUp: Int32 = 2 + + // idb target constant (third arg) + private static let buttonTargetHardware: Int32 = 0x33 + + // Target for arbitrary (page, usage) HID — the digitizer, matching the touch + // path that's honored on Xcode 26 (0x32). + private static let buttonHIDTarget: UInt32 = 0x32 + + /// Synchronously build + send a hardware-button message. Call only inside an + /// `inputQueue` block (button sequences below run there). + private func sendHIDButton(eventSource: Int32, direction: Int32) { + guard let buttonFunc = buttonFunc else { return } + guard let msg = buttonFunc(eventSource, direction, Self.buttonTargetHardware) else { + print("[hid] IndigoHIDMessageForButton returned nil") + return + } + rawSend(msg) + } + + // MARK: - Keyboard events + + /// Inject a USB HID keyboard key event (Usage Page 0x07). + /// - Parameters: + /// - type: "down" or "up" + /// - usage: HID usage code (e.g. 0x04 = 'A', 0x28 = Enter, 0xE1 = LeftShift) + func sendKey(type: String, usage: UInt32) { + let direction: UInt32 + switch type { + case "down": direction = 1 + case "up": direction = 2 + default: return + } + + guard let keyboardFunc = keyboardFunc else { + print("[hid] Keyboard injection unavailable") + return + } + + guard let msg = keyboardFunc(usage, direction) else { + print("[hid] IndigoHIDMessageForKeyboardArbitrary returned nil (usage=0x\(String(usage, radix: 16)))") + return + } + + hidLog("[hid] Key \(type) usage=0x\(String(usage, radix: 16))") + rawSend(msg) + } + + // MARK: - Digital Crown events + + /// Inject a Digital Crown rotation event. + /// - Parameter delta: Raw scroll delta, matching SimulatorKit's wheel-to-crown path. + func sendDigitalCrown(delta: Double) { + guard delta.isFinite, delta != 0 else { return } + guard let digitalCrownFunc else { + print("[hid] Digital Crown injection unavailable") + return + } + + guard let msg = digitalCrownFunc(delta) else { + print("[hid] IndigoHIDMessageForDigitalCrownEvent returned nil (delta=\(delta))") + return + } + + hidLog("[hid] Digital Crown delta=\(String(format:"%.4f", delta))") + rawSend(msg) + } + + // MARK: - Scroll events + // + // iOS treats the simulator display as a touchscreen — there is no hardware + // scroll wheel. Device Hub scrolls by capturing a real Mac trackpad and + // forwarding genuine HID scroll events through a privileged pointer service + // (`com.apple.private.hid.client.event-filter`); an unprivileged helper can't + // capture host HID or synthesize events iOS accepts (synthetic scroll to the + // pointer service 0x35 is silently dropped). See docs/scroll-injection-devicehub.md. + // + // So we scroll the way a finger does: translate the wheel delta into a touch + // drag on the digitizer (target 0x32) — the same path taps/swipes use, which + // is verified to scroll on iOS 27. A wheel burst becomes one continuous drag + // (begin → moves → end on idle), re-anchoring to center when it nears an edge + // so long scrolls aren't capped by the screen bounds. + + // Fraction of the display a finger travels per pixel of wheel delta. Wheel + // deltas are large (~120/notch); 1.0 maps a notch to a full-screen drag, which + // matches the feel of a wheel "page". Tunable. + private static let scrollDragGain: Double = 1.0 + private static let scrollEdgeMargin: Double = 0.08 // re-anchor inside this margin + private static let scrollGestureIdle: TimeInterval = 0.1 + + private var scrollDragActive = false + private var scrollFingerX = 0.5 + private var scrollFingerY = 0.5 + private var scrollAnchorX = 0.5 // where the gesture (re)starts — under the cursor + private var scrollAnchorY = 0.5 + private var scrollEndWork: DispatchWorkItem? + + private func clampFinger(_ v: Double) -> Double { + min(max(v, HIDInjector.scrollEdgeMargin), 1 - HIDInjector.scrollEdgeMargin) + } + + /// Touch down to (re)start the scroll drag, then let iOS register the + /// touch-down before the finger moves. Runs on `inputQueue`. + private func beginDrag(x: Double, y: Double) { + rawSendTouch(type: "begin", x: x, y: y) + usleep(8000) + } + + /// Inject a scroll-wheel / trackpad pan as a touch drag on the digitizer. + /// - Parameters: + /// - dx: Horizontal scroll delta in device pixels (positive = content right). + /// - dy: Vertical scroll delta in device pixels (positive = content down). + /// - anchorX/anchorY: Normalized (0–1) cursor position to begin the drag + /// under, so iOS pans the view beneath the pointer. Nil = screen center. + func sendScroll(dx: Double, dy: Double, anchorX: Double?, anchorY: Double?, screenWidth: Int, screenHeight: Int) async { + guard dx.isFinite, dy.isFinite, (dx != 0 || dy != 0), screenWidth > 0, screenHeight > 0 else { return } + + // Finger moves opposite to content: scrolling content down = swipe up. + let stepX = -(dx / Double(screenWidth)) * HIDInjector.scrollDragGain + let stepY = -(dy / Double(screenHeight)) * HIDInjector.scrollDragGain + let aX = clampFinger(anchorX.flatMap { $0.isFinite ? $0 : nil } ?? 0.5) + let aY = clampFinger(anchorY.flatMap { $0.isFinite ? $0 : nil } ?? 0.5) + + if !scrollDragActive { + // Anchor a fresh gesture under the cursor so iOS hit-tests the + // right scroll view (e.g. a bottom sheet vs. the map behind it). + scrollAnchorX = aX + scrollAnchorY = aY + scrollFingerX = aX + scrollFingerY = aY + beginDrag(x: scrollFingerX, y: scrollFingerY) + scrollDragActive = true + } + + var nextX = scrollFingerX + stepX + var nextY = scrollFingerY + stepY + + // Near an edge: lift, re-anchor back under the cursor, and continue. + // Re-beginning at the anchor keeps the gesture hit-testing the same view. + if nextX <= HIDInjector.scrollEdgeMargin || nextX >= 1 - HIDInjector.scrollEdgeMargin || + nextY <= HIDInjector.scrollEdgeMargin || nextY >= 1 - HIDInjector.scrollEdgeMargin { + rawSendTouch(type: "end", x: scrollFingerX, y: scrollFingerY) + scrollFingerX = scrollAnchorX + scrollFingerY = scrollAnchorY + beginDrag(x: scrollFingerX, y: scrollFingerY) + nextX = scrollFingerX + stepX + nextY = scrollFingerY + stepY + } + + scrollFingerX = clampFinger(nextX) + scrollFingerY = clampFinger(nextY) + rawSendTouch(type: "move", x: scrollFingerX, y: scrollFingerY) + + // End the drag shortly after the wheel goes idle. + scrollEndWork?.cancel() + let work = DispatchWorkItem { [self] in + guard scrollDragActive else { return } + rawSendTouch(type: "end", x: scrollFingerX, y: scrollFingerY) + scrollDragActive = false + } + scrollEndWork = work + + try? await Task.sleep(for: .seconds(HIDInjector.scrollGestureIdle)) + work.perform() + } + + /// Press an arbitrary hardware button identified by its HID (page, usage), + /// as shipped in DeviceKit chrome.json (`usagePage`/`usage`). Covers power, + /// volume up/down, the action button, and the watch side button. + /// - phase: "down" / "up" hold the button for natural long-presses (power + /// off slider, side-button menus); "press" sends a momentary down+up. + func sendButtonHID(page: UInt32, usage: UInt32, phase: String) async { + guard let arb = hidArbitraryFunc else { + print("[hid] Arbitrary HID injection unavailable (page=\(page) usage=\(usage))") + return + } + let target = Self.buttonHIDTarget + func emit(_ direction: UInt32) { + guard let msg = arb(target, page, usage, direction) else { + print("[hid] IndigoHIDMessageForHIDArbitrary returned nil (page=\(page) usage=\(usage) dir=\(direction))") + return + } + rawSend(msg) + } + hidLog("[hid] HID button page=\(page) usage=\(usage) phase=\(phase)") + switch phase { + case "down": emit(1) + case "up": emit(2) + default: + emit(1) + try? await Task.sleep(for: .seconds(0.05)) + emit(2) + } + } + + func sendButton(button: String) async { + hidLog("[hid] Sending button: \(button)") + + switch button { + case "swipe_home": + sendSwipeHome() + + case "siri": + // Holding Siri for ~300ms matches Simulator.app's "hold side button + // to invoke Siri" gesture; a tap is ignored. + sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonDown) + try? await Task.sleep(for: .seconds(0.3)) + sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonUp) + + case "side_button": + sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonDown) + sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonUp) + + default: + print("[hid] Unknown button: \(button)") + } + } + + // MARK: - SimDevice private control + + /// Toggle a CoreAnimation render debug flag on the simulator. Names are the + /// strings Simulator.app's Debug menu passes to `-[SimDevice + /// setCADebugOption:enabled:]` (CoreSimulator private category): + /// debug_color_blended / debug_color_copies / debug_color_misaligned + /// debug_color_offscreen / debug_slow_animations + func setCADebugOption(name: String, enabled: Bool) -> Bool { + guard let device = simDevice else { + fputs("[sim] setCADebugOption: no SimDevice\n", stderr) + return false + } + let sel = NSSelectorFromString("setCADebugOption:enabled:") + guard device.responds(to: sel) else { + fputs("[sim] setCADebugOption: selector not available on SimDevice\n", stderr) + return false + } + typealias Fn = @convention(c) (AnyObject, Selector, NSString, ObjCBool) -> ObjCBool + let imp = device.method(for: sel) + let fn = unsafeBitCast(imp, to: Fn.self) + let result = fn(device, sel, name as NSString, ObjCBool(enabled)) + hidLog("[sim] setCADebugOption(\(name), \(enabled)) → \(result.boolValue)") + return result.boolValue + } + + /// Toggle the on-screen software keyboard, exactly like Simulator.app's + /// I/O → Keyboard → Toggle Software Keyboard (⌘K): a momentary Indigo HID + /// button press (event source 0x3f0) sent through the legacy HID client. + /// Instant, and leaves the hardware-keyboard state untouched. + func toggleSoftwareKeyboard() { + guard buttonFunc != nil else { + print("[hid] Software keyboard toggle unavailable (IndigoHIDMessageForButton not loaded)") + return + } + sendHIDButton(eventSource: Self.buttonSourceSoftwareKeyboard, direction: Self.buttonDown) + sendHIDButton(eventSource: Self.buttonSourceSoftwareKeyboard, direction: Self.buttonUp) + } + + /// Ask CoreSimulator to broadcast a memory warning to the simulated OS. + /// Equivalent to Debug → Simulate Memory Warning and idb's + /// FBSimulatorMemoryCommands.simulateMemoryWarning. + func simulateMemoryWarning() { + guard let device = simDevice else { + fputs("[sim] simulateMemoryWarning: no SimDevice\n", stderr) + return + } + let sel = NSSelectorFromString("simulateMemoryWarning") + guard device.responds(to: sel) else { + fputs("[sim] simulateMemoryWarning: selector not available on SimDevice\n", stderr) + return + } + _ = device.perform(sel) + hidLog("[sim] simulateMemoryWarning dispatched") + } + + /// Synthesize a swipe-up-from-bottom gesture (Face ID "go home" gesture). + /// Uses IndigoHIDEdge.bottom to flag touches as system edge gestures, + /// which iOS interprets as the home indicator swipe. + private func sendSwipeHome() { + let xPos = 0.5 + let yStart = 0.95 + let yEnd = 0.35 + let steps = 10 + let stepDelay: TimeInterval = 0.016 // ~16ms per step + let edge = Self.edgeBottom + + // Touch down at bottom edge + rawSendTouch(type: "begin", x: xPos, y: yStart, edge: edge) + Thread.sleep(forTimeInterval: stepDelay) + + // Interpolated moves upward + for i in 1...steps { + let t = Double(i) / Double(steps) + let y = yStart + (yEnd - yStart) * t + rawSendTouch(type: "move", x: xPos, y: y, edge: edge) + Thread.sleep(forTimeInterval: stepDelay) + } + + // Touch up + rawSendTouch(type: "end", x: xPos, y: yEnd, edge: edge) + } + + // MARK: - Orientation (GSEvent via PurpleWorkspacePort) + + // UIDeviceOrientation values accepted by the guest's GraphicsServices. + static let orientationPortrait: UInt32 = 1 + static let orientationPortraitUpsideDown: UInt32 = 2 + static let orientationLandscapeRight: UInt32 = 3 + static let orientationLandscapeLeft: UInt32 = 4 + + // GSEvent wire-format constants. Reverse-engineered by idb from + // Simulator.app's ARM64 disassembly; see idb's SimulatorApp/GSEvent.h. + private static let gsEventTypeDeviceOrientationChanged: UInt32 = 50 + private static let gsEventHostFlag: UInt32 = 0x20000 + private static let gsEventMachMessageID: mach_msg_id_t = 0x7B + + /// Send a device-orientation GSEvent to the simulator. + /// + /// GSEvent messages travel a different path from Indigo HID: they go + /// through `mach_msg_send` → `PurpleWorkspacePort` → + /// `GraphicsServices._PurpleEventCallback` → backboardd. This is how + /// Simulator.app itself rotates the device, and how idb's + /// `FBSimulatorPurpleHID.orientationEvent:` is delivered. + func sendOrientation(orientation: UInt32) -> Bool { + guard let device = simDevice else { + fputs("[hid] sendOrientation: no SimDevice (setup not called?)\n", stderr) + return false + } + + let lookupSel = NSSelectorFromString("lookup:error:") + typealias LookupFunc = @convention(c) ( + AnyObject, Selector, NSString, AutoreleasingUnsafeMutablePointer + ) -> mach_port_t + guard let lookupIMP = class_getMethodImplementation(object_getClass(device)!, lookupSel) else { + fputs("[hid] sendOrientation: -[SimDevice lookup:error:] not found\n", stderr) + return false + } + let lookup = unsafeBitCast(lookupIMP, to: LookupFunc.self) + + var lookupError: NSError? + let purplePort = lookup(device, lookupSel, "PurpleWorkspacePort" as NSString, &lookupError) + if purplePort == 0 { + fputs("[hid] sendOrientation: PurpleWorkspacePort not found (\(lookupError?.localizedDescription ?? "no error")). Simulator.app must be running.\n", stderr) + return false + } + + // 112-byte aligned buffer (>= 108 = align4(4 + 0x6B), the msgh_size + // for a GSEvent with a 4-byte orientation payload). + var buf = [UInt8](repeating: 0, count: 112) + return buf.withUnsafeMutableBufferPointer { ptr in + let base = UnsafeMutableRawPointer(ptr.baseAddress!) + let header = base.assumingMemoryBound(to: mach_msg_header_t.self) + header.pointee.msgh_bits = mach_msg_bits_t(MACH_MSG_TYPE_COPY_SEND) + header.pointee.msgh_size = 108 + header.pointee.msgh_remote_port = purplePort + header.pointee.msgh_local_port = mach_port_t(MACH_PORT_NULL) + header.pointee.msgh_voucher_port = mach_port_t(MACH_PORT_NULL) + header.pointee.msgh_id = Self.gsEventMachMessageID + + // GSEvent type at offset 0x18 — record_info_size at 0x48 — payload at 0x4C. + base.storeBytes( + of: Self.gsEventTypeDeviceOrientationChanged | Self.gsEventHostFlag, + toByteOffset: 0x18, as: UInt32.self) + base.storeBytes(of: UInt32(4), toByteOffset: 0x48, as: UInt32.self) + base.storeBytes(of: orientation, toByteOffset: 0x4C, as: UInt32.self) + + let kr = mach_msg_send(header) + if kr != KERN_SUCCESS { + fputs("[hid] sendOrientation: mach_msg_send failed (\(kr))\n", stderr) + return false + } else { + hidLog("[hid] Orientation set to \(orientation)") + return true + } + } + } +} diff --git a/vendor/screen-capture/Sources/simlock-screen-capture/InputMode.swift b/vendor/screen-capture/Sources/simlock-screen-capture/InputMode.swift new file mode 100644 index 000000000..f9b6a8d54 --- /dev/null +++ b/vendor/screen-capture/Sources/simlock-screen-capture/InputMode.swift @@ -0,0 +1,91 @@ +// simlock's own, MIT-licensed (see NOTICE.md): the input mode of `simlock-screen-capture`. +// +// `simlock-screen-capture --input` writes no picture. It sets up serve-sim's HID injector for +// one simulator, tells the daemon the device's native size once, then injects what the daemon sends on +// stdin. Each message is the stream's envelope: a 4-byte big-endian length (covering the tag), a tag +// byte, and serve-sim's JSON for that tag: +// +// 0x03 touch {"type":"begin"|"move"|"end","x":0..1,"y":0..1} +// 0x06 key {"type":"down"|"up","usage":} +// 0x0b scroll {"dx":-1..1,"dy":-1..1,"x"?:0..1,"y"?:0..1} (fractions of the screen) +// +// stdout carries one envelope, tag 0x82, `{"width":,"height":}`, once the HID client exists. +// End of stdin ends the program, as it ends a capture. Home is not here: the daemon relaunches +// SpringBoard with `simctl`, which is what serve-sim does on Xcode 26 and later. +import Foundation +import SimlockCapture + +private let tagTouch: UInt8 = 0x03 +private let tagKey: UInt8 = 0x06 +private let tagScroll: UInt8 = 0x0b +private let tagConfig: UInt8 = 0x82 + +/// Reads exactly `count` bytes from stdin, or nil when stdin has ended. +private func readExactly(_ count: Int) -> Data? { + var data = Data() + while data.count < count { + guard let chunk = try? FileHandle.standardInput.read(upToCount: count - data.count), + !chunk.isEmpty else { return nil } + data.append(chunk) + } + return data +} + +private func emitConfig(width: Int, height: Int) { + let body = (try? JSONSerialization.data(withJSONObject: ["width": width, "height": height])) ?? Data() + let total = UInt32(body.count + 1) + var out = Data() + for shift in [24, 16, 8, 0] { out.append(UInt8((total >> UInt32(shift)) & 0xff)) } + out.append(tagConfig) + out.append(body) + out.withUnsafeBytes { pointer in + var written = 0 + while written < out.count { + let n = write(Wire.fd, pointer.baseAddress! + written, out.count - written) + if n <= 0 { return } + written += n + } + } +} + +func runInput(udid: String) async -> Never { + let injector = HIDInjector() + do { + try await injector.setup(deviceUDID: udid) + } catch { + fail("could not set up input for \(udid): \(error.localizedDescription)") + } + guard let device = FrameCapture.findSimDevice(udid: udid), + let size = FrameCapture.nativeScreenSizes(for: device).first else { + fail("could not read the screen size of \(udid)") + } + emitConfig(width: size.width, height: size.height) + + while let header = readExactly(4) { + let length = header.reduce(0) { ($0 << 8) | Int($1) } + guard length >= 1, let message = readExactly(length) else { break } + let tag = message[message.startIndex] + guard let fields = try? JSONSerialization.jsonObject(with: message.dropFirst()) as? [String: Any] + else { continue } + switch tag { + case tagTouch: + guard let type = fields["type"] as? String, + let x = fields["x"] as? Double, let y = fields["y"] as? Double else { continue } + await injector.sendTouch(type: type, x: x, y: y, screenWidth: size.width, screenHeight: size.height) + case tagKey: + guard let type = fields["type"] as? String, let usage = fields["usage"] as? Int, usage >= 0 else { continue } + await injector.sendKey(type: type, usage: UInt32(usage)) + case tagScroll: + guard let dx = fields["dx"] as? Double, let dy = fields["dy"] as? Double else { continue } + // The injector takes pixel deltas; the daemon sends fractions of the screen. + await injector.sendScroll( + dx: dx * Double(size.width), dy: dy * Double(size.height), + anchorX: fields["x"] as? Double, anchorY: fields["y"] as? Double, + screenWidth: size.width, screenHeight: size.height) + default: + continue + } + } + log("stdin closed") + exit(0) +} diff --git a/vendor/screen-capture/Sources/simlock-screen-capture/main.swift b/vendor/screen-capture/Sources/simlock-screen-capture/main.swift index 22cf47cef..7fabd73c5 100644 --- a/vendor/screen-capture/Sources/simlock-screen-capture/main.swift +++ b/vendor/screen-capture/Sources/simlock-screen-capture/main.swift @@ -3,6 +3,10 @@ // Usage: // simlock-screen-capture [--codec avcc|mjpeg] [--fps N] [--bitrate BPS] // [--max-dimension PX] [--duration SEC] +// simlock-screen-capture --input +// +// `--input` writes no picture: it injects the touches, scrolls and keys the daemon sends on stdin +// into the simulator and says its native size once on stdout (see InputMode.swift). // // Environment: // SIMLOCK_IOS_DEVICE_SET the `simctl --set` root the device lives in (see FrameCapture) @@ -39,8 +43,13 @@ import SimlockCapture // build: FrameCapture logs with `print` while it starts. Wire.claimStdout() -guard let options = parseOptions(Array(CommandLine.arguments.dropFirst())) else { - log("usage: simlock-screen-capture [--codec avcc|mjpeg] [--fps N] [--bitrate BPS] [--max-dimension PX] [--duration SEC]") +let arguments = Array(CommandLine.arguments.dropFirst()) +if arguments.count == 2, arguments[1] == "--input" { + await runInput(udid: arguments[0]) +} + +guard let options = parseOptions(arguments) else { + log("usage: simlock-screen-capture [--codec avcc|mjpeg] [--fps N] [--bitrate BPS] [--max-dimension PX] [--duration SEC] | simlock-screen-capture --input") exit(64) } diff --git a/website/src/content/docs/docs/using-devices/watching-the-screen.mdx b/website/src/content/docs/docs/using-devices/watching-the-screen.mdx index e36b109cf..447d198af 100644 --- a/website/src/content/docs/docs/using-devices/watching-the-screen.mdx +++ b/website/src/content/docs/docs/using-devices/watching-the-screen.mdx @@ -110,12 +110,14 @@ The body is Simlock's own framing of H.264 frames, made for the console's player A lease's holder can open a WebSocket for its lease at `GET /v1/leases//input` (with an `Upgrade: websocket` request), on a worker or on a gateway, with its agent token in an `Authorization: Bearer` header. An operator token can open any lease's. Turn it on with `capabilities.input.enabled` (off by default; see [Configuration reference](/simlock/docs/reference/configuration)); through a gateway, the gateway and the lease's worker both need it on. -The messages are expo serve-sim's: each is one tag byte and UTF-8 JSON, so a client that already drives serve-sim (T3 Code's device panel) needs nothing Simlock-specific. The server sends `0x83` (input is accepted from now on), then `0x82` with the screen as it is now, `{"width":1179,"height":2556,"orientation":"portrait"}`, and again whenever the size changes. A device that takes no input gets `0x82` `{"width":1,"height":1,"orientation":"portrait","inputUnavailable":true}`, and the socket stays open. An Android emulator and an enrolled Android phone take input (see below); an iOS device does not yet, so its sockets get that answer. +The messages are expo serve-sim's: each is one tag byte and UTF-8 JSON, so a client that already drives serve-sim (T3 Code's device panel) needs nothing Simlock-specific. The server sends `0x83` (input is accepted from now on), then `0x82` with the screen as it is now, `{"width":1179,"height":2556,"orientation":"portrait"}`, and again whenever the size changes. A device that takes no input gets `0x82` `{"width":1,"height":1,"orientation":"portrait","inputUnavailable":true}`, and the socket stays open. An Android emulator, an enrolled Android phone and an iOS simulator take input (see below); a physical iPhone does not, so its sockets get that answer. The client sends `0x03` touch (`{"type":"begin","x":0.4,"y":0.7}`, `type` is `begin`, `move` or `end`, positions are fractions from 0 to 1), `0x04` button (`home`, `app_switcher`, `lock` or `back`), `0x06` key (`{"type":"down","usage":4}`, a USB HID usage from 0 to 255) and `0x0b` scroll. A message that is not valid JSON, has an unknown tag, a position outside 0 to 1, or is over 64 KiB is dropped and the socket stays open. `0x07` (orientation) is always dropped. `0x10` is answered `0x90` `{"requestId":7,"ok":false}`. On Android, a tap, swipe, scroll, key or button reaches the device through a second, control-only scrcpy server that Simlock starts for the lease; it shows no picture and changes no device setting. Taps and scrolls land on the screen as it is now, in its current rotation, and Simlock reads the size every 2 seconds, so after an agent rotates the device a tap may use the old size for up to 2 seconds, and the socket then gets a new `0x82`. A scroll sends 1 wheel unit for each 1/16 of the screen, up to 16, at the given position or the screen's centre; a positive `dy` moves the content down and a positive `dx` moves it right. `home`, `app_switcher`, `lock` and `back` are Android's Home, Recents, Power and Back keys. A `0x06` key is a USB HID usage typed as a US keyboard: letters, digits, Enter, Escape, Backspace, Tab, Space, the punctuation row, the arrows and the modifiers (Shift, Ctrl, Alt, Meta) work, and a modifier stays held for the keys after it until its `up`; other usages (function keys, the keypad) are dropped. A dark phone screen is not woken, so a tap on it does nothing. A phone whose adb check is closed gets `inputUnavailable`. +On an iOS simulator, a tap, swipe, scroll and `home` work, and so do `0x06` keys on iOS 26 and older runtimes. A tap reaches the simulator through a helper process that Simlock starts for the lease; it shows no picture. `home` brings the home screen forward and sends the app to the background. A scroll is sent as a finger drag. `app_switcher`, `lock` and `back` are dropped, and rotation is not supported. On an iOS 27 or newer runtime the simulator ignores injected keys, so `0x06` keys are dropped and the `0x82` message carries `"keyboardUnavailable":true`. The `0x82` message always says `"orientation":"portrait"`. + Up to 8 sockets share one lease's device input. The socket closes with: | Code | Reason | When | From 1dbf0d061109ca60187c6db3b5c4d5dc1495a771 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:33:18 +0100 Subject: [PATCH 18/24] refactor(input): the iOS input is built beside the driver, not in its constructor (#1019) --- src/drivers/ios/index.ts | 32 +++++++++++++++++++------------- src/drivers/ios/input/index.ts | 2 +- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 52b8f3c1a..0b4db273b 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -355,19 +355,7 @@ export class IosSimctlDriver implements Driver { ? {} : { onLog: options.screenCapture.onLog }), }); - // why: the input is offered exactly when the screen capture is, because both are the one program. - this.input = - options.screenCapture === undefined - ? undefined - : createIosDeviceInput({ - baseEnvironment: options.screenCapture.baseEnvironment, - clock: options.clock, - command: configuredScreenCaptureCommand(options) ?? options.screenCapture.command, - ...(rootOptions === undefined ? {} : { deviceSet: deviceRoot }), - ...(options.logger === undefined ? {} : { logger: options.logger }), - launcher: options.screenCapture.launcher, - processRunner: options.processRunner, - }); + this.input = createInputFor(options, deviceRoot, rootOptions !== undefined); } readonly screenCapture: ScreenCapture | undefined; @@ -1798,6 +1786,24 @@ class IosRuntimeUnavailableForDownloadError extends RuntimeMissingError { } } +// why: the input is offered exactly when the screen capture is, because both are the one program. +function createInputFor( + options: IosSimctlDriverOptions, + deviceRoot: string, + ownedRoot: boolean, +): DeviceInputFactory | undefined { + if (options.screenCapture === undefined) return undefined; + return createIosDeviceInput({ + baseEnvironment: options.screenCapture.baseEnvironment, + clock: options.clock, + command: configuredScreenCaptureCommand(options) ?? options.screenCapture.command, + ...(ownedRoot ? { deviceSet: deviceRoot } : {}), + ...(options.logger === undefined ? {} : { logger: options.logger }), + launcher: options.screenCapture.launcher, + processRunner: options.processRunner, + }); +} + function configuredScreenCaptureCommand(options: IosSimctlDriverOptions): string | undefined { const configured = options.driverConfig["screenCaptureCommand"]; return typeof configured === "string" && configured !== "" ? configured : undefined; diff --git a/src/drivers/ios/input/index.ts b/src/drivers/ios/input/index.ts index d0ebe8063..6952f13c4 100644 --- a/src/drivers/ios/input/index.ts +++ b/src/drivers/ios/input/index.ts @@ -1 +1 @@ -export { createIosDeviceInput, type IosDeviceInputOptions } from "./device-input.js"; +export { createIosDeviceInput } from "./device-input.js"; From a259c8a6a4e4619d8bdc25a12a0de0d3ddd892f3 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:33:51 +0100 Subject: [PATCH 19/24] chore: drop scratch files from the branch (#1019) --- .agents/scratch/green7.patch | 23 ----------------------- .agents/scratch/pr-1019.md | 16 ---------------- 2 files changed, 39 deletions(-) delete mode 100644 .agents/scratch/green7.patch delete mode 100644 .agents/scratch/pr-1019.md diff --git a/.agents/scratch/green7.patch b/.agents/scratch/green7.patch deleted file mode 100644 index 8dc8cddb5..000000000 --- a/.agents/scratch/green7.patch +++ /dev/null @@ -1,23 +0,0 @@ -diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts -index 774cde56e..7a6d2ff47 100644 ---- a/src/drivers/ios/input/device-input.ts -+++ b/src/drivers/ios/input/device-input.ts -@@ -3,6 +3,7 @@ import { - type DeviceInputFactory, - encodeScreenFrame, - INPUT_TAG, -+ isPhysicalDriverDevice, - type InputMessage, - type InputScreenConfig, - type PhysicalDriverDevice, -@@ -40,6 +41,10 @@ const SPRINGBOARD = "com.apple.springboard"; - export function createIosDeviceInput(options: IosDeviceInputOptions): DeviceInputFactory { - return { - open: async (device: VirtualDriverDevice | PhysicalDriverDevice) => { -+ // why: nothing on a physical iPhone injects input (F23); only a simulator has a HID helper. -+ if (isPhysicalDriverDevice(device)) { -+ return { unavailable: "Input is not supported on physical iPhones" }; -+ } - const child = options.launcher.start(options.command, [device.deviceId, "--input"], { - env: environmentFor(options), - }); diff --git a/.agents/scratch/pr-1019.md b/.agents/scratch/pr-1019.md deleted file mode 100644 index 8d7970924..000000000 --- a/.agents/scratch/pr-1019.md +++ /dev/null @@ -1,16 +0,0 @@ -Closes #1019 - -### Status - -Build in progress: cycle 1 (touches reach the helper's stdin). - -- [ ] The two Tests lines and the `byte-pipe` test pass in `pnpm test`, and `src/source-hygiene.test.ts` passes. -- [ ] `pnpm build:capture` builds `simlock-screen-capture` with the input mode where Xcode is selected. -- [ ] On a simulator, `e2e/slow-agent-device.test.ts` opens a lease and the Appduct fixture through agent-device, opens `WS /v1/leases/{id}/input`, gets 0x83 and 0x82, taps the centre of `tab-status` as agent-device's snapshot places it, and the next snapshot shows the Status tab's screen (slow lane, iOS simulator). -- [ ] `watching-the-screen.mdx` lists what input does on iOS: touch, scroll, Home, keys on iOS 26 runtimes; not the app switcher, Lock, rotation or keys on iOS 27 (#1024). - -## Assumptions - -- Assumption: where the iOS device input lives — `src/drivers/ios/input/` (an `index.ts` and `device-input.ts`, as Android's `input/`) instead of the spec's single `input.ts`, so its test goes through an `index.ts` (testing rule 6). - -_Written by an agent._ From e2082f3f112b50b5882bcac12db766bc77f4d1ba Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:39:34 +0100 Subject: [PATCH 20/24] chore(input): the iOS input has a manual, and its tests pin the env, the 10 s limit and the unset-position scroll (#1019) --- docs/internal/ARCHITECTURE.md | 2 +- src/drivers/ios/input/AGENTS.md | 21 +++++++++++++++++++++ src/drivers/ios/input/CLAUDE.md | 1 + src/drivers/ios/input/device-input.test.ts | 14 +++++++++++--- src/drivers/ios/input/device-input.ts | 8 ++++---- 5 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 src/drivers/ios/input/AGENTS.md create mode 100644 src/drivers/ios/input/CLAUDE.md diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index 53c0d5046..d33c70e77 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -43,7 +43,7 @@ worker daemon ── uplink (WebSocket to /v1/uplink, worker dials) ──┘ | core/warm-pool | What stays booted, and warm targets | [warm-pool](../../src/core/warm-pool/AGENTS.md) | | core/capacity | Limits, RAM budget, reservations | [capacity](../../src/core/capacity/AGENTS.md) | | core/usage | Usage figures from the history rows, read on a thread | [usage](../../src/core/usage/AGENTS.md) | -| drivers | The `Driver` port's two implementations | [drivers](../../src/drivers/AGENTS.md) · [android](../../src/drivers/android/AGENTS.md) · [android screen](../../src/drivers/android/screen/AGENTS.md) · [android input](../../src/drivers/android/input/AGENTS.md) · [ios](../../src/drivers/ios/AGENTS.md) | +| drivers | The `Driver` port's two implementations | [drivers](../../src/drivers/AGENTS.md) · [android](../../src/drivers/android/AGENTS.md) · [android screen](../../src/drivers/android/screen/AGENTS.md) · [android input](../../src/drivers/android/input/AGENTS.md) · [ios](../../src/drivers/ios/AGENTS.md) · [ios input](../../src/drivers/ios/input/AGENTS.md) | | gateway | Fleet queue, routing, forwarding to workers, relay | [gateway](../../src/gateway/AGENTS.md) · [routing](../../src/gateway/routing/AGENTS.md) | | alerts | Gateway only: chosen fleet events POSTed to webhook destinations | [alerts](../../src/alerts/AGENTS.md) | | link | Byte channels over the uplink | [link](../../src/link/AGENTS.md) | diff --git a/src/drivers/ios/input/AGENTS.md b/src/drivers/ios/input/AGENTS.md new file mode 100644 index 000000000..c9f70a627 --- /dev/null +++ b/src/drivers/ios/input/AGENTS.md @@ -0,0 +1,21 @@ +# iOS device input + +Does: taps, scrolls, Home and keys on an iOS simulator, through `simlock-screen-capture --input`; a physical iPhone answers unavailable. +Entry: `index.ts` → `createIosDeviceInput(options)`, built by `IosSimctlDriver` as its `input` whenever it has a `screenCapture`. It is the implementation of core's `DeviceInputFactory` (`core/input.ts`). +Uses: core (`DeviceInput`, `InputMessage`, the screen envelope), ports (`BytePipeLauncher`, `Clock`, `Logger`, `ProcessRunner`), `../runtime-version.ts`, `../screen-capture.ts` (`stopProgram`). Parent: [iOS driver](../AGENTS.md). Consumer: [input](../../../input/AGENTS.md). + +## Files + +| File | Role | +| ----------------- | ------------------------------------------- | +| `index.ts` | `createIosDeviceInput`, entered only here | +| `device-input.ts` | One input: open, send, Home, failure, close | + +## How it works + +The wire, the 10 s ready limit, Home, the dropped buttons and keys, and close are described under "Input" in the [iOS driver manual](../AGENTS.md). + +## Rules + +- The helper's HID client and the runtime rule for keys stay here; core and the input module name neither. +- Home is `xcrun simctl` here and nowhere in the helper (one owner). diff --git a/src/drivers/ios/input/CLAUDE.md b/src/drivers/ios/input/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/src/drivers/ios/input/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/src/drivers/ios/input/device-input.test.ts b/src/drivers/ios/input/device-input.test.ts index 23b04ecb1..9d39c14ec 100644 --- a/src/drivers/ios/input/device-input.test.ts +++ b/src/drivers/ios/input/device-input.test.ts @@ -28,7 +28,7 @@ function simulator(runtimeId?: string): VirtualDriverDevice { }; } -function setup() { +function setup(inOwnSet = true) { const launcher = new FakeBytePipeLauncher(); const clock = new FakeClock(); const processRunner = new ScriptedProcessRunner([ @@ -41,10 +41,10 @@ function setup() { }, ]); const factory = createIosDeviceInput({ - baseEnvironment: { PATH: "/usr/bin" }, + baseEnvironment: { PATH: "/usr/bin", SIMLOCK_IOS_DEVICE_SET: "/daemon/own-set" }, clock, command: COMMAND, - deviceSet: DEVICE_SET, + ...(inOwnSet ? { deviceSet: DEVICE_SET } : {}), launcher, processRunner, }); @@ -116,6 +116,12 @@ describe("iOS device input", () => { ]); expect(isUnavailableInput(input)).toBe(false); + const inXcodeSet = setup(false); + const xcodeOpen = inXcodeSet.factory.open(simulator(), new AbortController().signal); + expect(inXcodeSet.launcher.starts.map((start) => start.env)).toEqual([{ PATH: "/usr/bin" }]); + inXcodeSet.launcher.last.write(envelope(INPUT_TAG.config, { height: 2556, width: 1179 })); + await xcodeOpen; + for (const runtimeId of [ "com.apple.CoreSimulator.SimRuntime.iOS-27-0", "com.apple.CoreSimulator.SimRuntime.iOS-28-1", @@ -166,6 +172,8 @@ describe("iOS device input", () => { await new Promise((resolve) => setImmediate(resolve)); expect(settled).toBe(false); silent.clock.advance(100); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(true); expect(await silentOpen).toBe("The input helper reported no screen size within 10 s"); expect(silent.launcher.last.stdinClosed).toBe(true); diff --git a/src/drivers/ios/input/device-input.ts b/src/drivers/ios/input/device-input.ts index de8e45c89..4e0deca5c 100644 --- a/src/drivers/ios/input/device-input.ts +++ b/src/drivers/ios/input/device-input.ts @@ -122,8 +122,8 @@ function within(milliseconds: number, clock: Clock, work: Promise): Promis }); } -// why: the helper reads serve-sim's JSON for each tag; a scroll's position is left out when the -// client sent none, so the helper scrolls under the centre. +// why: the helper reads serve-sim's JSON for each tag; JSON leaves out a scroll position the client +// sent none of, so the helper scrolls under the centre. function frameOfMessage(message: Exclude): Uint8Array { switch (message.kind) { case "touch": @@ -132,8 +132,8 @@ function frameOfMessage(message: Exclude): Uin return frameOf(INPUT_TAG.scroll, { dx: message.dx, dy: message.dy, - ...(message.x === undefined ? {} : { x: message.x }), - ...(message.y === undefined ? {} : { y: message.y }), + x: message.x, + y: message.y, }); case "key": return frameOf(INPUT_TAG.key, { type: message.type, usage: message.usage }); From 2ffb9aedb295f545abdb291b28ffc2691842bbd9 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 26 Oct 2026 22:51:41 +0100 Subject: [PATCH 21/24] fix(input): scroll messages overlap on the injector so a wheel burst stays one drag (#1019) --- .../Sources/simlock-screen-capture/InputMode.swift | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/vendor/screen-capture/Sources/simlock-screen-capture/InputMode.swift b/vendor/screen-capture/Sources/simlock-screen-capture/InputMode.swift index f9b6a8d54..7710ac0e0 100644 --- a/vendor/screen-capture/Sources/simlock-screen-capture/InputMode.swift +++ b/vendor/screen-capture/Sources/simlock-screen-capture/InputMode.swift @@ -78,10 +78,15 @@ func runInput(udid: String) async -> Never { case tagScroll: guard let dx = fields["dx"] as? Double, let dy = fields["dy"] as? Double else { continue } // The injector takes pixel deltas; the daemon sends fractions of the screen. - await injector.sendScroll( - dx: dx * Double(size.width), dy: dy * Double(size.height), - anchorX: fields["x"] as? Double, anchorY: fields["y"] as? Double, - screenWidth: size.width, screenHeight: size.height) + // why: sendScroll sleeps to end the drag after the wheel idles; awaiting it would + // make every message its own drag and hold stdin, so calls overlap on the actor. + let anchorX = fields["x"] as? Double, anchorY = fields["y"] as? Double + Task { + await injector.sendScroll( + dx: dx * Double(size.width), dy: dy * Double(size.height), + anchorX: anchorX, anchorY: anchorY, + screenWidth: size.width, screenHeight: size.height) + } default: continue } From 61436c5eb2286885d2e43e0bc6d4a25b9efb43a5 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 27 Oct 2026 21:31:32 +0100 Subject: [PATCH 22/24] docs(adr): ADR 0049 is Accepted (#1019) --- ...0049-a-lease-holder-watches-and-drives-its-devices-screen.md | 2 +- docs/internal/adr/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/internal/adr/0049-a-lease-holder-watches-and-drives-its-devices-screen.md b/docs/internal/adr/0049-a-lease-holder-watches-and-drives-its-devices-screen.md index 0a4baadb8..a3a948c00 100644 --- a/docs/internal/adr/0049-a-lease-holder-watches-and-drives-its-devices-screen.md +++ b/docs/internal/adr/0049-a-lease-holder-watches-and-drives-its-devices-screen.md @@ -1,6 +1,6 @@ # 0049. A lease holder watches and drives its device's screen -- **Status:** Accepted — not yet implemented +- **Status:** Accepted - **Date:** 2026-10-26 - **Issue:** [#1013](https://github.com/callstackincubator/simlock/issues/1013) - **Narrows:** [ADR 0045](0045-a-devices-screen-streams-to-operators-not-a-leases.md) diff --git a/docs/internal/adr/README.md b/docs/internal/adr/README.md index 51f6a88ed..8e35efbd4 100644 --- a/docs/internal/adr/README.md +++ b/docs/internal/adr/README.md @@ -87,4 +87,4 @@ the status is stale. | [0046](0046-a-standalone-worker-serves-agent-device-by-default-with-a-private-profile.md) | A standalone worker serves agent-device by default and hands local agents a private profile | Accepted — not yet implemented; narrows 0023 §1, §3–4, 0030 §2, 0039 and 0043 §4 | | [0047](0047-device-exec-fills-in-the-leases-device.md) | `device.exec` fills in the lease's device, and refuses another | Accepted — not yet implemented; extends 0005 §19a; narrows 0003 §11 for one MCP tool | | [0048](0048-an-ios-worker-may-keep-its-simulators-in-xcodes-device-set.md) | An iOS worker may keep its simulators in Xcode's device set | Accepted; narrows 0001 §1, §2, §3, §5, §6 and §7 for iOS under `deviceSet: "default"` | -| [0049](0049-a-lease-holder-watches-and-drives-its-devices-screen.md) | A lease holder watches and drives its device's screen | Accepted — not yet implemented | +| [0049](0049-a-lease-holder-watches-and-drives-its-devices-screen.md) | A lease holder watches and drives its device's screen | Accepted | From e760afc912f1fa7f08a095b65ed4b2e272e44443 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 27 Oct 2026 21:33:52 +0100 Subject: [PATCH 23/24] docs(adr): ADRs 0030 and 0033 are Accepted (#1019) --- ...modules-share-one-contract-and-load-only-when-turned-on.md | 2 +- ...proxy-is-a-second-way-to-sign-in-and-auth-is-one-module.md | 2 +- docs/internal/adr/README.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/internal/adr/0030-optional-modules-share-one-contract-and-load-only-when-turned-on.md b/docs/internal/adr/0030-optional-modules-share-one-contract-and-load-only-when-turned-on.md index 06bf8691f..fa2f6a143 100644 --- a/docs/internal/adr/0030-optional-modules-share-one-contract-and-load-only-when-turned-on.md +++ b/docs/internal/adr/0030-optional-modules-share-one-contract-and-load-only-when-turned-on.md @@ -1,6 +1,6 @@ # 0030. Optional modules share one contract and load only when turned on -- **Status:** Accepted — not yet implemented; §2 narrowed by [ADR 0046](0046-a-standalone-worker-serves-agent-device-by-default-with-a-private-profile.md); §3 narrowed by [ADR 0049](0049-a-lease-holder-watches-and-drives-its-devices-screen.md) +- **Status:** Accepted; §2 narrowed by [ADR 0046](0046-a-standalone-worker-serves-agent-device-by-default-with-a-private-profile.md); §3 narrowed by [ADR 0049](0049-a-lease-holder-watches-and-drives-its-devices-screen.md) - **Date:** 2026-10-15 - **Issue:** [#591](https://github.com/callstackincubator/simlock/issues/591) - **Supersedes:** nothing. Narrows [ADR diff --git a/docs/internal/adr/0033-a-sign-in-proxy-is-a-second-way-to-sign-in-and-auth-is-one-module.md b/docs/internal/adr/0033-a-sign-in-proxy-is-a-second-way-to-sign-in-and-auth-is-one-module.md index 208b4d332..4a8a8951d 100644 --- a/docs/internal/adr/0033-a-sign-in-proxy-is-a-second-way-to-sign-in-and-auth-is-one-module.md +++ b/docs/internal/adr/0033-a-sign-in-proxy-is-a-second-way-to-sign-in-and-auth-is-one-module.md @@ -1,6 +1,6 @@ # 0033. A sign-in proxy is a second way to sign in, and auth is one module -- **Status:** Accepted — not yet implemented +- **Status:** Accepted - **Date:** 2026-10-19 - **Issue:** [#701](https://github.com/callstackincubator/simlock/issues/701) - **Supersedes:** nothing. Amends [ADR diff --git a/docs/internal/adr/README.md b/docs/internal/adr/README.md index 8e35efbd4..083690b1e 100644 --- a/docs/internal/adr/README.md +++ b/docs/internal/adr/README.md @@ -68,10 +68,10 @@ the status is stale. | [0027](0027-dependency-cruiser-checks-every-component-boundary.md) | dependency-cruiser checks every component's boundary and the module graph | Accepted | | [0028](0028-the-uplink-carries-byte-channels-and-small-messages-go-first.md) | The uplink carries byte channels beside its messages, and small messages go first | Accepted; §4 narrowed by 0049 | | [0029](0029-host-load-is-an-optional-module-whose-readings-ride-on-status.md) | Host load is an optional module whose readings ride on status | Accepted | -| [0030](0030-optional-modules-share-one-contract-and-load-only-when-turned-on.md) | Optional modules share one contract and load only when turned on | Accepted — not yet implemented; §2 narrowed by 0046; §3 narrowed by 0049 | +| [0030](0030-optional-modules-share-one-contract-and-load-only-when-turned-on.md) | Optional modules share one contract and load only when turned on | Accepted; §2 narrowed by 0046; §3 narrowed by 0049 | | [0031](0031-a-physical-iphone-streams-through-coredevice-behind-a-screen-source-seam.md) | A physical iPhone streams through CoreDevice's media stream, behind a screen-source seam | Accepted | | [0032](0032-the-event-history-is-a-sqlite-database-with-projections-beside-it.md) | The event history is a SQLite database, with projections beside it | Accepted — not yet implemented; §1 narrowed by 0036 | -| [0033](0033-a-sign-in-proxy-is-a-second-way-to-sign-in-and-auth-is-one-module.md) | A sign-in proxy is a second way to sign in, and auth is one module | Accepted — not yet implemented | +| [0033](0033-a-sign-in-proxy-is-a-second-way-to-sign-in-and-auth-is-one-module.md) | A sign-in proxy is a second way to sign in, and auth is one module | Accepted | | [0034](0034-one-worker-per-user-and-the-folder-says-what-a-daemon-is.md) | One worker per user, and the folder says what a daemon is | Accepted — not yet implemented | | [0035](0035-a-gateway-sends-chosen-fleet-events-to-webhook-destinations.md) | A gateway sends chosen fleet events to webhook destinations | Accepted — not yet implemented | | [0036](0036-daemon-state-is-a-sqlite-database-whose-tables-each-owner-keeps.md) | Daemon state is a SQLite database, and each owner keeps its own tables | Accepted — not yet implemented | From 631eb643580d6d147ae001537bb65bf8ee3306f9 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 27 Oct 2026 21:59:25 +0100 Subject: [PATCH 24/24] docs(ios): tighten the driver manual to 150 lines (#1019) --- src/drivers/ios/AGENTS.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index 36d0b0bef..54e2311a1 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -95,14 +95,9 @@ Programs: dist/bin from `pnpm build`, or `drivers.ios.screenCaptureCommand` and program enforces its pin: a CoreDevice major other than 642 exits 69 with the reason, a 503 for the viewer. Asked to stop while still starting, a program stops and goes inside its 2 s. -Input — `input` (offered with `screenCapture`, same launcher): `simlock-screen-capture --input` -with the same `SIMLOCK_IOS_DEVICE_SET` rule. stdout: one `0x82` `{width,height}` (the device type's native -size) within 10 s of spawn, else the open rejects (also for a missing program or an early exit; the helper is -stopped). stdin: envelopes `[len][tag]{json}` for touch `0x03`, key `0x06` and scroll `0x0b`; a failed write or a later -exit calls `onFailed`. Home runs `xcrun simctl [--set ] launch com.apple.springboard` (10 s; a failure is -logged); the app switcher, Lock and Back are dropped. The config is always `portrait`, with `keyboardUnavailable: true` -when the runtime is iOS 27 or later or unreadable, and keys are then dropped. Close: stdin, 2 s, SIGKILL. A physical iPhone answers -`{unavailable}` and starts nothing. +Input — `input` (offered with `screenCapture`, same launcher): `simlock-screen-capture --input` with the same `SIMLOCK_IOS_DEVICE_SET` rule. stdout: one `0x82` `{width,height}` (the device type's native size) within 10 s of spawn, else the open rejects (also for a missing program or an early exit; the helper is stopped). +stdin: envelopes `[len][tag]{json}` for touch `0x03`, key `0x06` and scroll `0x0b`; a failed write or a later exit calls `onFailed`. Home runs `xcrun simctl [--set ] launch com.apple.springboard` (10 s; a failure is logged); the app switcher, Lock and Back are dropped. +The config is always `portrait`, with `keyboardUnavailable: true` when the runtime is iOS 27 or later or unreadable, and keys are then dropped. Close: stdin, 2 s, SIGKILL. A physical iPhone answers `{unavailable}` and starts nothing. Advisories — `runtime-cache-unreclaimable` (asset bundles no installed runtime mounts); `slim-runtime-unsupported` (installed runtimes below 18.5, only when `ios.defaultMode` is `slim`).