From b03b9143cf8d69740179237d4aaae02eca4a68ae Mon Sep 17 00:00:00 2001 From: Seto Elkahfi <1797197+setoelkahfi@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:44:57 +0200 Subject: [PATCH 1/3] Add tvOS platform support tvOS simulators now surface through the MCP server. - Accept `tvos` in the platform unions and validation in the mobilecli wrapper. - Drop the hard `ios` filter on simulator discovery so tvOS simulators are returned, and report each device's real platform to analytics. - List the tvOS Siri Remote buttons (UP, DOWN, LEFT, RIGHT, SELECT, MENU, PLAY_PAUSE) in the press-button tool description. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/mobilecli.ts | 6 +++--- src/server.ts | 12 +++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/mobilecli.ts b/src/mobilecli.ts index 74edf84..040fda4 100644 --- a/src/mobilecli.ts +++ b/src/mobilecli.ts @@ -30,7 +30,7 @@ export interface MobilecliAgentStatusResponse { export interface MobilecliDevicesOptions { includeOffline?: boolean; - platform?: "ios" | "android"; + platform?: "ios" | "android" | "tvos"; type?: "real" | "emulator" | "simulator"; } @@ -186,8 +186,8 @@ export class Mobilecli { } if (options.platform) { - if (options.platform !== "ios" && options.platform !== "android") { - throw new Error(`Invalid platform: ${options.platform}. Must be "ios" or "android"`); + if (options.platform !== "ios" && options.platform !== "android" && options.platform !== "tvos") { + throw new Error(`Invalid platform: ${options.platform}. Must be "ios", "android" or "tvos"`); } args.push("--platform", options.platform); diff --git a/src/server.ts b/src/server.ts index 178de1e..49e4648 100644 --- a/src/server.ts +++ b/src/server.ts @@ -22,7 +22,7 @@ const ALLOWED_RECORDING_EXTENSIONS = [".mp4"]; interface MobilecliDevice { id: string; name: string; - platform: "android" | "ios"; + platform: "android" | "ios" | "tvos"; type: "real" | "emulator" | "simulator"; version: string; state: "online" | "offline"; @@ -188,7 +188,6 @@ export const createMcpServer = (): McpServer => { // Check if it's a simulator (will later replace all other device types as well) const response = mobilecli.getDevices({ - platform: "ios", type: "simulator", includeOffline: false, }); @@ -205,7 +204,7 @@ export const createMcpServer = (): McpServer => { agentVerifiedSimulators.add(deviceId); } - posthog("get_robot", { "DevicePlatform": "ios", "DeviceType": "simulator" }).then(); + posthog("get_robot", { "DevicePlatform": device.platform, "DeviceType": "simulator" }).then(); return new MobileDevice(deviceId); } } @@ -262,10 +261,9 @@ export const createMcpServer = (): McpServer => { // If go-ios is not available, silently skip } - // Get iOS simulators from mobilecli, including offline ones so we can - // report how many are installed vs booted. only booted ones are returned. + // Get iOS and tvOS simulators from mobilecli, including offline ones so + // telemetry can report how many are installed versus booted. const response = mobilecli.getDevices({ - platform: "ios", type: "simulator", includeOffline: true, }); @@ -531,7 +529,7 @@ export const createMcpServer = (): McpServer => { "Press a button on device", { device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."), - button: z.string().describe("The button to press. Supported buttons: BACK (android only), HOME, VOLUME_UP, VOLUME_DOWN, ENTER, DPAD_CENTER (android tv only), DPAD_UP (android tv only), DPAD_DOWN (android tv only), DPAD_LEFT (android tv only), DPAD_RIGHT (android tv only)"), + button: z.string().describe("The button to press. Supported buttons: BACK (android only), HOME, VOLUME_UP, VOLUME_DOWN, ENTER, DPAD_CENTER (android tv only), DPAD_UP (android tv only), DPAD_DOWN (android tv only), DPAD_LEFT (android tv only), DPAD_RIGHT (android tv only), UP (tvOS only), DOWN (tvOS only), LEFT (tvOS only), RIGHT (tvOS only), SELECT (tvOS only), MENU (tvOS only), PLAY_PAUSE (tvOS only)"), }, { destructiveHint: true }, async ({ device, button }) => { From 5c339d78efd5b2db07dba0bea977cd384d697484 Mon Sep 17 00:00:00 2001 From: Seto Elkahfi <1797197+setoelkahfi@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:52:58 +0200 Subject: [PATCH 2/3] Surface and drive real Apple TV devices Detect physical Apple TV units by product type and report platform tvos in mobile_list_available_devices instead of hardcoding ios. Route real tvOS devices through mobilecli (which installs and drives the tvOS runner) while iPhones and iPads keep using the WebDriverAgent-based IosRobot. Add a unit test for platform detection. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ios.ts | 13 ++++++++++++- src/server.ts | 24 +++++++++++++++++++++--- test/ios-platform.ts | 20 ++++++++++++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 test/ios-platform.ts diff --git a/src/ios.ts b/src/ios.ts index 382b6d9..733893b 100644 --- a/src/ios.ts +++ b/src/ios.ts @@ -31,6 +31,16 @@ export interface IosDevice { deviceName: string; } +export type IosPlatform = "ios" | "tvos"; + +/** + * Real Apple TV units are enumerated over the same go-ios connection as iPhones and + * iPads, so they are distinguished by their product type (e.g. "AppleTV14,1") rather + * than a separate device class. + */ +export const platformFromProductType = (productType: string): IosPlatform => + productType.startsWith("AppleTV") ? "tvos" : "ios"; + const getGoIosPath = (): string => { if (process.env.GO_IOS_PATH) { return process.env.GO_IOS_PATH; @@ -282,7 +292,7 @@ export class IosManager { return devices; } - public listDevicesWithDetails(): Array { + public listDevicesWithDetails(): Array { if (!this.isGoIosInstalled()) { console.error("go-ios is not installed, no physical iOS devices can be detected"); return []; @@ -296,6 +306,7 @@ export class IosManager { deviceId: device, deviceName: info.DeviceName, version: info.ProductVersion, + platform: platformFromProductType(info.ProductType), }; }); diff --git a/src/server.ts b/src/server.ts index 49e4648..a34a3b1 100644 --- a/src/server.ts +++ b/src/server.ts @@ -9,7 +9,7 @@ import { ChildProcess } from "node:child_process"; import { error, trace } from "./logger"; import { AndroidRobot, AndroidDeviceManager } from "./android"; import { ActionableError, Robot } from "./robot"; -import { IosManager, IosRobot } from "./ios"; +import { IosManager, IosRobot, platformFromProductType } from "./ios"; import { PNG } from "./png"; import { isScalingAvailable, Image } from "./image-utils"; import { Mobilecli } from "./mobilecli"; @@ -173,6 +173,24 @@ export const createMcpServer = (): McpServer => { const iosDevices = iosManager.listDevices(); const iosDevice = iosDevices.find(d => d.deviceId === deviceId); if (iosDevice) { + // Real Apple TV units are driven through mobilecli, which installs and drives + // the tvOS runner (re-signed with a provisioning profile). iPhones and iPads + // keep using the WebDriverAgent-based IosRobot. + const platform = platformFromProductType(iosManager.getDeviceInfo(deviceId).ProductType); + if (platform === "tvos") { + if (!agentVerifiedSimulators.has(deviceId)) { + const agentStatus = mobilecli.agentStatus(deviceId); + if (agentStatus.status === "fail") { + mobilecli.agentInstall(deviceId); + } + + agentVerifiedSimulators.add(deviceId); + } + + posthog("get_robot", { "DevicePlatform": "tvos", "DeviceType": "real" }).then(); + return new MobileDevice(deviceId); + } + posthog("get_robot", { "DevicePlatform": "ios", "DeviceType": "real" }).then(); return new IosRobot(deviceId); } @@ -242,7 +260,7 @@ export const createMcpServer = (): McpServer => { }); } - // Get iOS physical devices with details + // Get iOS and tvOS physical devices with details telemetry.IosRealCount = 0; try { const iosDevices = iosManager.listDevicesWithDetails(); @@ -251,7 +269,7 @@ export const createMcpServer = (): McpServer => { devices.push({ id: device.deviceId, name: device.deviceName, - platform: "ios", + platform: device.platform, type: "real", version: device.version, state: "online", diff --git a/test/ios-platform.ts b/test/ios-platform.ts new file mode 100644 index 0000000..60ca794 --- /dev/null +++ b/test/ios-platform.ts @@ -0,0 +1,20 @@ +import { test, expect } from "@playwright/test"; + +import { platformFromProductType } from "../src/ios"; + +test.describe("ios platform detection", () => { + + test("maps Apple TV product types to tvos", () => { + expect(platformFromProductType("AppleTV14,1")).toBe("tvos"); + expect(platformFromProductType("AppleTV11,1")).toBe("tvos"); + }); + + test("maps iPhone and iPad product types to ios", () => { + expect(platformFromProductType("iPhone15,3")).toBe("ios"); + expect(platformFromProductType("iPad13,1")).toBe("ios"); + }); + + test("defaults unknown product types to ios", () => { + expect(platformFromProductType("")).toBe("ios"); + }); +}); From 7e7c0f2d1164e881712f20b584c22ad518b47fad Mon Sep 17 00:00:00 2001 From: Seto Elkahfi <1797197+setoelkahfi@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:39:15 +0200 Subject: [PATCH 3/3] feat(tvos): add TvosRobot for real Apple TV automation Add a dedicated TvosRobot (mobilecli/DeviceKit-backed) for real Apple TV, keeping IosRobot/WebDriverAgent unchanged for iPhone/iPad (D2). Route real Apple TVs (platform tvos) to TvosRobot via getRobotFromDevice; iOS real devices continue to IosRobot. - TvosRobot: Siri Remote buttons (UP/DOWN/LEFT/RIGHT/SELECT/MENU/ PLAY_PAUSE); app/screenshot/view-tree/screen-size delegated to mobilecli; explicit unsupported-on-tvOS ActionableErrors for tap/doubleTap/longPress/swipe/orientation/text; best-effort openUrl - New mobile_focus_by_identifier MCP tool + Mobilecli.focusByIdentifier shelling to mobilecli io focus (accessibility-identity focus) - IosManager.listDevices retains platform so routing can distinguish Apple TV; extracted and tested robotForIosDevice helper - Device-free unit tests for routing, button mapping, unsupported ops, and focus Implements: SPECS/quality/real-tvos-device-support/SPEC.md (Milestone 5, 6) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ios.ts | 265 +++++++++++++++++++++++++++++++---------- src/mobilecli.ts | 26 ++++ src/server.ts | 58 ++++++--- src/tvos.ts | 221 ++++++++++++++++++++++++++++++++++ test/ios-coredevice.ts | 58 +++++++++ test/tvos.ts | 156 ++++++++++++++++++++++++ 6 files changed, 707 insertions(+), 77 deletions(-) create mode 100644 src/tvos.ts create mode 100644 test/ios-coredevice.ts create mode 100644 test/tvos.ts diff --git a/src/ios.ts b/src/ios.ts index 733893b..f6460dc 100644 --- a/src/ios.ts +++ b/src/ios.ts @@ -26,12 +26,63 @@ interface InfoCommandOutput { TimeZone: string; } +interface CoreDeviceListOutput { + result?: { + devices?: CoreDeviceEntry[]; + }; +} + +interface CoreDeviceDetailsOutput { + result?: { + connectionProperties?: { + transportType?: string; + }; + deviceProperties?: { + name?: string; + osVersionNumber?: string; + }; + hardwareProperties?: { + productType?: string; + udid?: string; + }; + }; +} + +interface CoreDeviceAppsOutput { + result?: { + apps?: Array<{ + bundleIdentifier?: string; + name?: string; + version?: string; + }>; + }; +} + +interface CoreDeviceEntry { + connectionProperties?: { + pairingState?: string; + transportType?: string; + tunnelState?: string; + }; + deviceProperties?: { + bootState?: string; + name?: string; + osVersionNumber?: string; + }; + hardwareProperties?: { + productType?: string; + reality?: string; + udid?: string; + }; +} + export interface IosDevice { deviceId: string; deviceName: string; } export type IosPlatform = "ios" | "tvos"; +export type IosDeviceWithDetails = IosDevice & { version: string; platform: IosPlatform }; /** * Real Apple TV units are enumerated over the same go-ios connection as iPhones and @@ -41,6 +92,34 @@ export type IosPlatform = "ios" | "tvos"; export const platformFromProductType = (productType: string): IosPlatform => productType.startsWith("AppleTV") ? "tvos" : "ios"; +export const coreDeviceDevicesFromJson = (output: string): IosDeviceWithDetails[] => { + const json = JSON.parse(output) as CoreDeviceListOutput; + const entries = json.result?.devices ?? []; + + return entries + .filter(entry => entry.hardwareProperties?.reality === "physical") + .filter(entry => entry.connectionProperties?.pairingState === "paired") + .filter(entry => { + const bootState = entry.deviceProperties?.bootState ?? ""; + const tunnelState = entry.connectionProperties?.tunnelState ?? ""; + const transportType = entry.connectionProperties?.transportType ?? ""; + return bootState === "booted" || tunnelState === "connected" || transportType === "usb" || transportType === "localNetwork"; + }) + .map(entry => { + const deviceId = entry.hardwareProperties?.udid?.trim() ?? ""; + const deviceName = entry.deviceProperties?.name?.trim() ?? ""; + const version = entry.deviceProperties?.osVersionNumber?.trim() ?? ""; + const productType = entry.hardwareProperties?.productType?.trim() ?? ""; + return { + deviceId, + deviceName, + version, + platform: platformFromProductType(productType), + }; + }) + .filter(device => device.deviceId.length > 0); +}; + const getGoIosPath = (): string => { if (process.env.GO_IOS_PATH) { return process.env.GO_IOS_PATH; @@ -106,10 +185,34 @@ export class IosRobot implements Robot { return execFileSync(getGoIosPath(), ["--udid", this.deviceId, ...args], {}).toString(); } + private devicectl(...args: string[]): string { + return execFileSync("xcrun", ["devicectl", ...args, "--json-output", "-"], { + stdio: ["pipe", "pipe", "ignore"], + }).toString(); + } + + private getCoreDeviceInfo(): InfoCommandOutput { + const output = this.devicectl("device", "info", "details", "--device", this.deviceId); + const json = JSON.parse(output) as CoreDeviceDetailsOutput; + return { + DeviceClass: "iPhone", + DeviceName: json.result?.deviceProperties?.name ?? this.deviceId, + ProductName: json.result?.hardwareProperties?.productType ?? "", + ProductType: json.result?.hardwareProperties?.productType ?? "", + ProductVersion: json.result?.deviceProperties?.osVersionNumber ?? "", + PhoneNumber: "", + TimeZone: "", + }; + } + public async getIosVersion(): Promise { - const output = await this.ios("info"); - const json = JSON.parse(output); - return json.ProductVersion; + try { + const output = await this.ios("info"); + const json = JSON.parse(output); + return json.ProductVersion; + } catch { + return this.getCoreDeviceInfo().ProductVersion; + } } private async isTunnelRequired(): Promise { @@ -134,32 +237,47 @@ export class IosRobot implements Robot { } public async listApps(): Promise { - await this.assertTunnelRunning(); - - const output = await this.ios("apps", "--all", "--list"); - return output - .split("\n") - .map(line => { - const [packageName, appName] = line.split(" "); - return { - packageName, - appName, - }; - }); + try { + await this.assertTunnelRunning(); + + const output = await this.ios("apps", "--all", "--list"); + return output + .split("\n") + .map(line => { + const [packageName, appName] = line.split(" "); + return { + packageName, + appName, + }; + }); + } catch { + const output = this.devicectl("device", "info", "apps", "--device", this.deviceId); + const json = JSON.parse(output) as CoreDeviceAppsOutput; + return (json.result?.apps ?? []) + .filter(app => (app.bundleIdentifier ?? "").length > 0) + .map(app => ({ + packageName: app.bundleIdentifier ?? "", + appName: app.name ?? "", + })); + } } public async launchApp(packageName: string, locale?: string): Promise { validatePackageName(packageName); - await this.assertTunnelRunning(); - const args = ["launch", packageName]; - if (locale) { - validateLocale(locale); - const locales = locale.split(",").map(l => l.trim()); - args.push("-AppleLanguages", `(${locales.join(", ")})`); - args.push("-AppleLocale", locales[0]); - } + try { + await this.assertTunnelRunning(); + const args = ["launch", packageName]; + if (locale) { + validateLocale(locale); + const locales = locale.split(",").map(l => l.trim()); + args.push("-AppleLanguages", `(${locales.join(", ")})`); + args.push("-AppleLocale", locales[0]); + } - await this.ios(...args); + await this.ios(...args); + } catch { + this.devicectl("device", "process", "launch", "--device", this.deviceId, packageName); + } } public async terminateApp(packageName: string): Promise { @@ -254,53 +372,23 @@ export class IosRobot implements Robot { export class IosManager { - public isGoIosInstalled(): boolean { + private listCoreDeviceDevicesWithDetails(): IosDeviceWithDetails[] { try { - const output = execFileSync(getGoIosPath(), ["version"], { stdio: ["pipe", "pipe", "ignore"] }).toString(); - const json: VersionCommandOutput = JSON.parse(output); - return json.version !== undefined && (/^v?\d+\.\d+\.\d+/.test(json.version) || json.version === "local-build"); + const output = execFileSync("xcrun", ["devicectl", "list", "devices", "--json-output", "-"], { stdio: ["pipe", "pipe", "ignore"] }).toString(); + return coreDeviceDevicesFromJson(output); } catch (error) { - return false; - } - } - - public getDeviceName(deviceId: string): string { - const output = execFileSync(getGoIosPath(), ["info", "--udid", deviceId]).toString(); - const json: InfoCommandOutput = JSON.parse(output); - return json.DeviceName; - } - - public getDeviceInfo(deviceId: string): InfoCommandOutput { - const output = execFileSync(getGoIosPath(), ["info", "--udid", deviceId]).toString(); - const json: InfoCommandOutput = JSON.parse(output); - return json; - } - - public listDevices(): IosDevice[] { - if (!this.isGoIosInstalled()) { - console.error("go-ios is not installed, no physical iOS devices can be detected"); return []; } - - const output = execFileSync(getGoIosPath(), ["list"]).toString(); - const json: ListCommandOutput = JSON.parse(output); - const devices = json.deviceList.map(device => ({ - deviceId: device, - deviceName: this.getDeviceName(device), - })); - - return devices; } - public listDevicesWithDetails(): Array { + private listGoIosDevicesWithDetails(): IosDeviceWithDetails[] { if (!this.isGoIosInstalled()) { - console.error("go-ios is not installed, no physical iOS devices can be detected"); return []; } const output = execFileSync(getGoIosPath(), ["list"]).toString(); const json: ListCommandOutput = JSON.parse(output); - const devices = json.deviceList.map(device => { + return json.deviceList.map(device => { const info = this.getDeviceInfo(device); return { deviceId: device, @@ -309,7 +397,64 @@ export class IosManager { platform: platformFromProductType(info.ProductType), }; }); + } + + private mergeUniqueDevices(devices: IosDeviceWithDetails[]): IosDeviceWithDetails[] { + const seen = new Set(); + return devices.filter(device => { + if (seen.has(device.deviceId)) { + return false; + } + seen.add(device.deviceId); + return true; + }); + } + + public isGoIosInstalled(): boolean { + try { + const output = execFileSync(getGoIosPath(), ["version"], { stdio: ["pipe", "pipe", "ignore"] }).toString(); + const json: VersionCommandOutput = JSON.parse(output); + return json.version !== undefined && (/^v?\d+\.\d+\.\d+/.test(json.version) || json.version === "local-build"); + } catch (error) { + return false; + } + } + + public getDeviceName(deviceId: string): string { + return this.getDeviceInfo(deviceId).DeviceName; + } + + public getDeviceInfo(deviceId: string): InfoCommandOutput { + try { + const output = execFileSync(getGoIosPath(), ["info", "--udid", deviceId]).toString(); + const json: InfoCommandOutput = JSON.parse(output); + return json; + } catch { + const output = execFileSync("xcrun", ["devicectl", "device", "info", "details", "--device", deviceId, "--json-output", "-"], { + stdio: ["pipe", "pipe", "ignore"], + }).toString(); + const json = JSON.parse(output) as CoreDeviceDetailsOutput; + return { + DeviceClass: "iPhone", + DeviceName: json.result?.deviceProperties?.name ?? deviceId, + ProductName: json.result?.hardwareProperties?.productType ?? "", + ProductType: json.result?.hardwareProperties?.productType ?? "", + ProductVersion: json.result?.deviceProperties?.osVersionNumber ?? "", + PhoneNumber: "", + TimeZone: "", + }; + } + } + + public listDevices(): IosDeviceWithDetails[] { + return this.listDevicesWithDetails(); + } - return devices; + public listDevicesWithDetails(): IosDeviceWithDetails[] { + const devices = [ + ...this.listGoIosDevicesWithDetails(), + ...this.listCoreDeviceDevicesWithDetails(), + ]; + return this.mergeUniqueDevices(devices); } } diff --git a/src/mobilecli.ts b/src/mobilecli.ts index 040fda4..7f2eabb 100644 --- a/src/mobilecli.ts +++ b/src/mobilecli.ts @@ -56,6 +56,14 @@ export interface MobilecliDevicesResponse { }; } +export interface MobilecliFocusResponse { + status: "ok" | "error"; + data?: { + element: unknown; + }; + error?: string; +} + const TIMEOUT = 30000; const MAX_BUFFER_SIZE = 1024 * 1024 * 8; @@ -177,6 +185,24 @@ export class Mobilecli { this.executeCommand(["agent", "install", "--device", deviceId]); } + pressButton(deviceId: string, button: string): void { + this.executeCommand(["io", "button", button, "--device", deviceId]); + } + + focusByIdentifier(deviceId: string, identifier?: string, label?: string): MobilecliFocusResponse { + const args = ["io", "focus", "--device", deviceId]; + if (identifier) { + args.push("--identifier", identifier); + } + + if (label) { + args.push("--label", label); + } + + const output = this.executeCommand(args); + return JSON.parse(output) as MobilecliFocusResponse; + } + getDevices(options?: MobilecliDevicesOptions): MobilecliDevicesResponse { const args = ["devices"]; diff --git a/src/server.ts b/src/server.ts index a34a3b1..9b794cf 100644 --- a/src/server.ts +++ b/src/server.ts @@ -9,11 +9,12 @@ import { ChildProcess } from "node:child_process"; import { error, trace } from "./logger"; import { AndroidRobot, AndroidDeviceManager } from "./android"; import { ActionableError, Robot } from "./robot"; -import { IosManager, IosRobot, platformFromProductType } from "./ios"; +import { IosManager, IosRobot, IosPlatform } from "./ios"; import { PNG } from "./png"; import { isScalingAvailable, Image } from "./image-utils"; import { Mobilecli } from "./mobilecli"; import { MobileDevice } from "./mobile-device"; +import { TvosRobot } from "./tvos"; import { validateOutputPath, validateFileExtension } from "./utils"; const ALLOWED_SCREENSHOT_EXTENSIONS = [".png", ".jpg", ".jpeg"]; @@ -43,6 +44,14 @@ export const getAgentVersion = (): string => { return json.version; }; +/** + * Pure platform->robot selection for iOS-family devices (D2). Real Apple TVs + * (platform === "tvos") require the Siri Remote / DeviceKit-backed TvosRobot, + * while iPhones/iPads keep the unchanged WDA-backed IosRobot. + */ +export const robotForIosDevice = (deviceId: string, platform: IosPlatform): Robot => + platform === "tvos" ? new TvosRobot(deviceId) : new IosRobot(deviceId); + export const createMcpServer = (): McpServer => { const server = new McpServer({ @@ -173,26 +182,16 @@ export const createMcpServer = (): McpServer => { const iosDevices = iosManager.listDevices(); const iosDevice = iosDevices.find(d => d.deviceId === deviceId); if (iosDevice) { - // Real Apple TV units are driven through mobilecli, which installs and drives - // the tvOS runner (re-signed with a provisioning profile). iPhones and iPads - // keep using the WebDriverAgent-based IosRobot. - const platform = platformFromProductType(iosManager.getDeviceInfo(deviceId).ProductType); - if (platform === "tvos") { - if (!agentVerifiedSimulators.has(deviceId)) { - const agentStatus = mobilecli.agentStatus(deviceId); - if (agentStatus.status === "fail") { - mobilecli.agentInstall(deviceId); - } - - agentVerifiedSimulators.add(deviceId); - } - + // Real Apple TVs are enumerated over the same connection as iPhones but + // require a dedicated Siri Remote / DeviceKit-backed robot (D2). Route + // them before the platform-blind iPhone branch. + if (iosDevice.platform === "tvos") { posthog("get_robot", { "DevicePlatform": "tvos", "DeviceType": "real" }).then(); - return new MobileDevice(deviceId); + return new TvosRobot(deviceId); } posthog("get_robot", { "DevicePlatform": "ios", "DeviceType": "real" }).then(); - return new IosRobot(deviceId); + return robotForIosDevice(deviceId, iosDevice.platform); } // Check if it's an Android device @@ -557,6 +556,31 @@ export const createMcpServer = (): McpServer => { } ); + tool( + "mobile_focus_by_identifier", + "Focus Element By Identity", + "Focus an on-screen element by accessibility identifier and/or label using Siri Remote navigation. tvOS (real Apple TV) only. At least one of identifier or label must be provided.", + { + device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."), + identifier: z.string().optional().describe("The accessibility identifier of the element to focus."), + label: z.string().optional().describe("The accessibility label of the element to focus."), + }, + { destructiveHint: true }, + async ({ device, identifier, label }) => { + if (!identifier && !label) { + throw new ActionableError("At least one of identifier or label must be provided."); + } + + const robot = getRobotFromDevice(device); + if (!(robot instanceof TvosRobot)) { + throw new ActionableError("focus by identity is only supported on tvOS (real Apple TV)."); + } + + const element = await robot.focus(identifier, label); + return `Focused element: ${JSON.stringify(element)}`; + } + ); + tool( "mobile_open_url", "Open URL", diff --git a/src/tvos.ts b/src/tvos.ts new file mode 100644 index 0000000..1462f8e --- /dev/null +++ b/src/tvos.ts @@ -0,0 +1,221 @@ +import { Mobilecli } from "./mobilecli"; +import { ActionableError, Button, InstalledApp, Orientation, Robot, ScreenElement, ScreenSize, SwipeDirection } from "./robot"; + +interface InstalledAppsResponse { + status: "ok", + data: Array<{ + packageName: string; + appName?: string; + version?: string; + }>; +} + +interface DeviceInfoResponse { + status: "ok", + data: { + device: { + id: string; + name: string; + platform: string; + type: string; + version: string; + state: string; + screenSize?: { + width: number; + height: number; + scale: number; + }; + }; + }; +} + +interface UIElementResponse { + type: string; + label?: string; + text?: string; + name?: string; + value?: string; + identifier?: string; + rect: { + x: number; + y: number; + width: number; + height: number; + }; + focused?: boolean; +} + +interface DumpUIResponse { + status: "ok", + data: { + elements: UIElementResponse[]; + }; +} + +/** + * Siri Remote buttons exposed by the tvOS device.io.button handler. These are + * distinct from the physical/D-pad buttons in the shared `Button` union (which + * stays untouched for iPhone/Android per decision D2). + */ +export type TvosButton = "UP" | "DOWN" | "LEFT" | "RIGHT" | "SELECT" | "MENU" | "PLAY_PAUSE"; + +const TVOS_BUTTONS: readonly TvosButton[] = ["UP", "DOWN", "LEFT", "RIGHT", "SELECT", "MENU", "PLAY_PAUSE"]; + +const isTvosButton = (button: string): button is TvosButton => + (TVOS_BUTTONS as readonly string[]).includes(button); + +/** + * TvosRobot drives a real Apple TV through the mobilecli binary (backed by the + * DeviceKit runner over the CoreDevice tunnel). Touch and orientation operations + * do not apply to tvOS and surface explicit `ActionableError`s rather than silent + * no-ops. + */ +export class TvosRobot implements Robot { + + private mobilecli: Mobilecli; + + public constructor(private deviceId: string) { + this.mobilecli = new Mobilecli(); + } + + private runCommand(args: string[]): string { + const fullArgs = [...args, "--device", this.deviceId]; + return this.mobilecli.executeCommand(fullArgs); + } + + public async getScreenSize(): Promise { + const response = JSON.parse(this.runCommand(["device", "info"])) as DeviceInfoResponse; + if (response.data.device.screenSize) { + return response.data.device.screenSize; + } + + return { width: 0, height: 0, scale: 1.0 }; + } + + public async swipe(_direction: SwipeDirection): Promise { + throw new ActionableError("swipe is not supported on tvOS"); + } + + public async swipeFromCoordinate(_x: number, _y: number, _direction: SwipeDirection, _distance?: number): Promise { + throw new ActionableError("swipe is not supported on tvOS"); + } + + public async getScreenshot(): Promise { + const fullArgs = ["screenshot", "--device", this.deviceId, "--format", "png", "--output", "-"]; + return this.mobilecli.executeCommandBuffer(fullArgs); + } + + public async listApps(): Promise { + const response = JSON.parse(this.runCommand(["apps", "list"])) as InstalledAppsResponse; + return response.data.map(app => ({ + appName: app.appName || app.packageName, + packageName: app.packageName, + })) as InstalledApp[]; + } + + public async launchApp(packageName: string, locale?: string): Promise { + const args = ["apps", "launch", packageName]; + if (locale) { + args.push("--locale", locale); + } + + this.runCommand(args); + } + + public async terminateApp(packageName: string): Promise { + this.runCommand(["apps", "terminate", packageName]); + } + + public async installApp(path: string): Promise { + this.runCommand(["apps", "install", path]); + } + + public async uninstallApp(bundleId: string): Promise { + this.runCommand(["apps", "uninstall", bundleId]); + } + + public async openUrl(url: string): Promise { + // Best-effort on tvOS (D5): attempt the open and surface the underlying + // error as an ActionableError rather than a silent failure. + try { + this.runCommand(["url", url]); + } catch (err: any) { + throw new ActionableError(`Failed to open URL on tvOS: ${err instanceof Error ? err.message : String(err)}`); + } + } + + public async sendKeys(_text: string): Promise { + throw new ActionableError("text entry is not supported on tvOS"); + } + + public async pressButton(button: Button): Promise { + if (!isTvosButton(button as string)) { + throw new ActionableError(`Button "${button}" is not supported on tvOS. Supported Siri Remote buttons: ${TVOS_BUTTONS.join(", ")}`); + } + + this.mobilecli.pressButton(this.deviceId, button); + } + + public async tap(_x: number, _y: number): Promise { + throw new ActionableError("tap is not supported on tvOS"); + } + + public async doubleTap(_x: number, _y: number): Promise { + throw new ActionableError("doubleTap is not supported on tvOS"); + } + + public async longPress(_x: number, _y: number, _duration: number): Promise { + throw new ActionableError("longPress is not supported on tvOS"); + } + + public async getElementsOnScreen(): Promise { + const response = JSON.parse(this.runCommand(["dump", "ui"])) as DumpUIResponse; + return response.data.elements.map(element => ({ + type: element.type, + label: element.label, + text: element.text, + name: element.name, + value: element.value, + identifier: element.identifier, + rect: element.rect, + focused: element.focused, + })); + } + + public async setOrientation(_orientation: Orientation): Promise { + throw new ActionableError("setOrientation is not supported on tvOS"); + } + + public async getOrientation(): Promise { + throw new ActionableError("getOrientation is not supported on tvOS"); + } + + /** + * Focus an on-screen element by accessibility identifier and/or label, driving + * Siri Remote focus through the DeviceKit `device.io.focus` RPC exposed by + * mobilecli. Returns the focused element as reported by the runner. + */ + public async focus(identifier?: string, label?: string): Promise { + if (!identifier && !label) { + throw new ActionableError("focus requires at least one of identifier or label"); + } + + let response; + try { + response = this.mobilecli.focusByIdentifier(this.deviceId, identifier, label); + } catch (err: any) { + throw new ActionableError(`Failed to focus element on tvOS: ${err instanceof Error ? err.message : String(err)}`); + } + + if (response.status !== "ok") { + throw new ActionableError(response.error ?? "Failed to focus element on tvOS"); + } + + const focusedElement = response.data?.element; + if (focusedElement === undefined || focusedElement === null) { + throw new ActionableError("focus returned no element on tvOS"); + } + + return focusedElement; + } +} diff --git a/test/ios-coredevice.ts b/test/ios-coredevice.ts new file mode 100644 index 0000000..d0af845 --- /dev/null +++ b/test/ios-coredevice.ts @@ -0,0 +1,58 @@ +import { test, expect } from "@playwright/test"; + +import { coreDeviceDevicesFromJson } from "../src/ios"; + +test.describe("ios coredevice discovery", () => { + + test("uses the hardware UDID for connected physical devices", () => { + const devices = coreDeviceDevicesFromJson(JSON.stringify({ + result: { + devices: [ + { + connectionProperties: { + pairingState: "paired", + transportType: "localNetwork", + tunnelState: "connected", + }, + deviceProperties: { + bootState: "booted", + name: "iPhone-J7KRQL2Q75", + osVersionNumber: "26.5.2", + }, + hardwareProperties: { + productType: "iPhone17,5", + reality: "physical", + udid: "00008140-001975D9349B801C", + }, + }, + { + connectionProperties: { + pairingState: "paired", + transportType: "sameMachine", + tunnelState: "disconnected", + }, + deviceProperties: { + bootState: "booted", + name: "UnitTests (iOS)", + osVersionNumber: "26.2", + }, + hardwareProperties: { + productType: "iPhone18,3", + reality: "simulated", + udid: "56BABF10-C8A0-43F4-93B0-1891E98BE95E", + }, + }, + ], + }, + })); + + expect(devices).toEqual([ + { + deviceId: "00008140-001975D9349B801C", + deviceName: "iPhone-J7KRQL2Q75", + version: "26.5.2", + platform: "ios", + }, + ]); + }); +}); diff --git a/test/tvos.ts b/test/tvos.ts new file mode 100644 index 0000000..78aa5f4 --- /dev/null +++ b/test/tvos.ts @@ -0,0 +1,156 @@ +import { test, expect } from "@playwright/test"; + +import { TvosRobot } from "../src/tvos"; +import { IosRobot } from "../src/ios"; +import { ActionableError } from "../src/robot"; +import { coreDeviceDevicesFromJson } from "../src/ios"; +import { robotForIosDevice } from "../src/server"; + +test.describe("tvos", () => { + + test.describe("platform routing", () => { + test("retains tvos platform for Apple TV devices (routing key)", () => { + const devices = coreDeviceDevicesFromJson(JSON.stringify({ + result: { + devices: [ + { + connectionProperties: { + pairingState: "paired", + transportType: "localNetwork", + tunnelState: "connected", + }, + deviceProperties: { + bootState: "booted", + name: "Bedroom", + osVersionNumber: "18.5", + }, + hardwareProperties: { + productType: "AppleTV14,1", + reality: "physical", + udid: "00008110-000A1B2C3D4E5F60", + }, + }, + ], + }, + })); + + expect(devices).toEqual([ + { + deviceId: "00008110-000A1B2C3D4E5F60", + deviceName: "Bedroom", + version: "18.5", + platform: "tvos", + }, + ]); + }); + + test("routes tvos devices to TvosRobot (D2)", () => { + const robot = robotForIosDevice("apple-tv-1", "tvos"); + expect(robot).toBeInstanceOf(TvosRobot); + }); + + test("routes non-tvos iOS devices to IosRobot, not TvosRobot", () => { + const robot = robotForIosDevice("iphone-1", "ios"); + expect(robot).toBeInstanceOf(IosRobot); + expect(robot).not.toBeInstanceOf(TvosRobot); + }); + }); + + test.describe("pressButton", () => { + test("maps valid Siri Remote tokens to mobilecli io button", async () => { + const robot = new TvosRobot("apple-tv-1"); + const calls: Array<{ deviceId: string; button: string }> = []; + (robot as any).mobilecli.pressButton = (deviceId: string, button: string) => { + calls.push({ deviceId, button }); + }; + + for (const button of ["UP", "DOWN", "LEFT", "RIGHT", "SELECT", "MENU", "PLAY_PAUSE"] as const) { + await robot.pressButton(button as any); + } + + expect(calls).toEqual([ + { deviceId: "apple-tv-1", button: "UP" }, + { deviceId: "apple-tv-1", button: "DOWN" }, + { deviceId: "apple-tv-1", button: "LEFT" }, + { deviceId: "apple-tv-1", button: "RIGHT" }, + { deviceId: "apple-tv-1", button: "SELECT" }, + { deviceId: "apple-tv-1", button: "MENU" }, + { deviceId: "apple-tv-1", button: "PLAY_PAUSE" }, + ]); + }); + + test("rejects unknown tokens with ActionableError", async () => { + const robot = new TvosRobot("apple-tv-1"); + (robot as any).mobilecli.pressButton = () => { + throw new Error("should not be called"); + }; + + await expect(robot.pressButton("HOME" as any)).rejects.toThrow(ActionableError); + await expect(robot.pressButton("VOLUME_UP" as any)).rejects.toThrow(/not supported on tvOS/); + }); + }); + + test.describe("unsupported operations", () => { + const robot = new TvosRobot("apple-tv-1"); + + test("tap throws a tvOS ActionableError", async () => { + await expect(robot.tap(1, 2)).rejects.toThrow(ActionableError); + await expect(robot.tap(1, 2)).rejects.toThrow(/not supported on tvOS/); + }); + + test("doubleTap throws a tvOS ActionableError", async () => { + await expect(robot.doubleTap(1, 2)).rejects.toThrow(/not supported on tvOS/); + }); + + test("longPress throws a tvOS ActionableError", async () => { + await expect(robot.longPress(1, 2, 500)).rejects.toThrow(/not supported on tvOS/); + }); + + test("swipe throws a tvOS ActionableError", async () => { + await expect(robot.swipe("up")).rejects.toThrow(/not supported on tvOS/); + }); + + test("swipeFromCoordinate throws a tvOS ActionableError", async () => { + await expect(robot.swipeFromCoordinate(1, 2, "up")).rejects.toThrow(/not supported on tvOS/); + }); + + test("setOrientation throws a tvOS ActionableError", async () => { + await expect(robot.setOrientation("portrait")).rejects.toThrow(/not supported on tvOS/); + }); + + test("getOrientation throws a tvOS ActionableError", async () => { + await expect(robot.getOrientation()).rejects.toThrow(/not supported on tvOS/); + }); + + test("sendKeys throws a tvOS ActionableError", async () => { + await expect(robot.sendKeys("hello")).rejects.toThrow(/text entry is not supported on tvOS/); + }); + }); + + test.describe("focus", () => { + test("requires at least one of identifier or label", async () => { + const robot = new TvosRobot("apple-tv-1"); + await expect(robot.focus()).rejects.toThrow(/at least one of identifier or label/); + }); + + test("returns the focused element on success", async () => { + const robot = new TvosRobot("apple-tv-1"); + const calls: Array<{ deviceId: string; identifier?: string; label?: string }> = []; + (robot as any).mobilecli.focusByIdentifier = (deviceId: string, identifier?: string, label?: string) => { + calls.push({ deviceId, identifier, label }); + return { status: "ok", data: { element: { identifier: "sportButton", type: "Button" } } }; + }; + + const element = await robot.focus("sportButton"); + expect(element).toEqual({ identifier: "sportButton", type: "Button" }); + expect(calls).toEqual([{ deviceId: "apple-tv-1", identifier: "sportButton", label: undefined }]); + }); + + test("surfaces the underlying error as ActionableError", async () => { + const robot = new TvosRobot("apple-tv-1"); + (robot as any).mobilecli.focusByIdentifier = () => ({ status: "error", error: "element not found after 50 moves" }); + + await expect(robot.focus("missing")).rejects.toThrow(/element not found after 50 moves/); + }); + }); +});