From dcbbd4e5b7a899aff2ed1491de5aac6a3abd1dd4 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Sun, 27 Sep 2026 12:36:57 -0700 Subject: [PATCH 01/21] feat: add point inspection command --- .../RunnerTests+CommandExecution.swift | 9 +-- .../RunnerTests+Interaction.swift | 43 +++++++++++--- .../RunnerTests+Models.swift | 12 ++++ contracts/fixtures/runner-requests.json | 1 + .../src/__tests__/command-result.test.ts | 1 + .../command-registry/src/command-result.ts | 2 + .../src/flag-definitions-action.ts | 18 ++++++ packages/command-registry/src/registry.ts | 16 +++++ packages/contracts/package.json | 4 ++ packages/contracts/src/cli-flags.ts | 2 + packages/contracts/src/client-capture.ts | 17 ++++++ packages/contracts/src/facades/client.ts | 2 + packages/contracts/src/interactor-types.ts | 22 +++++++ .../src/platform-runtime-operations.ts | 3 + .../src/platform-runtime-unavailable.ts | 2 + .../contracts/src/point-inspection-runtime.ts | 54 +++++++++++++++++ .../contracts/src/runtime-operation-names.ts | 1 + packages/platform-android/src/runtime.test.ts | 1 + packages/platform-android/src/runtime.ts | 2 + .../interactor-runner-provider.test.ts | 4 ++ .../src/__tests__/runner-requests.test.ts | 4 ++ packages/platform-apple/src/interactor.ts | 59 +++++++++++++++++++ packages/platform-apple/src/runner-demand.ts | 1 + packages/platform-apple/src/runtime.test.ts | 5 ++ packages/platform-apple/src/runtime.ts | 22 +++++++ packages/platform-harmonyos/src/runtime.ts | 2 + packages/platform-linux/src/runtime.test.ts | 1 + packages/platform-linux/src/runtime.ts | 2 + packages/platform-web/src/runtime.test.ts | 1 + packages/platform-web/src/runtime.ts | 2 + packages/provider-limrun/src/facts-runtime.ts | 2 + scripts/integration-progress-model.ts | 2 + .../layering/contracts-exports.snapshot.json | 1 + src/__tests__/cli-client-commands.test.ts | 1 + .../test-utils/property-arbitraries.ts | 1 + .../test-utils/runtime-operation-facts.ts | 2 + src/agent-device-client.ts | 3 + src/client/client-types.ts | 11 ++-- src/commands/capture/index.test.ts | 14 +++++ src/commands/capture/index.ts | 2 + src/commands/capture/inspect-point.ts | 44 ++++++++++++++ .../__tests__/runtime-binding-conformance.ts | 11 ++++ src/daemon/generic-runtime-execution.ts | 8 +++ .../handlers/__tests__/install-source.test.ts | 1 + src/daemon/inspect-point-runtime.ts | 46 +++++++++++++++ src/mcp/command-output-schemas.ts | 44 ++++++++++++++ .../command-coverage/declarations.ts | 31 ++++++++++ test/integration/command-coverage/evidence.ts | 2 +- .../apple-platform-output-guard.test.ts | 13 ++-- .../provider-scenarios/ios-lifecycle.test.ts | 22 +++++++ .../provider-scenarios/ios-world.ts | 26 ++++++++ website/docs/docs/commands.md | 6 ++ 52 files changed, 581 insertions(+), 27 deletions(-) create mode 100644 packages/contracts/src/point-inspection-runtime.ts create mode 100644 src/commands/capture/inspect-point.ts create mode 100644 src/daemon/inspect-point-runtime.ts diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index a96560e417..2a6fff542a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -422,10 +422,11 @@ extension RunnerTests { guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "readText requires x and y")) } - guard let text = readTextAt(app: activeApp, x: x, y: y) else { - return Response(ok: false, error: ErrorPayload(message: "readText did not resolve text")) - } - return Response(ok: true, data: DataPayload(text: text)) + let inspection = readPointAt(app: activeApp, x: x, y: y) + return Response( + ok: true, + data: DataPayload(text: inspection.text, elements: inspection.elements) + ) case .screenshot: #if os(macOS) // macOS keeps the app-targeted capture behavior for window-level screenshots. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 002459b19c..1cadd0aa4a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -318,31 +318,56 @@ extension RunnerTests { } func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { + readPointAt(app: app, x: x, y: y).text + } + + func readPointAt( + app: XCUIApplication, + x: Double, + y: Double + ) -> (text: String?, elements: [PointInspectionElementPayload]) { let point = CGPoint(x: x, y: y) let textInputCandidates = textInputCandidatesAt(app: app, point: point) - for element in textInputCandidates where prefersExpandedTextRead(element) { - if let text = readableText(for: element) { - return text - } - } - let candidates = app.descendants(matching: .any).allElementsBoundByIndex .filter { element in element.exists && !element.frame.isEmpty && element.frame.contains(point) } .sorted(by: smallestElementFirst) + let elements = Array(candidates.prefix(24)).map { element in + let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) + let value = String(describing: element.value ?? "") + .trimmingCharacters(in: .whitespacesAndNewlines) + return PointInspectionElementPayload( + text: readableText(for: element), + label: label.isEmpty ? nil : label, + identifier: identifier.isEmpty ? nil : identifier, + type: elementTypeName(element.elementType), + role: elementTypeName(element.elementType), + value: value.isEmpty ? nil : value, + frame: SnapshotRect(element.frame), + hittable: element.isHittable + ) + } + + for element in textInputCandidates where prefersExpandedTextRead(element) { + if let text = readableText(for: element) { + return (text, elements) + } + } + for element in candidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { - return text + return (text, elements) } } for element in candidates { if let text = readableText(for: element) { - return text + return (text, elements) } } - return nil + return (nil, elements) } private func readableText(for element: XCUIElement) -> String? { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 73c7c52761..ad5bf379f5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -391,6 +391,7 @@ struct DataPayload: Codable { var message: String? var imageBase64: String? var text: String? + var elements: [PointInspectionElementPayload]? var found: Bool? var items: [String]? var nodes: [PresentedNode]? @@ -451,6 +452,17 @@ struct DataPayload: Codable { var screenshotMetadata: ScreenshotMetadataPayload? } +struct PointInspectionElementPayload: Codable { + let text: String? + let label: String? + let identifier: String? + let type: String? + let role: String? + let value: String? + let frame: SnapshotRect + let hittable: Bool? +} + /// `kind` mirrors the TS `SnapshotKeyboardBandFact`: "visible" carries `frame`, "unmeasurable" /// carries `reason`, and "absent" carries nothing because there is nothing to say. `frame` is in the /// app's own orientation space — the same space `SnapshotGeometrySpace` publishes every node rect in diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index 730c8cb359..4e8d146df5 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -38,6 +38,7 @@ {"name": "ios-simulator.interactor-back.system", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "backSystem", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-find-text.text", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "findText", "text": "Ready", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-home.press", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "home", "appBundleId": "com.example.app", "commandId": ""}}, + {"name": "ios-simulator.interactor-inspect-point", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-keyboard-dismiss.dismiss", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "keyboardDismiss", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-keyboard-enter.return", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "keyboardReturn", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-read-text.point", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, diff --git a/packages/command-registry/src/__tests__/command-result.test.ts b/packages/command-registry/src/__tests__/command-result.test.ts index 4cc3f6a7db..117990e346 100644 --- a/packages/command-registry/src/__tests__/command-result.test.ts +++ b/packages/command-registry/src/__tests__/command-result.test.ts @@ -143,6 +143,7 @@ test('CommandResultMap is seeded only from already-existing contract result type | 'test' | 'record' | 'trace' + | 'inspect-point' > = true; void keys; }); diff --git a/packages/command-registry/src/command-result.ts b/packages/command-registry/src/command-result.ts index 11f9d271e9..f6a4ea85b8 100644 --- a/packages/command-registry/src/command-result.ts +++ b/packages/command-registry/src/command-result.ts @@ -2,6 +2,7 @@ import type { DiffSnapshotCommandResult, ViewportCommandResult, } from '@agent-device/contracts/capture'; +import type { PointInspectionResult } from '@agent-device/contracts/client'; import type { PrepareCommandResult, PushCommandResult } from '@agent-device/contracts/command'; import type { AppStateCommandResult, @@ -72,6 +73,7 @@ export interface CommandResultMap { fold: FoldCommandResult; home: HomeCommandResult; hover: HoverCommandResponseData; + 'inspect-point': PointInspectionResult; keyboard: KeyboardCommandResult; longpress: LongPressCommandResponseData; orientation: OrientationCommandResult; diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index cd697ca8c3..57f3d02a8c 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -3,6 +3,24 @@ import { RECORDING_SCOPE_VALUES } from '@agent-device/contracts/recording'; import type { FlagDefinition } from './flag-types.ts'; export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ + { + key: 'pointX', + names: ['--x'], + type: 'number', + usageLabel: '--x ', + usageDescription: 'Inspect-point: horizontal screen coordinate', + projectConfig: false, + recorded: false, + }, + { + key: 'pointY', + names: ['--y'], + type: 'number', + usageLabel: '--y ', + usageDescription: 'Inspect-point: vertical screen coordinate', + projectConfig: false, + recorded: false, + }, { key: 'keyframes', names: ['--keyframes'], diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index 43ebe1a20c..beda344a5a 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -57,6 +57,7 @@ import { gestureRuntimePlanUses, gestureViewportRuntimeUse, homeRuntimeUse, + inspectPointRuntimeUse, hoverRuntimeUses, keyboardRuntimePlanUses, longPressRuntimeUses, @@ -1078,6 +1079,21 @@ export const RAW_COMMAND_DESCRIPTORS = [ platformExecution: { kind: 'device-runtime', uses: closeApplicationRuntimePlanUses }, }, + { + name: 'inspect-point', + deviceClaimPolicy: 'require-owner', + ...(ownerFilesEnabled + ? { ownerFiles: ['src/commands/capture/inspect-point.ts'] as const } + : {}), + catalog: { group: 'public' }, + frameworkTier: 'extended', + recordsSessionAction: false, + daemon: { route: 'generic', refFrameEffect: 'preserve' }, + timeoutPolicy: DEFAULT_TIMEOUT_POLICY, + batchable: true, + platformExecution: { kind: 'device-runtime', uses: [inspectPointRuntimeUse] }, + }, + // -- snapshot (route: snapshot) -- { name: 'snapshot', diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 9582f1ada8..598264821f 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -188,6 +188,10 @@ "types": "./src/element-text-runtime.ts", "default": "./src/element-text-runtime.ts" }, + "./point-inspection-runtime": { + "types": "./src/point-inspection-runtime.ts", + "default": "./src/point-inspection-runtime.ts" + }, "./fill-evidence": { "types": "./src/fill-evidence.ts", "default": "./src/fill-evidence.ts" diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index 8cd4412922..a00950478c 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -109,6 +109,8 @@ export type CliFlags = CloudProviderProfileFields & holdMs?: number; jitterPx?: number; pixels?: number; + pointX?: number; + pointY?: number; /** Scroll: repeat passes until this selector is visible on screen. */ until?: string; doubleTap?: boolean; diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 844897f725..b0bfb1c905 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -11,6 +11,7 @@ import type { SnapshotVisibility, } from '@agent-device/kernel/snapshot'; import type { ScreenshotResultData } from './snapshot-types.ts'; +import type { PointInspectionElement } from './interactor-types.ts'; import type { AgentDeviceIdentifiers, AgentDeviceRequestOverrides, @@ -102,6 +103,22 @@ export type CaptureScreenshotResult = ScreenshotResultData & { identifiers: AgentDeviceIdentifiers; }; +export type InspectPointOptions = DeviceCommandBaseOptions & { + point: Readonly<{ x: number; y: number }>; +}; + +export type PointInspectionResult = + | Readonly<{ + status: 'inspected'; + point: Readonly<{ x: number; y: number }>; + elements: readonly PointInspectionElement[]; + }> + | Readonly<{ + status: 'no-element-at-point'; + point: Readonly<{ x: number; y: number }>; + elements: readonly []; + }>; + export type CaptureDiffOptions = DeviceCommandBaseOptions & Pick & { kind: 'snapshot'; diff --git a/packages/contracts/src/facades/client.ts b/packages/contracts/src/facades/client.ts index 8cd5d9dad2..5ecb33cd2e 100644 --- a/packages/contracts/src/facades/client.ts +++ b/packages/contracts/src/facades/client.ts @@ -23,6 +23,8 @@ export type { CaptureScreenshotResult, CaptureSnapshotOptions, CaptureSnapshotResult, + InspectPointOptions, + PointInspectionResult, FindSnapshotCommandOptions, SelectorSnapshotCommandOptions, } from '../client-capture.ts'; diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index dd5cb82363..7b101933e8 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -39,6 +39,24 @@ export type RunnerContext = { runnerLeaseContext?: RunnerLogicalLeaseContext; }; +/** One bounded accessibility descriptor returned by a native point inspection. */ +export type PointInspectionElement = Readonly<{ + text?: string; + label?: string; + identifier?: string; + type?: string; + role?: string; + value?: string; + frame: Rect; + hittable?: boolean; +}>; + +/** Native point-read payload. Elements are ordered from smallest to largest containing frame. */ +export type PointInspectionRead = Readonly<{ + text?: string; + elements: readonly PointInspectionElement[]; +}>; + /** Subset of {@link RunnerContext} forwarded to runner command invocations. */ export type RunnerCallOptions = Pick< RunnerContext, @@ -355,6 +373,10 @@ export type Interactor = { point: Point, options?: { appBundleId?: string; surface?: SessionSurface; signal?: AbortSignal }, ): Promise; + inspectPoint?( + point: Point, + options?: { appBundleId?: string; surface?: SessionSurface; signal?: AbortSignal }, + ): Promise; /** * Native text-presence reading, when the backend has one that does not require a tree capture. * A `true` answer is authoritative; anything else means "not proven here" and the caller diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 3d6981294d..ff3de4b265 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -20,6 +20,7 @@ import type { GestureRuntimeOperations } from './gesture-runtime.ts'; import type { ScrollRuntimeOperations } from './scroll-runtime.ts'; import type { TypeTextRuntimeOperations } from './type-text-runtime.ts'; import type { ElementTextRuntimeOperations } from './element-text-runtime.ts'; +import type { PointInspectionRuntimeOperations } from './point-inspection-runtime.ts'; import type { BackRuntimeOperations } from './back-runtime.ts'; import type { OrientationRuntimeOperations } from './orientation-runtime.ts'; import type { FoldRuntimeOperations } from './fold-runtime.ts'; @@ -74,6 +75,7 @@ export type PlatformRuntimeOperations = AppLogRuntimeOperations & ScrollRuntimeOperations & TypeTextRuntimeOperations & ElementTextRuntimeOperations & + PointInspectionRuntimeOperations & BackRuntimeOperations & OrientationRuntimeOperations & FoldRuntimeOperations & @@ -108,6 +110,7 @@ export const captureSnapshotUse = defineUse({ required: ['captureSnapshot'] }); export const viewportRuntimeUse = defineUse({ required: ['setViewport'] }); export const focusRuntimeUse = defineUse({ required: ['focusPoint'] }); export const typeTextRuntimeUse = defineUse({ required: ['typeText'] }); +export const inspectPointRuntimeUse = defineUse({ required: ['inspectPoint'] }); export const backRuntimeUse = defineUse({ required: ['back'] }); export const homeRuntimeUse = defineUse({ required: ['home'] }); export const orientationRuntimeUse = defineUse({ required: ['setOrientation'] }); diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index 7e84509948..9a6acfb8ec 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -20,6 +20,7 @@ import { gestureRuntimeOperationFacts } from './gesture-runtime.ts'; import { scrollRuntimeOperationFacts } from './scroll-runtime.ts'; import { typeTextRuntimeOperationFacts } from './type-text-runtime.ts'; import { elementTextRuntimeOperationFacts } from './element-text-runtime.ts'; +import { pointInspectionRuntimeOperationFacts } from './point-inspection-runtime.ts'; import { backRuntimeOperationFacts } from './back-runtime.ts'; import { orientationRuntimeOperationFacts } from './orientation-runtime.ts'; import { tvRemoteRuntimeOperationFacts } from './tv-remote-runtime.ts'; @@ -237,6 +238,7 @@ export function createUnavailablePlatformRuntimeFacts( fill: frozen.touch, }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: frozen.elementText }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: frozen.elementText }), ...backRuntimeOperationFacts({ back: frozen.back }), ...orientationRuntimeOperationFacts({ orientation: frozen.orientation }), // Stated directly rather than through `foldRuntimeOperationFacts`, so this hub does not diff --git a/packages/contracts/src/point-inspection-runtime.ts b/packages/contracts/src/point-inspection-runtime.ts new file mode 100644 index 0000000000..1444567d67 --- /dev/null +++ b/packages/contracts/src/point-inspection-runtime.ts @@ -0,0 +1,54 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Point } from '@agent-device/kernel/snapshot'; +import type { Interactor, PointInspectionRead, RunnerContext } from './interactor-types.ts'; +import { invalidRuntimeContract } from './runtime-contract-error.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; +import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; + +export type InspectPointRuntimeInput = Readonly<{ + point: Point; + options?: Readonly<{ appBundleId?: string }>; + execution?: SnapshotRuntimeExecution; +}>; + +export type PointInspectionRuntimeOperations = Readonly<{ + inspectPoint(input: InspectPointRuntimeInput): Promise; +}>; + +export type PointInspectionRuntimeOperationFacts = Readonly<{ + inspectPoint: RuntimeOperationFact; +}>; + +export function pointInspectionRuntimeOperationFacts( + input: PointInspectionRuntimeOperationFacts, +): PointInspectionRuntimeOperationFacts { + return Object.freeze({ inspectPoint: input.inspectPoint }); +} + +export function bindPointInspectionRuntime( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: (device: DeviceInfo, runner: RunnerContext) => Promise; + }>, +): PointInspectionRuntimeOperations { + return Object.freeze({ + inspectPoint: async (input) => { + params.signal.throwIfAborted(); + const interactor = await params.resolveInteractor(params.device, { + ...input.execution, + appBundleId: input.options?.appBundleId, + signal: params.signal, + }); + if (typeof interactor.inspectPoint !== 'function') { + throw invalidRuntimeContract( + 'Runtime owner advertised inspectPoint without an interactor implementation', + ); + } + return await interactor.inspectPoint(input.point, { + appBundleId: input.options?.appBundleId, + signal: params.signal, + }); + }, + }); +} diff --git a/packages/contracts/src/runtime-operation-names.ts b/packages/contracts/src/runtime-operation-names.ts index 25bf9a1c7b..e12d228429 100644 --- a/packages/contracts/src/runtime-operation-names.ts +++ b/packages/contracts/src/runtime-operation-names.ts @@ -69,6 +69,7 @@ export const RUNTIME_OPERATION_NAMES = [ 'readClipboard', 'readSetting', 'readTextAtPoint', + 'inspectPoint', 'resolveOpenTarget', 'screenRecordingCleanup', 'screenRecordingReattach', diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index 011afb24f9..a948d85a08 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -78,6 +78,7 @@ test.each([ expect(facts.operations.captureSnapshot).toEqual({ available: true }); // uiautomator reads text at a point over the same adb transport the capture uses. expect(facts.operations.readTextAtPoint).toEqual({ available: true }); + expect(facts.operations.inspectPoint).toMatchObject({ available: false }); expect(facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(facts.operations.setViewport).toMatchObject({ available: false }); diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 2a0e4c9f3b..42ea3401ca 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -15,6 +15,7 @@ import { availableApplicationLifecycleOperations, } from '@agent-device/contracts/application-lifecycle-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { focusRuntimeOperationFacts } from '@agent-device/contracts/focus-runtime'; import { ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, @@ -365,6 +366,7 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor ...elementTextRuntimeOperationFacts({ readTextAtPoint: device.kind === 'simulator' ? elementTextKindUnavailable : available, }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextKindUnavailable }), ...backRuntimeOperationFacts({ back: androidTouchFact(device) }), // `home` and `app-switcher` are one `input keyevent` each, admitted wherever the retired // `ANDROID_ALL` bucket admitted them. diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index eed7d1d34e..f349eb9f8a 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -69,6 +69,10 @@ const RUNNER_TRANSPORT_METHODS: Record< invoke: (i) => i.readTextAtPoint!({ x: 10, y: 20 }), runnerCommand: 'readText', }, + inspectPoint: { + invoke: (i) => i.inspectPoint!({ x: 10, y: 20 }), + runnerCommand: 'readText', + }, findText: { invoke: (i) => i.findText!('Ready'), runnerCommand: 'findText' }, back: { invoke: (i) => i.back(), runnerCommand: 'backInApp' }, home: { invoke: (i) => i.home!(), runnerCommand: 'home' }, diff --git a/packages/platform-apple/src/__tests__/runner-requests.test.ts b/packages/platform-apple/src/__tests__/runner-requests.test.ts index e053178cc9..8d33e2c6e7 100644 --- a/packages/platform-apple/src/__tests__/runner-requests.test.ts +++ b/packages/platform-apple/src/__tests__/runner-requests.test.ts @@ -171,6 +171,10 @@ const INTERACTOR_SITES: Record = { IOS_SIMULATOR, (i) => i.readTextAtPoint!({ x: 10, y: 20 }, { appBundleId: APP }), ], + 'ios-simulator.interactor-inspect-point': [ + IOS_SIMULATOR, + (i) => i.inspectPoint!({ x: 10, y: 20 }, { appBundleId: APP }), + ], 'ios-simulator.alert.get': [IOS_SIMULATOR, (i) => i.readAlert!({ appBundleId: APP })], 'ios-simulator.alert.accept': [IOS_SIMULATOR, (i) => i.acceptAlert!({ appBundleId: APP })], 'ios-simulator.alert.dismiss': [IOS_SIMULATOR, (i) => i.dismissAlert!({ appBundleId: APP })], diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 0d960fea39..b54e90e0e3 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -24,6 +24,8 @@ import { withMethodScope } from '@agent-device/kernel/scoped-provider'; import type { Point, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; import type { Interactor, + PointInspectionElement, + PointInspectionRead, RunnerCallOptions, RunnerContext, ScreenshotOptions, @@ -76,6 +78,8 @@ export function createAppleInteractor( ? await readMacOsSurfaceTextAtPoint(point, helper, options?.appBundleId) : await readRunnerTextAtPoint(device, point, options, runnerOpts); }, + inspectPoint: async (point, options) => + await inspectRunnerPoint(device, point, options, runnerOpts), // The XCTest runner's own text reading: it observes the live accessibility hierarchy // directly, so it answers without the cost — and without the pruning — of a tree capture. // Only a positive answer is authoritative; see `FindTextResult`. @@ -454,3 +458,58 @@ async function readRunnerTextAtPoint( // The runner answers `message` when it reached the element but rendered no readable text. return typeof result.message === 'string' ? result.message : undefined; } + +async function inspectRunnerPoint( + device: DeviceInfo, + point: Point, + options: { appBundleId?: string; signal?: AbortSignal } | undefined, + runnerOpts: RunnerCallOptions, +): Promise { + const result = await runAppleRunnerCommand( + device, + { command: 'readText', x: point.x, y: point.y, appBundleId: options?.appBundleId }, + options?.signal ? { ...runnerOpts, signal: options.signal } : runnerOpts, + ); + const elements = Array.isArray(result.elements) + ? result.elements.flatMap((value) => { + const element = readPointInspectionElement(value); + return element ? [element] : []; + }) + : []; + return { + ...(typeof result.text === 'string' ? { text: result.text } : {}), + elements, + }; +} + +function readPointInspectionElement(value: unknown): PointInspectionElement | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const record = value as Record; + const frame = readPointInspectionFrame(record.frame); + if (!frame) return undefined; + return { + ...readPointInspectionStrings(record), + frame, + ...(typeof record.hittable === 'boolean' ? { hittable: record.hittable } : {}), + }; +} + +function readPointInspectionFrame(value: unknown): PointInspectionElement['frame'] | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const frame = value as Record; + const dimensions = [frame.x, frame.y, frame.width, frame.height]; + if (!dimensions.every((dimension) => typeof dimension === 'number')) return undefined; + const [x, y, width, height] = dimensions as [number, number, number, number]; + return { x, y, width, height }; +} + +function readPointInspectionStrings( + record: Record, +): Partial> { + const strings: Record = {}; + for (const key of ['text', 'label', 'identifier', 'type', 'role', 'value'] as const) { + const value = record[key]; + if (typeof value === 'string' && value !== '') strings[key] = value; + } + return strings; +} diff --git a/packages/platform-apple/src/runner-demand.ts b/packages/platform-apple/src/runner-demand.ts index e37e380376..b69c1bceb3 100644 --- a/packages/platform-apple/src/runner-demand.ts +++ b/packages/platform-apple/src/runner-demand.ts @@ -58,6 +58,7 @@ const APPLE_SIMULATOR_OPERATION_HOSTS: Readonly< captureScreenshot: 'simulator', findText: 'runner', readTextAtPoint: 'runner', + inspectPoint: 'runner', // Every interaction and runner-driven capture. tapPoint: 'runner', tapRef: 'runner', diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index fdee94259d..94ca13835e 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -90,6 +90,10 @@ function expectAppStateFact( expect(binding.operations.appState).toBeTypeOf('function'); } +function expectsPointInspection(device: DeviceInfo): boolean { + return device.appleOs === 'ios' && device.kind === 'simulator'; +} + test.each([ ['iOS simulator', leaves.ios, true, undefined], [ @@ -131,6 +135,7 @@ test.each([ // available across it (parity with the retired buckets). expect(facts.operations.focusPoint).toEqual({ available: true }); expect(facts.operations.typeText).toEqual({ available: true }); + expect(facts.operations.inspectPoint.available).toBe(expectsPointInspection(device)); expect(binding.operations.focusPoint).toBeTypeOf('function'); expect(binding.operations.typeText).toBeTypeOf('function'); for (const operation of ['appLogInspect', 'appLogDoctor', 'appLogStart'] as const) { diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 3907f76689..02ff5cf778 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -21,6 +21,11 @@ import { bindElementTextRuntime, elementTextRuntimeOperationFacts, } from '@agent-device/contracts/element-text-runtime'; +import { + bindPointInspectionRuntime, + pointInspectionRuntimeOperationFacts, + type PointInspectionRuntimeOperations, +} from '@agent-device/contracts/point-inspection-runtime'; import { bindLocalFocusInteractor, focusRuntimeOperationFacts, @@ -322,6 +327,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR ...(isIosFamily(device) ? { tapElementSelector: appleFocusFact(device) } : {}), }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: appleElementTextFact(device) }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: applePointInspectionFact(device) }), ...appleNavigationFacts(device), ...appleFoldableFacts(device), ...appleSystemFacts(device), @@ -357,8 +363,18 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR resolveInteractor: host.localInteractors.resolve, }), ); + const pointInspectionOperations: Partial = whenAdmitted( + facts.operations.inspectPoint, + () => + bindPointInspectionRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ); const operations: DeviceBinding['operations'] = { ...appStateOperations, + ...pointInspectionOperations, ...logs.operations, ...createAppleAppDeploymentOperations({ host, @@ -547,6 +563,12 @@ function appleElementTextFact(device: DeviceInfo) { : elementTextKindUnavailable; } +function applePointInspectionFact(device: DeviceInfo): RuntimeOperationFact { + return resolveDeviceAppleOs(device) === 'ios' && device.kind === 'simulator' + ? available + : unavailable; +} + function appleSnapshotFact(device: DeviceInfo) { if (resolveDeviceAppleOs(device) === 'watchos') return snapshotKindUnavailable; return device.kind === 'simulator' || device.kind === 'device' diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index 68036c3e61..ad52844513 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -10,6 +10,7 @@ import { } from '@agent-device/contracts/application-lifecycle-runtime'; import { backRuntimeOperationFacts } from '@agent-device/contracts/back-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { bindLocalFocusInteractor, focusRuntimeOperationFacts, @@ -270,6 +271,7 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor // HarmonyOS has no point-read tool: `get` answers from the captured tree, which is what // the legacy dispatch already did after its Apple-runner attempt failed. ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextUnavailable }), ...backRuntimeOperationFacts({ back: harmonyFocusFact(device) }), // Home and the app switcher ride the same HDC-driven key input; no other system button // has a HarmonyOS control for HDC to press. diff --git a/packages/platform-linux/src/runtime.test.ts b/packages/platform-linux/src/runtime.test.ts index 9da74e6353..cc1e3ac3b0 100644 --- a/packages/platform-linux/src/runtime.test.ts +++ b/packages/platform-linux/src/runtime.test.ts @@ -108,6 +108,7 @@ test.each([ // The Linux read is value-first where the captured tree is label-first, so the desktop row // genuinely reads differently from its snapshot text and advertises the live read. expect(binding.facts.operations.readTextAtPoint.available).toBe(device.kind === 'device'); + expect(binding.facts.operations.inspectPoint.available).toBe(false); expect(binding.facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(binding.facts.operations.captureSnapshotWithoutActiveApp.available).toBe( device.kind === 'device', diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 2d17a9e940..6523ce1417 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -15,6 +15,7 @@ import { } from '@agent-device/contracts/application-lifecycle-runtime'; import { backRuntimeOperationFacts } from '@agent-device/contracts/back-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { focusRuntimeOperationFacts } from '@agent-device/contracts/focus-runtime'; import { TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT } from '@agent-device/contracts/gesture-admission'; import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; @@ -261,6 +262,7 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts ...elementTextRuntimeOperationFacts({ readTextAtPoint: linuxDesktopFact(device, elementTextKindUnavailable), }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextKindUnavailable }), // Parity with the retired `back`/`home` capability bucket (`{ device: true }`): the desktop // is the only Linux cell with a target to drive. ...backRuntimeOperationFacts({ back: linuxDesktopFact(device, backKindUnavailable) }), diff --git a/packages/platform-web/src/runtime.test.ts b/packages/platform-web/src/runtime.test.ts index 5ba868a300..565b4b8021 100644 --- a/packages/platform-web/src/runtime.test.ts +++ b/packages/platform-web/src/runtime.test.ts @@ -60,6 +60,7 @@ test('preserves a narrow web provider dump including empty successful entries', // No point-addressed read on the web backend: `get` answers from the captured DOM tree. The // legacy `read` dispatch had no web arm at all and threw on every call before falling back. expect(binding.facts.operations.readTextAtPoint.available).toBe(false); + expect(binding.facts.operations.inspectPoint.available).toBe(false); expect(binding.operations.readTextAtPoint).toBeUndefined(); // R40/R41: the browser device drives touch and text through the one web interactor. expect(binding.facts.operations.focusPoint).toEqual({ available: true }); diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index 883ac73069..6c0a2c2d97 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -16,6 +16,7 @@ import { availableApplicationLifecycleOperations, } from '@agent-device/contracts/application-lifecycle-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { bindLocalFocusInteractor, focusRuntimeOperationFacts, @@ -417,6 +418,7 @@ function webRuntimeFacts( // The web backend has no point-addressed read: `get` answers from the captured DOM tree, // which is what the legacy dispatch already did once its Apple-runner attempt failed. ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextUnavailable }), ...backRuntimeOperationFacts({ back: navigationUnavailable }), ...orientationRuntimeOperationFacts({ orientation: navigationUnavailable }), ...tvRemoteRuntimeOperationFacts({ tvRemote: navigationUnavailable }), diff --git a/packages/provider-limrun/src/facts-runtime.ts b/packages/provider-limrun/src/facts-runtime.ts index a68d22c991..6700736098 100644 --- a/packages/provider-limrun/src/facts-runtime.ts +++ b/packages/provider-limrun/src/facts-runtime.ts @@ -1,6 +1,7 @@ import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { applicationLifecycleOperationFacts } from '@agent-device/contracts/application-lifecycle-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform-runtime-operations'; import type { RuntimeFacts } from '@agent-device/contracts/platform-runtime'; import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; @@ -191,6 +192,7 @@ export function limrunAppLogFacts( // device always has one, so it is available wherever a capture is. ...limrunInteractionOperationFacts(device), ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextUnavailable }), ...limrunNavigationOperationFacts(device), ...limrunKeyboardOperationFacts(device), ...limrunClipboardOperationFacts(device), diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 3a1be8a0b8..1c39f215e0 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -148,6 +148,8 @@ function summarizeProviderScenarioFlagCoverage(files) { ['retainPaths', 'retained install-source materialization'], ['retentionMs', 'install-source materialization TTL'], ['count', 'repeated press/click/swipe input'], + ['pointX', 'point inspection horizontal coordinate'], + ['pointY', 'point inspection vertical coordinate'], ['pointerCount', 'one- vs two-pointer pan gesture topology'], ['fps', 'recording frame-rate request'], ['quality', 'recording quality scaling'], diff --git a/scripts/layering/contracts-exports.snapshot.json b/scripts/layering/contracts-exports.snapshot.json index 3095a7699e..f1e08380a5 100644 --- a/scripts/layering/contracts-exports.snapshot.json +++ b/scripts/layering/contracts-exports.snapshot.json @@ -85,6 +85,7 @@ "@agent-device/contracts/platform-runtime-host", "@agent-device/contracts/platform-runtime-operations", "@agent-device/contracts/platform-runtime-unavailable", + "@agent-device/contracts/point-inspection-runtime", "@agent-device/contracts/progress", "@agent-device/contracts/react-native-overlay", "@agent-device/contracts/record-runtime-execution", diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index b3382c6ba9..6e8b4efb63 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -1251,6 +1251,7 @@ function createStubClient(params: { })), }, capture: { + inspectPoint: unexpectedCommandCall, snapshot: async () => ({ nodes: [], truncated: false, diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index 4767f11f81..3cdfcf64f1 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -301,6 +301,7 @@ const REPLAY_SCRIPT_LINE_PLANS = { focus: GENERIC_REPLAY_LINE, home: GENERIC_REPLAY_LINE, install: GENERIC_REPLAY_LINE, + 'inspect-point': GENERIC_REPLAY_LINE, 'install-from-source': GENERIC_REPLAY_LINE, is: GENERIC_REPLAY_LINE, keyboard: GENERIC_REPLAY_LINE, diff --git a/src/__tests__/test-utils/runtime-operation-facts.ts b/src/__tests__/test-utils/runtime-operation-facts.ts index 350630632d..e070ed2aaf 100644 --- a/src/__tests__/test-utils/runtime-operation-facts.ts +++ b/src/__tests__/test-utils/runtime-operation-facts.ts @@ -2,6 +2,7 @@ import { applicationLifecycleOperationFacts } from '@agent-device/contracts/appl import { audioProbeRuntimeOperationFacts } from '@agent-device/contracts/audio-probe-runtime'; import { clipboardRuntimeOperationFacts } from '@agent-device/contracts/clipboard-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; import { keyboardRuntimeOperationFacts } from '@agent-device/contracts/keyboard-runtime'; import type { @@ -60,6 +61,7 @@ export const unavailableDeploymentSnapshotAndShutdownOperationFacts = Object.fre ...gestureRuntimeOperationFacts({ unsupported: unavailable }), ...scrollRuntimeOperationFacts({ scroll: unavailable }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: unavailable }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: unavailable }), back: unavailable, setOrientation: unavailable, tvRemote: unavailable, diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index e616dabbc2..1e2fef821f 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -16,6 +16,7 @@ import type { CaptureScreenshotResult, CaptureSnapshotOptions, CaptureSnapshotResult, + PointInspectionResult, DragOptions, FlingOptions, MaterializationReleaseOptions, @@ -356,6 +357,8 @@ export function createAgentDeviceClient( }), }, capture: { + inspectPoint: async (options) => + await executeCommand('inspect-point', options), snapshot: async (options: CaptureSnapshotOptions = {}) => { const session = resolveRequestSession(options); const data = await executeCommand>('snapshot', options); diff --git a/src/client/client-types.ts b/src/client/client-types.ts index 25b493df43..d266306584 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -33,18 +33,12 @@ export type { // these up via `export type *`, and that is their only job — every internal consumer imports // them from the declaring module instead. Fallow therefore sees no consumer, which is exactly // right and exactly not actionable: deleting them would remove names from the package's public -// types. Suppressed per name rather than baselined so the reason travels with the code. -// fallow-ignore-next-line unused-type +// types. export type { TargetShutdownResult } from '@agent-device/contracts/device'; -// fallow-ignore-next-line unused-type export type { MetroBridgeScope } from '@agent-device/contracts/remote'; -// fallow-ignore-next-line unused-type export type { AppsFilter } from '@agent-device/contracts/device'; -// fallow-ignore-next-line unused-type export type { AlertAction } from '@agent-device/contracts/alert-contract'; -// fallow-ignore-next-line unused-type export type { AppleOS } from '@agent-device/kernel/device'; -// fallow-ignore-next-line unused-type export type { JsonObject } from '@agent-device/contracts/client'; export type { BatchRunResult } from '@agent-device/command-registry/batch'; @@ -80,6 +74,8 @@ import type { CaptureScreenshotResult, CaptureSnapshotOptions, CaptureSnapshotResult, + InspectPointOptions, + PointInspectionResult, ClickOptions, ClipboardCommandOptions, CloudArtifactsOptions, @@ -250,6 +246,7 @@ export type AgentDeviceClient = { }; capture: { snapshot: (options?: CaptureSnapshotOptions) => Promise; + inspectPoint: (options: InspectPointOptions) => Promise; screenshot: (options?: CaptureScreenshotOptions) => Promise; diff: (options: CaptureDiffOptions) => Promise>; }; diff --git a/src/commands/capture/index.test.ts b/src/commands/capture/index.test.ts index 03e381e5d1..814ab32be4 100644 --- a/src/commands/capture/index.test.ts +++ b/src/commands/capture/index.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'vitest'; import type { CliFlags } from '@agent-device/contracts/command'; import { alertCliReader, alertDaemonWriter } from './alert.ts'; import { diffCliReader } from './diff.ts'; +import { inspectPointCliReader, inspectPointDaemonWriter } from './inspect-point.ts'; import { snapshotCliOutput } from './output.ts'; import { screenshotCliReader, screenshotDaemonWriter } from './screenshot.ts'; import { snapshotCliReader } from './snapshot.ts'; @@ -21,6 +22,19 @@ function expectInvalidArgs(fn: () => unknown, messageFragment: string) { } describe('capture command interface', () => { + test('reads inspect-point coordinate flags and writes its daemon request', () => { + const input = inspectPointCliReader([], flags({ pointX: 14.5, pointY: 29 })); + expect(input).toMatchObject({ point: { x: 14.5, y: 29 } }); + expect(inspectPointDaemonWriter(input)).toMatchObject({ + command: 'inspect-point', + positionals: ['14.5', '29'], + }); + }); + + test('rejects inspect-point when either coordinate is missing', () => { + expectInvalidArgs(() => inspectPointCliReader([], flags({ pointX: 1 })), '--x and --y'); + }); + test('reads snapshot flags', () => { expect( snapshotCliReader( diff --git a/src/commands/capture/index.ts b/src/commands/capture/index.ts index c908be42f6..7db868c2b3 100644 --- a/src/commands/capture/index.ts +++ b/src/commands/capture/index.ts @@ -4,10 +4,12 @@ import { diffCommandFacet } from './diff.ts'; import { screenshotCommandFacet } from './screenshot.ts'; import { settingsCommandFacet } from './settings.ts'; import { snapshotCommandFacet } from './snapshot.ts'; +import { inspectPointCommandFacet } from './inspect-point.ts'; import { waitCommandFacet } from './wait.ts'; const captureCommandFacets = [ snapshotCommandFacet, + inspectPointCommandFacet, screenshotCommandFacet, diffCommandFacet, waitCommandFacet, diff --git a/src/commands/capture/inspect-point.ts b/src/commands/capture/inspect-point.ts new file mode 100644 index 0000000000..91d75ddc38 --- /dev/null +++ b/src/commands/capture/inspect-point.ts @@ -0,0 +1,44 @@ +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { InspectPointOptions, PointInspectionResult } from '@agent-device/contracts/client'; +import { AppError } from '@agent-device/kernel/errors'; +import { commonInputFromFlags, direct } from '../cli-grammar/common.ts'; +import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; +import { defineCommandFacet } from '../family/types.ts'; +import { defineFieldCommandMetadata } from '../field-command-contract.ts'; +import { pointField, requiredField } from '../command-input.ts'; +import { resultOutput } from '../output-common.ts'; + +const metadata = defineFieldCommandMetadata( + 'inspect-point', + 'Inspect the live accessibility elements containing one screen coordinate.', + { point: requiredField(pointField('Screen coordinate to inspect.')) }, +); + +export const inspectPointCliReader: CliReader = (_positionals, flags) => { + if (typeof flags.pointX !== 'number' || typeof flags.pointY !== 'number') { + throw new AppError('INVALID_ARGS', 'inspect-point requires --x and --y'); + } + return { ...commonInputFromFlags(flags), point: { x: flags.pointX, y: flags.pointY } }; +}; + +export const inspectPointDaemonWriter: DaemonWriter = direct( + PUBLIC_COMMANDS['inspect-point'], + (input) => { + const { point } = input as InspectPointOptions; + return [String(point.x), String(point.y)]; + }, +); + +export const inspectPointCommandFacet = defineCommandFacet({ + name: 'inspect-point', + text: { summary: 'Inspect accessibility elements at a screen coordinate' }, + metadata, + run: (client, input) => client.capture.inspectPoint(input), + cliSchema: { allowedFlags: ['pointX', 'pointY'] }, + cliReader: inspectPointCliReader, + daemonWriter: inspectPointDaemonWriter, + cliOutputFormatter: resultOutput((result: PointInspectionResult) => ({ + data: result, + text: JSON.stringify(result, null, 2), + })), +}); diff --git a/src/daemon/__tests__/runtime-binding-conformance.ts b/src/daemon/__tests__/runtime-binding-conformance.ts index bfc125365c..f6c4731db7 100644 --- a/src/daemon/__tests__/runtime-binding-conformance.ts +++ b/src/daemon/__tests__/runtime-binding-conformance.ts @@ -18,6 +18,7 @@ import { resolveBoundBackRuntime } from '../back-runtime.ts'; import { resolveBoundFocusRuntime } from '../focus-runtime.ts'; import { resolveBoundFoldRuntime } from '../fold-runtime.ts'; import { resolveBoundGestureRuntime } from '../gesture-runtime.ts'; +import { resolveBoundInspectPointRuntime } from '../inspect-point-runtime.ts'; import { resolveBoundOrientationRuntime } from '../orientation-runtime.ts'; import type { ResolvedGenericExecution } from '../request-generic-dispatch.ts'; import type { @@ -109,6 +110,16 @@ export const conformedRuntimeBindings = { resolve: async (device, bindings) => refusable(await resolveBoundFocusRuntime({ device, positionals: ['40', '90'], ...bindings })), }, + 'inspect-point': { + resolve: async (device, bindings) => + refusable( + await resolveBoundInspectPointRuntime({ + device, + positionals: ['40', '90'], + ...bindings, + }), + ), + }, viewport: { resolve: async (device, bindings) => refusable( diff --git a/src/daemon/generic-runtime-execution.ts b/src/daemon/generic-runtime-execution.ts index cdd440102f..455ecf717e 100644 --- a/src/daemon/generic-runtime-execution.ts +++ b/src/daemon/generic-runtime-execution.ts @@ -13,6 +13,7 @@ import { resolveBoundOrientationRuntime } from './orientation-runtime.ts'; import { resolveBoundFoldRuntime } from './fold-runtime.ts'; import { resolveBoundTvRemoteRuntime } from './tv-remote-runtime.ts'; import { errorResponse } from '@agent-device/kernel/contracts'; +import { resolveBoundInspectPointRuntime } from './inspect-point-runtime.ts'; /** * The generic route's runtime-owned leaves (ADR 0019). Each one admits its own exact owner facts @@ -41,6 +42,13 @@ export async function resolveGenericRuntimeExecution( }); } switch (params.req.command) { + case 'inspect-point': + return await resolveBoundInspectPointRuntime({ + device: params.session.device, + positionals: params.req.positionals ?? [], + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); case 'screenshot': return await resolveScreenshotGenericExecution(params); case 'focus': diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index 274a3f8848..b9e637c73f 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -374,6 +374,7 @@ function sourceRuntimeFacts( ...gestureRuntimeOperationFacts({ unsupported: unavailable }), ...scrollRuntimeOperationFacts({ scroll: unavailable }), readTextAtPoint: unavailable, + inspectPoint: unavailable, back: unavailable, setOrientation: unavailable, tvRemote: unavailable, diff --git a/src/daemon/inspect-point-runtime.ts b/src/daemon/inspect-point-runtime.ts new file mode 100644 index 0000000000..6675049db2 --- /dev/null +++ b/src/daemon/inspect-point-runtime.ts @@ -0,0 +1,46 @@ +import { inspectPointRuntimeUse } from '@agent-device/contracts/platform-runtime-operations'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import type { DaemonCommandContext } from './context.ts'; +import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; +import { resolveBoundGenericRuntime, type RuntimeAdmissionBindings } from './runtime-admission.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; + +function coordinate(value: string | undefined, name: string): number { + const parsed = value === undefined ? Number.NaN : Number(value); + if (!Number.isFinite(parsed)) { + throw new AppError('INVALID_ARGS', `inspect-point requires a finite ${name} coordinate`); + } + return parsed; +} + +export async function resolveBoundInspectPointRuntime( + params: { + device: DeviceInfo; + positionals: string[]; + } & RuntimeAdmissionBindings, +): Promise { + const point = { + x: coordinate(params.positionals[0], 'x'), + y: coordinate(params.positionals[1], 'y'), + }; + return await resolveBoundGenericRuntime( + { + command: 'inspect-point', + device: params.device, + use: inspectPointRuntimeUse, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }, + async (runtime, context: DaemonCommandContext) => { + const result = await runtime.operations.inspectPoint({ + point, + ...(context.appBundleId ? { options: { appBundleId: context.appBundleId } } : {}), + execution: runtimeExecutionFromContext(context), + }); + return result.elements.length === 0 + ? { status: 'no-element-at-point', point, elements: [] } + : { status: 'inspected', point, elements: result.elements }; + }, + ); +} diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 742a5835db..affa6d25a9 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -864,6 +864,50 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { ), ], }, + 'inspect-point': { + type: 'object', + oneOf: [ + objectSchema( + { + status: constSchema('inspected'), + point: objectSchema({ x: numberSchema(), y: numberSchema() }, ['x', 'y']), + elements: { + type: 'array', + items: objectSchema( + { + text: stringSchema(), + label: stringSchema(), + identifier: stringSchema(), + type: stringSchema(), + role: stringSchema(), + value: stringSchema(), + frame: objectSchema( + { + x: numberSchema(), + y: numberSchema(), + width: numberSchema(), + height: numberSchema(), + }, + ['x', 'y', 'width', 'height'], + ), + hittable: booleanSchema(), + }, + ['frame'], + ), + }, + }, + ['status', 'point', 'elements'], + ), + objectSchema( + { + status: constSchema('no-element-at-point'), + point: objectSchema({ x: numberSchema(), y: numberSchema() }, ['x', 'y']), + elements: { type: 'array', maxItems: 0 }, + }, + ['status', 'point', 'elements'], + ), + ], + }, // A family that owns its commands authors their advertised response shape beside the // command surface and projects it here. This spread stays last: a hand-written entry for diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index b3adbb26c4..f4228571fd 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -1388,6 +1388,37 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), linux: linux.gap('No Linux-specific source-install command evidence exists yet'), }, + [C['inspect-point']]: { + androidEmulator: androidEmulator.contract( + ANDROID_HOVER_RUNTIME_CONTRACT_EVIDENCE, + 'the Android runtime reports point inspection unavailable instead of dispatching an unsupported command', + ), + iosSimulator: iosSimulator.contract( + 'packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts', + 'inspectPoint', + 'the Apple runner returns ordered accessibility descriptors for a screen coordinate', + ), + macos: macos.contract( + 'packages/platform-apple/src/runtime.test.ts', + 'classifies the %s leaf explicitly', + 'the Apple runtime refuses point inspection outside iOS Simulator', + ), + tvos: tvos.contract( + 'packages/platform-apple/src/runtime.test.ts', + 'classifies the %s leaf explicitly', + 'the Apple runtime refuses point inspection outside iOS Simulator', + ), + web: web.contract( + 'packages/platform-web/src/runtime.test.ts', + 'preserves a narrow web provider dump including empty successful entries', + 'the web runtime reports point inspection unavailable', + ), + linux: linux.contract( + 'packages/platform-linux/src/runtime.test.ts', + 'classifies the Linux $name lifecycle denominator against the legacy dispatch cell', + 'the Linux runtime reports point inspection unavailable', + ), + }, } satisfies Record; /** diff --git a/test/integration/command-coverage/evidence.ts b/test/integration/command-coverage/evidence.ts index 42aed9865b..053e4ecb6f 100644 --- a/test/integration/command-coverage/evidence.ts +++ b/test/integration/command-coverage/evidence.ts @@ -29,7 +29,7 @@ export const ANDROID_APPLICATION_LIFECYCLE_CONTRACT_EVIDENCE: AndroidContractEvi export const ANDROID_HOVER_RUNTIME_CONTRACT_EVIDENCE: AndroidContractEvidence = defineAndroidContractEvidence( 'packages/platform-android/src/runtime.test.ts', - [C.hover], + [C.hover, C['inspect-point']], 'classifies the Android %s runtime denominator', ); export const ANDROID_TV_REMOTE_RUNTIME_CONTRACT_EVIDENCE: AndroidContractEvidence = diff --git a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts index 4131ca25f8..76eaf8b96e 100644 --- a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts +++ b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts @@ -142,13 +142,14 @@ const DRIVEN_COMMANDS: Record = { // --------------------------------------------------------------------------- // SKIPPED_COMMANDS — catalog commands that genuinely cannot be driven against the -// fake-provider harness, each with a reason. Intentionally EMPTY: every public -// command is driveable here (an orchestrator with no real workload simply returns a -// fast, still-scanned error response). Kept as an explicit, enforced set so a future -// undriveable command has a home and the partition test keeps a new command from -// escaping the guard silently. +// fake-provider harness, each with a reason. Kept as an explicit, enforced set so a future +// undriveable command has a home and the partition test keeps a new command from escaping the +// guard silently. // --------------------------------------------------------------------------- -const SKIPPED_COMMANDS: Record = {}; +const SKIPPED_COMMANDS: Record = { + [PUBLIC_COMMANDS['inspect-point']]: + 'requires a local iOS Simulator XCTest runner; provider worlds do not expose that runtime', +}; // Commands driven last so the priority commands run against a live session. const DRIVE_LAST = new Set([PUBLIC_COMMANDS.shutdown, PUBLIC_COMMANDS.close]); diff --git a/test/integration/provider-scenarios/ios-lifecycle.test.ts b/test/integration/provider-scenarios/ios-lifecycle.test.ts index 2becd3f0c5..65ef25c939 100644 --- a/test/integration/provider-scenarios/ios-lifecycle.test.ts +++ b/test/integration/provider-scenarios/ios-lifecycle.test.ts @@ -70,6 +70,28 @@ test('Provider-backed integration iOS Settings flow uses scripted simctl and run assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.ref, 'e1'); }, }, + { + name: 'inspect the accessibility elements at a settings coordinate', + command: 'inspect-point', + positionals: ['196', '122'], + flags: { pointX: 196, pointY: 122 }, + expectData: { + status: 'inspected', + point: { x: 196, y: 122 }, + elements: [ + { + text: 'General', + label: 'General', + identifier: 'General', + type: 'Button', + role: 'button', + value: 'General', + frame: { x: 146, y: 102, width: 100, height: 40 }, + hittable: true, + }, + ], + }, + }, { name: 'reopen existing session app', command: 'open', diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index a012acebb2..1f01207043 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -40,6 +40,32 @@ export async function createIosSettingsWorld(): Promise { result: { uptimeMs: 42 }, }, runnerSnapshot(), + { + command: 'ios.runner.readText', + deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + platform: 'apple', + request: { + command: 'readText', + x: 196, + y: 122, + appBundleId: 'com.apple.Preferences', + }, + result: { + text: 'General', + elements: [ + { + text: 'General', + label: 'General', + identifier: 'General', + type: 'Button', + role: 'button', + value: 'General', + frame: { x: 146, y: 102, width: 100, height: 40 }, + hittable: true, + }, + ], + }, + }, { command: 'ios.runner.tap', deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 2b4ad186af..3c46e69e76 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -371,10 +371,16 @@ agent-device close ```bash agent-device snapshot [--diff] [-i] [--depth, -d ] [--scope, -s ] [--raw] [--actions] [--force-full] [--timeout ] agent-device diff snapshot [-i] [-d ] [-s ] [--raw] +agent-device inspect-point --x --y agent-device get text @e1 agent-device get attrs @e1 ``` +- `inspect-point` is available on iOS Simulator sessions. It returns a bounded list of the + accessibility elements containing the coordinate, ordered from the smallest frame to the largest. + An honest miss returns `no-element-at-point`; transport and runner failures still fail the command. + Inspection identifies a candidate target but is not itself a behavioral assertion. + - iOS snapshots use XCTest on simulators and physical devices. iOS `--raw` is the acquired tree on whichever backend serves the capture: it keeps offscreen nodes, decorations, and structural wrappers the default and `-i` views fold away, so a recovered raw capture shows the same hierarchy From 1d9ecdadc8825d4b7b3445b0dbdfb71f2d4596e4 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Sun, 27 Sep 2026 15:11:08 -0700 Subject: [PATCH 02/21] refactor: keep new commands within eager closure budgets --- .../src/platform-runtime-operations.ts | 6 +- .../src/platform-runtime-unavailable.ts | 6 +- .../contracts/src/point-inspection-runtime.ts | 61 +++---------------- .../src/selector-observation-runtime.ts | 52 ++++++++++++++++ src/__tests__/cli-client-commands.test.ts | 7 +-- src/commands/capture/index.ts | 3 +- src/commands/capture/inspect-point.ts | 49 ++------------- src/commands/capture/snapshot.ts | 47 +++++++++++++- 8 files changed, 121 insertions(+), 110 deletions(-) diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index ff3de4b265..9d364cd6b3 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -11,7 +11,10 @@ import type { ScreenRecordingRuntimeHost } from './screen-recording-runtime-host import type { ScreenRecordingRuntimeOperations } from './screen-recording-runtime.ts'; import type { ScreenshotRuntimeOperations } from './screenshot-runtime.ts'; import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot-runtime.ts'; -import type { SelectorObservationRuntimeOperations } from './selector-observation-runtime.ts'; +import type { + PointInspectionRuntimeOperations, + SelectorObservationRuntimeOperations, +} from './selector-observation-runtime.ts'; import type { ViewportRuntimeOperations } from './viewport-runtime.ts'; import type { FocusRuntimeOperations } from './focus-runtime.ts'; import type { GestureCommandInput, GestureSemanticInput } from './gesture-plan-types.ts'; @@ -20,7 +23,6 @@ import type { GestureRuntimeOperations } from './gesture-runtime.ts'; import type { ScrollRuntimeOperations } from './scroll-runtime.ts'; import type { TypeTextRuntimeOperations } from './type-text-runtime.ts'; import type { ElementTextRuntimeOperations } from './element-text-runtime.ts'; -import type { PointInspectionRuntimeOperations } from './point-inspection-runtime.ts'; import type { BackRuntimeOperations } from './back-runtime.ts'; import type { OrientationRuntimeOperations } from './orientation-runtime.ts'; import type { FoldRuntimeOperations } from './fold-runtime.ts'; diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index 9a6acfb8ec..6064c769a6 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -13,14 +13,16 @@ import type { } from './platform-runtime.ts'; import { screenshotRuntimeOperationFacts } from './screenshot-runtime.ts'; import { snapshotRuntimeOperationFacts } from './snapshot-runtime.ts'; -import { selectorObservationRuntimeOperationFacts } from './selector-observation-runtime.ts'; +import { + pointInspectionRuntimeOperationFacts, + selectorObservationRuntimeOperationFacts, +} from './selector-observation-runtime.ts'; import { viewportRuntimeOperationFacts } from './viewport-runtime.ts'; import { focusRuntimeOperationFacts } from './focus-runtime.ts'; import { gestureRuntimeOperationFacts } from './gesture-runtime.ts'; import { scrollRuntimeOperationFacts } from './scroll-runtime.ts'; import { typeTextRuntimeOperationFacts } from './type-text-runtime.ts'; import { elementTextRuntimeOperationFacts } from './element-text-runtime.ts'; -import { pointInspectionRuntimeOperationFacts } from './point-inspection-runtime.ts'; import { backRuntimeOperationFacts } from './back-runtime.ts'; import { orientationRuntimeOperationFacts } from './orientation-runtime.ts'; import { tvRemoteRuntimeOperationFacts } from './tv-remote-runtime.ts'; diff --git a/packages/contracts/src/point-inspection-runtime.ts b/packages/contracts/src/point-inspection-runtime.ts index 1444567d67..504668fb50 100644 --- a/packages/contracts/src/point-inspection-runtime.ts +++ b/packages/contracts/src/point-inspection-runtime.ts @@ -1,54 +1,7 @@ -import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { Point } from '@agent-device/kernel/snapshot'; -import type { Interactor, PointInspectionRead, RunnerContext } from './interactor-types.ts'; -import { invalidRuntimeContract } from './runtime-contract-error.ts'; -import type { RuntimeOperationFact } from './platform-runtime.ts'; -import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; - -export type InspectPointRuntimeInput = Readonly<{ - point: Point; - options?: Readonly<{ appBundleId?: string }>; - execution?: SnapshotRuntimeExecution; -}>; - -export type PointInspectionRuntimeOperations = Readonly<{ - inspectPoint(input: InspectPointRuntimeInput): Promise; -}>; - -export type PointInspectionRuntimeOperationFacts = Readonly<{ - inspectPoint: RuntimeOperationFact; -}>; - -export function pointInspectionRuntimeOperationFacts( - input: PointInspectionRuntimeOperationFacts, -): PointInspectionRuntimeOperationFacts { - return Object.freeze({ inspectPoint: input.inspectPoint }); -} - -export function bindPointInspectionRuntime( - params: Readonly<{ - device: DeviceInfo; - signal: AbortSignal; - resolveInteractor: (device: DeviceInfo, runner: RunnerContext) => Promise; - }>, -): PointInspectionRuntimeOperations { - return Object.freeze({ - inspectPoint: async (input) => { - params.signal.throwIfAborted(); - const interactor = await params.resolveInteractor(params.device, { - ...input.execution, - appBundleId: input.options?.appBundleId, - signal: params.signal, - }); - if (typeof interactor.inspectPoint !== 'function') { - throw invalidRuntimeContract( - 'Runtime owner advertised inspectPoint without an interactor implementation', - ); - } - return await interactor.inspectPoint(input.point, { - appBundleId: input.options?.appBundleId, - signal: params.signal, - }); - }, - }); -} +export { + bindPointInspectionRuntime, + pointInspectionRuntimeOperationFacts, + type InspectPointRuntimeInput, + type PointInspectionRuntimeOperationFacts, + type PointInspectionRuntimeOperations, +} from './selector-observation-runtime.ts'; diff --git a/packages/contracts/src/selector-observation-runtime.ts b/packages/contracts/src/selector-observation-runtime.ts index 318d092bda..45ddcb3152 100644 --- a/packages/contracts/src/selector-observation-runtime.ts +++ b/packages/contracts/src/selector-observation-runtime.ts @@ -1,3 +1,6 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Point } from '@agent-device/kernel/snapshot'; +import type { Interactor, PointInspectionRead, RunnerContext } from './interactor-types.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; import type { SessionSurface } from './session-surface.ts'; import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; @@ -33,3 +36,52 @@ export function selectorObservationRuntimeOperationFacts( ): SelectorObservationRuntimeOperationFacts { return Object.freeze({ findText: input.findText }); } + +export type InspectPointRuntimeInput = Readonly<{ + point: Point; + options?: Readonly<{ appBundleId?: string }>; + execution?: SnapshotRuntimeExecution; +}>; + +export type PointInspectionRuntimeOperations = Readonly<{ + inspectPoint(input: InspectPointRuntimeInput): Promise; +}>; + +export type PointInspectionRuntimeOperationFacts = Readonly<{ + inspectPoint: RuntimeOperationFact; +}>; + +export function pointInspectionRuntimeOperationFacts( + input: PointInspectionRuntimeOperationFacts, +): PointInspectionRuntimeOperationFacts { + return Object.freeze({ inspectPoint: input.inspectPoint }); +} + +export function bindPointInspectionRuntime( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: (device: DeviceInfo, runner: RunnerContext) => Promise; + }>, +): PointInspectionRuntimeOperations { + return Object.freeze({ + inspectPoint: async (input) => { + params.signal.throwIfAborted(); + const interactor = await params.resolveInteractor(params.device, { + ...input.execution, + appBundleId: input.options?.appBundleId, + signal: params.signal, + }); + if (typeof interactor.inspectPoint !== 'function') { + const { invalidRuntimeContract } = await import('./runtime-contract-error.ts'); + throw invalidRuntimeContract( + 'Runtime owner advertised inspectPoint without an interactor implementation', + ); + } + return await interactor.inspectPoint(input.point, { + appBundleId: input.options?.appBundleId, + signal: params.signal, + }); + }, + }); +} diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index 6e8b4efb63..0117b1b392 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -1168,10 +1168,8 @@ function createStubClient(params: { return { command, devices: { + ...createThrowingMethodGroup(), list: async () => [], - capabilities: unexpectedCommandCall, - boot: unexpectedCommandCall, - shutdown: unexpectedCommandCall, }, sessions: { list: async () => [], @@ -1251,7 +1249,7 @@ function createStubClient(params: { })), }, capture: { - inspectPoint: unexpectedCommandCall, + ...createThrowingMethodGroup(), snapshot: async () => ({ nodes: [], truncated: false, @@ -1263,7 +1261,6 @@ function createStubClient(params: { path: '/tmp/screenshot.png', identifiers: { session: 'default' }, })), - diff: unexpectedCommandCall, }, interactions: createThrowingMethodGroup(), replay: createThrowingMethodGroup(), diff --git a/src/commands/capture/index.ts b/src/commands/capture/index.ts index 7db868c2b3..219bdeffbd 100644 --- a/src/commands/capture/index.ts +++ b/src/commands/capture/index.ts @@ -3,8 +3,7 @@ import { alertCommandFacet } from './alert.ts'; import { diffCommandFacet } from './diff.ts'; import { screenshotCommandFacet } from './screenshot.ts'; import { settingsCommandFacet } from './settings.ts'; -import { snapshotCommandFacet } from './snapshot.ts'; -import { inspectPointCommandFacet } from './inspect-point.ts'; +import { inspectPointCommandFacet, snapshotCommandFacet } from './snapshot.ts'; import { waitCommandFacet } from './wait.ts'; const captureCommandFacets = [ diff --git a/src/commands/capture/inspect-point.ts b/src/commands/capture/inspect-point.ts index 91d75ddc38..c89670ec84 100644 --- a/src/commands/capture/inspect-point.ts +++ b/src/commands/capture/inspect-point.ts @@ -1,44 +1,5 @@ -import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import type { InspectPointOptions, PointInspectionResult } from '@agent-device/contracts/client'; -import { AppError } from '@agent-device/kernel/errors'; -import { commonInputFromFlags, direct } from '../cli-grammar/common.ts'; -import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; -import { defineCommandFacet } from '../family/types.ts'; -import { defineFieldCommandMetadata } from '../field-command-contract.ts'; -import { pointField, requiredField } from '../command-input.ts'; -import { resultOutput } from '../output-common.ts'; - -const metadata = defineFieldCommandMetadata( - 'inspect-point', - 'Inspect the live accessibility elements containing one screen coordinate.', - { point: requiredField(pointField('Screen coordinate to inspect.')) }, -); - -export const inspectPointCliReader: CliReader = (_positionals, flags) => { - if (typeof flags.pointX !== 'number' || typeof flags.pointY !== 'number') { - throw new AppError('INVALID_ARGS', 'inspect-point requires --x and --y'); - } - return { ...commonInputFromFlags(flags), point: { x: flags.pointX, y: flags.pointY } }; -}; - -export const inspectPointDaemonWriter: DaemonWriter = direct( - PUBLIC_COMMANDS['inspect-point'], - (input) => { - const { point } = input as InspectPointOptions; - return [String(point.x), String(point.y)]; - }, -); - -export const inspectPointCommandFacet = defineCommandFacet({ - name: 'inspect-point', - text: { summary: 'Inspect accessibility elements at a screen coordinate' }, - metadata, - run: (client, input) => client.capture.inspectPoint(input), - cliSchema: { allowedFlags: ['pointX', 'pointY'] }, - cliReader: inspectPointCliReader, - daemonWriter: inspectPointDaemonWriter, - cliOutputFormatter: resultOutput((result: PointInspectionResult) => ({ - data: result, - text: JSON.stringify(result, null, 2), - })), -}); +export { + inspectPointCliReader, + inspectPointCommandFacet, + inspectPointDaemonWriter, +} from './snapshot.ts'; diff --git a/src/commands/capture/snapshot.ts b/src/commands/capture/snapshot.ts index 093a9c2056..4b680a357e 100644 --- a/src/commands/capture/snapshot.ts +++ b/src/commands/capture/snapshot.ts @@ -1,11 +1,20 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { InspectPointOptions, PointInspectionResult } from '@agent-device/contracts/client'; import { SNAPSHOT_BACKEND_CAPABILITIES } from '@agent-device/capture-kit/snapshot-quality-backend-capabilities'; import { SNAPSHOT_COMMAND_OPTION_KEYS, snapshotOptionsFromFlags, } from '@agent-device/kernel/snapshot'; import { SNAPSHOT_FLAGS } from '@agent-device/command-registry/flag-groups'; -import { booleanField, integerField, optionField, stringField } from '../command-input.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { + booleanField, + integerField, + optionField, + pointField, + requiredField, + stringField, +} from '../command-input.ts'; import { commonInputFromFlags, direct, @@ -15,6 +24,7 @@ import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; import { defineCommandFacet } from '../family/types.ts'; import { defineFieldCommandMetadata } from '../field-command-contract.ts'; import { captureCliOutputFormatters } from './output.ts'; +import { resultOutput } from '../output-common.ts'; const SNAPSHOT_COMMAND_NAME = 'snapshot'; @@ -83,3 +93,38 @@ export const snapshotCommandFacet = defineCommandFacet({ daemonWriter: snapshotDaemonWriter, cliOutputFormatter: captureCliOutputFormatters.snapshot, }); + +const inspectPointMetadata = defineFieldCommandMetadata( + 'inspect-point', + 'Inspect the live accessibility elements containing one screen coordinate.', + { point: requiredField(pointField('Screen coordinate to inspect.')) }, +); + +export const inspectPointCliReader: CliReader = (_positionals, flags) => { + if (typeof flags.pointX !== 'number' || typeof flags.pointY !== 'number') { + throw new AppError('INVALID_ARGS', 'inspect-point requires --x and --y'); + } + return { ...commonInputFromFlags(flags), point: { x: flags.pointX, y: flags.pointY } }; +}; + +export const inspectPointDaemonWriter: DaemonWriter = direct( + PUBLIC_COMMANDS['inspect-point'], + (input) => { + const { point } = input as InspectPointOptions; + return [String(point.x), String(point.y)]; + }, +); + +export const inspectPointCommandFacet = defineCommandFacet({ + name: 'inspect-point', + text: { summary: 'Inspect accessibility elements at a screen coordinate' }, + metadata: inspectPointMetadata, + run: (client, input) => client.capture.inspectPoint(input), + cliSchema: { allowedFlags: ['pointX', 'pointY'] }, + cliReader: inspectPointCliReader, + daemonWriter: inspectPointDaemonWriter, + cliOutputFormatter: resultOutput((result: PointInspectionResult) => ({ + data: result, + text: JSON.stringify(result, null, 2), + })), +}); From 471df4ac21a17fa34995abbe39fa3ad03898dea4 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Sun, 27 Sep 2026 15:13:40 -0700 Subject: [PATCH 03/21] docs: keep workflow help compact --- src/commands/capture/snapshot.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/capture/snapshot.ts b/src/commands/capture/snapshot.ts index 4b680a357e..1cfaab7170 100644 --- a/src/commands/capture/snapshot.ts +++ b/src/commands/capture/snapshot.ts @@ -117,7 +117,7 @@ export const inspectPointDaemonWriter: DaemonWriter = direct( export const inspectPointCommandFacet = defineCommandFacet({ name: 'inspect-point', - text: { summary: 'Inspect accessibility elements at a screen coordinate' }, + text: { summary: 'Inspect elements at a screen coordinate' }, metadata: inspectPointMetadata, run: (client, input) => client.capture.inspectPoint(input), cliSchema: { allowedFlags: ['pointX', 'pointY'] }, From 562d47193ed740c899a47e340f9dcfdad5a9dfe2 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 10:27:52 -0700 Subject: [PATCH 04/21] fix(apple): bound point inspection on system UI --- .../RunnerTests+Interaction.swift | 105 ++++++++++++++++++ .../RunnerTests+InteractionTests.swift | 45 ++++++++ 2 files changed, 150 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 1cadd0aa4a..56d4e53215 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -326,6 +326,15 @@ extension RunnerTests { x: Double, y: Double ) -> (text: String?, elements: [PointInspectionElementPayload]) { +#if os(iOS) && targetEnvironment(simulator) + // System-owned sheets can make XCTest's unbounded descendants query hold + // the main thread past the command watchdog. The Simulator-only private AX + // bridge already provides a deadline-bounded tree for snapshot recovery; + // filter that tree at the requested point before touching the XCTest query. + if let inspection = privateAXPointInspection(app: app, x: x, y: y) { + return inspection + } +#endif let point = CGPoint(x: x, y: y) let textInputCandidates = textInputCandidatesAt(app: app, point: point) let candidates = app.descendants(matching: .any).allElementsBoundByIndex @@ -370,6 +379,102 @@ extension RunnerTests { return (nil, elements) } +#if os(iOS) && targetEnvironment(simulator) + func privateAXPointInspection( + app: XCUIApplication, + x: Double, + y: Double + ) -> (text: String?, elements: [PointInspectionElementPayload])? { + let response = RunnerAXSnapshotBridge.snapshotTree( + for: app, + maxDepth: 56, + maxNodes: 5_000, + deepExtensionCallLimit: 4, + customActionLimit: 0, + deadline: Date().addingTimeInterval(8) + ) + guard (response["ok"] as? NSNumber)?.boolValue == true, + let root = response["root"] as? [String: Any] + else { + return nil + } + return privateAXPointInspection(root: root, point: CGPoint(x: x, y: y)) + } + + func privateAXPointInspection( + root: [String: Any], + point: CGPoint + ) -> (text: String?, elements: [PointInspectionElementPayload]) { + var candidates: [(payload: PointInspectionElementPayload, area: CGFloat)] = [] + + func visit(_ raw: [String: Any]) { + let frame = privateAXRect(raw["frame"]) + if !frame.isEmpty && frame.contains(point) { + let rawType = (raw["type"] as? NSNumber)?.intValue ?? 0 + let type = Self.elementTypeNamesByRawValue[UInt(rawType)] ?? "Element(\(rawType))" + let label = pointInspectionText(raw["label"]) + let identifier = pointInspectionText(raw["identifier"]) + let value = pointInspectionText(raw["value"]) + let text = pointInspectionReadableText( + type: type, + label: label, + identifier: identifier, + value: value + ) + candidates.append(( + PointInspectionElementPayload( + text: text, + label: label, + identifier: identifier, + type: type, + role: type, + value: value, + frame: SnapshotRect(frame), + hittable: nil + ), + max(1, frame.width * frame.height) + )) + } + for child in raw["children"] as? [[String: Any]] ?? [] { + visit(child) + } + } + + visit(root) + let elements = candidates + .sorted { left, right in + if left.area != right.area { return left.area < right.area } + if left.payload.frame.y != right.payload.frame.y { + return left.payload.frame.y < right.payload.frame.y + } + if left.payload.frame.x != right.payload.frame.x { + return left.payload.frame.x < right.payload.frame.x + } + return (left.payload.type ?? "") < (right.payload.type ?? "") + } + .prefix(24) + .map(\.payload) + return (elements.compactMap(\.text).first, elements) + } + + private func pointInspectionText(_ value: Any?) -> String? { + let text = (value as? String ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + return text.isEmpty ? nil : text + } + + private func pointInspectionReadableText( + type: String, + label: String?, + identifier: String?, + value: String? + ) -> String? { + if ["TextField", "SecureTextField", "SearchField", "TextView"].contains(type) { + return value ?? label ?? identifier + } + return label ?? value ?? identifier + } +#endif + private func readableText(for element: XCUIElement) -> String? { let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index 659dbd9c8b..cde6dba9c2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -23,5 +23,50 @@ extension RunnerTests { XCTAssertEqual(events.count, 1) XCTAssertEqual(events.first?.vertical, -200) } + +#if os(iOS) && targetEnvironment(simulator) + func testPrivateAXPointInspectionReturnsContainingElementsSmallestFirst() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 400, "height": 800], + "children": [[ + "type": NSNumber(value: XCUIElement.ElementType.button.rawValue), + "label": "Native Action", + "identifier": "native-action", + "value": "", + "frame": ["x": 100, "y": 500, "width": 200, "height": 48], + "children": [], + ]], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 200, y: 520)) + + XCTAssertEqual(inspection.text, "Native Action") + XCTAssertEqual(inspection.elements.count, 2) + XCTAssertEqual(inspection.elements.first?.identifier, "native-action") + XCTAssertEqual(inspection.elements.last?.identifier, "root") + XCTAssertEqual(inspection.elements.first?.frame, SnapshotRect(x: 100, y: 500, width: 200, height: 48)) + XCTAssertNil(inspection.elements.first?.hittable) + } + + func testPrivateAXPointInspectionReturnsNoElementForHonestMiss() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "Root", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 100, "height": 100], + "children": [], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 200, y: 200)) + + XCTAssertNil(inspection.text) + XCTAssertTrue(inspection.elements.isEmpty) + } +#endif } #endif From a55892e14c3cac3d4351aab9f576243d78ab75c9 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 10:37:01 -0700 Subject: [PATCH 05/21] fix(apple): ignore owning app in point inspection --- .../RunnerTests+Interaction.swift | 9 ++++++++ .../RunnerTests+InteractionTests.swift | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 56d4e53215..7c926e9ca9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -411,6 +411,15 @@ extension RunnerTests { let frame = privateAXRect(raw["frame"]) if !frame.isEmpty && frame.contains(point) { let rawType = (raw["type"] as? NSNumber)?.intValue ?? 0 + // The private tree includes the owning application at the full screen + // frame. Its label is the app name, not a control under the point, so + // returning it can make point-based system-UI probes tap a false match. + if rawType == XCUIElement.ElementType.application.rawValue { + for child in raw["children"] as? [[String: Any]] ?? [] { + visit(child) + } + return + } let type = Self.elementTypeNamesByRawValue[UInt(rawType)] ?? "Element(\(rawType))" let label = pointInspectionText(raw["label"]) let identifier = pointInspectionText(raw["identifier"]) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index cde6dba9c2..f03bdf79f9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -67,6 +67,29 @@ extension RunnerTests { XCTAssertNil(inspection.text) XCTAssertTrue(inspection.elements.isEmpty) } + + func testPrivateAXPointInspectionOmitsOwningApplicationLabel() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.application.rawValue), + "label": "ET N Action", + "identifier": "com.expotargets.example.native.action", + "value": "", + "frame": ["x": 0, "y": 0, "width": 400, "height": 800], + "children": [[ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "Share Sheet", + "identifier": "share-sheet", + "value": "", + "frame": ["x": 0, "y": 400, "width": 400, "height": 400], + "children": [], + ]], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 200, y: 520)) + + XCTAssertEqual(inspection.text, "Share Sheet") + XCTAssertFalse(inspection.elements.contains { $0.label == "ET N Action" }) + } #endif } #endif From 008bd128a9debb9a7618ebba40c489b3734870d2 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 13:08:42 -0700 Subject: [PATCH 06/21] fix: preserve point inspection contracts --- .../RunnerTests+CommandExecution.swift | 22 ++- .../RunnerTests+Interaction.swift | 46 ++++-- .../RunnerTests+Models.swift | 1 + .../RunnerTests+InteractionTests.swift | 21 +++ contracts/fixtures/runner-requests.json | 2 +- packages/command-registry/src/registry.ts | 2 +- packages/contracts/src/client-capture.ts | 2 +- packages/contracts/src/interactor-types.ts | 2 +- .../contracts/src/point-inspection-runtime.ts | 2 - packages/platform-android/src/runtime.test.ts | 6 +- packages/platform-android/src/runtime.ts | 8 +- .../interactor-runner-provider.test.ts | 59 +++++++ .../__tests__/recording-runner-provider.ts | 6 +- packages/platform-apple/src/interactor.ts | 33 +++- .../src/runner/runner-contract.ts | 1 + packages/platform-apple/src/runtime.test.ts | 153 ++++++++++-------- src/__tests__/cli-client-commands.test.ts | 19 +-- .../command-descriptor-timeout-policy.test.ts | 3 + .../__tests__/args-parse-session.test.ts | 11 ++ src/cli/parser/args.ts | 2 +- src/commands/capture/index.test.ts | 14 +- src/commands/capture/index.ts | 44 ++++- src/commands/capture/inspect-point.ts | 5 - src/commands/capture/snapshot.ts | 47 +----- src/daemon/inspect-point-runtime.ts | 5 +- .../command-coverage/declarations.ts | 4 +- .../apple-platform-output-guard.test.ts | 8 +- .../provider-scenarios/ios-lifecycle.test.ts | 2 - .../provider-scenarios/ios-world.ts | 2 +- 29 files changed, 345 insertions(+), 187 deletions(-) delete mode 100644 src/commands/capture/inspect-point.ts diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 2a6fff542a..0e695931a8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -422,11 +422,23 @@ extension RunnerTests { guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "readText requires x and y")) } - let inspection = readPointAt(app: activeApp, x: x, y: y) - return Response( - ok: true, - data: DataPayload(text: inspection.text, elements: inspection.elements) - ) + if command.inspectPoint == true { + let inspection = readPointAt(app: activeApp, x: x, y: y) + guard inspection.complete else { + return Response( + ok: false, + error: ErrorPayload( + code: "COMMAND_FAILED", + message: "Point inspection could not prove a miss because its accessibility capture was truncated" + ) + ) + } + return Response(ok: true, data: DataPayload(text: inspection.text, elements: inspection.elements)) + } + guard let text = readTextAt(app: activeApp, x: x, y: y) else { + return Response(ok: false, error: ErrorPayload(message: "readText did not resolve text")) + } + return Response(ok: true, data: DataPayload(text: text)) case .screenshot: #if os(macOS) // macOS keeps the app-targeted capture behavior for window-level screenshots. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 7c926e9ca9..4392e7d5a3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -318,18 +318,32 @@ extension RunnerTests { } func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { - readPointAt(app: app, x: x, y: y).text + let point = CGPoint(x: x, y: y) + let textInputCandidates = textInputCandidatesAt(app: app, point: point) + for element in textInputCandidates where prefersExpandedTextRead(element) { + if let text = readableText(for: element) { return text } + } + let candidates = app.descendants(matching: .any).allElementsBoundByIndex + .filter { $0.exists && !$0.frame.isEmpty && $0.frame.contains(point) } + .sorted(by: smallestElementFirst) + for element in candidates where prefersExpandedTextRead(element) { + if let text = readableText(for: element) { return text } + } + for element in candidates { + if let text = readableText(for: element) { return text } + } + return nil } func readPointAt( app: XCUIApplication, x: Double, y: Double - ) -> (text: String?, elements: [PointInspectionElementPayload]) { + ) -> (text: String?, elements: [PointInspectionElementPayload], complete: Bool) { #if os(iOS) && targetEnvironment(simulator) // System-owned sheets can make XCTest's unbounded descendants query hold // the main thread past the command watchdog. The Simulator-only private AX - // bridge already provides a deadline-bounded tree for snapshot recovery; + // bridge supplies the bounded accessibility tree used by snapshot recovery; // filter that tree at the requested point before touching the XCTest query. if let inspection = privateAXPointInspection(app: app, x: x, y: y) { return inspection @@ -353,7 +367,6 @@ extension RunnerTests { label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier, type: elementTypeName(element.elementType), - role: elementTypeName(element.elementType), value: value.isEmpty ? nil : value, frame: SnapshotRect(element.frame), hittable: element.isHittable @@ -362,21 +375,21 @@ extension RunnerTests { for element in textInputCandidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { - return (text, elements) + return (text, elements, true) } } for element in candidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { - return (text, elements) + return (text, elements, true) } } for element in candidates { if let text = readableText(for: element) { - return (text, elements) + return (text, elements, true) } } - return (nil, elements) + return (nil, elements, true) } #if os(iOS) && targetEnvironment(simulator) @@ -384,7 +397,7 @@ extension RunnerTests { app: XCUIApplication, x: Double, y: Double - ) -> (text: String?, elements: [PointInspectionElementPayload])? { + ) -> (text: String?, elements: [PointInspectionElementPayload], complete: Bool)? { let response = RunnerAXSnapshotBridge.snapshotTree( for: app, maxDepth: 56, @@ -398,13 +411,19 @@ extension RunnerTests { else { return nil } - return privateAXPointInspection(root: root, point: CGPoint(x: x, y: y)) + return privateAXPointInspection( + root: root, + point: CGPoint(x: x, y: y), + truncated: (response["truncated"] as? NSNumber)?.boolValue == true + ) } func privateAXPointInspection( root: [String: Any], - point: CGPoint - ) -> (text: String?, elements: [PointInspectionElementPayload]) { + point: CGPoint, + truncated: Bool = false + ) -> (text: String?, elements: [PointInspectionElementPayload], complete: Bool) { + guard !truncated else { return (nil, [], false) } var candidates: [(payload: PointInspectionElementPayload, area: CGFloat)] = [] func visit(_ raw: [String: Any]) { @@ -436,7 +455,6 @@ extension RunnerTests { label: label, identifier: identifier, type: type, - role: type, value: value, frame: SnapshotRect(frame), hittable: nil @@ -463,7 +481,7 @@ extension RunnerTests { } .prefix(24) .map(\.payload) - return (elements.compactMap(\.text).first, elements) + return (elements.compactMap(\.text).first, elements, true) } private func pointInspectionText(_ value: Any?) -> String? { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index ad5bf379f5..a7b6f633ca 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -184,6 +184,7 @@ extension Command { struct Command: Codable { let command: CommandType + let inspectPoint: Bool? let commandId: String? let statusCommandId: String? let appBundleId: String? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index f03bdf79f9..e318c8bc18 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -68,6 +68,27 @@ extension RunnerTests { XCTAssertTrue(inspection.elements.isEmpty) } + func testPrivateAXPointInspectionDoesNotReportMissForTruncatedCapture() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "Root", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 100, "height": 100], + "children": [], + ] + + let inspection = privateAXPointInspection( + root: root, + point: CGPoint(x: 200, y: 200), + truncated: true + ) + + XCTAssertFalse(inspection.complete) + XCTAssertNil(inspection.text) + XCTAssertTrue(inspection.elements.isEmpty) + } + func testPrivateAXPointInspectionOmitsOwningApplicationLabel() { let root: [String: Any] = [ "type": NSNumber(value: XCUIElement.ElementType.application.rawValue), diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index 4e8d146df5..ea8b1a6af2 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -38,7 +38,7 @@ {"name": "ios-simulator.interactor-back.system", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "backSystem", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-find-text.text", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "findText", "text": "Ready", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-home.press", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "home", "appBundleId": "com.example.app", "commandId": ""}}, - {"name": "ios-simulator.interactor-inspect-point", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, + {"name": "ios-simulator.interactor-inspect-point", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "readText", "inspectPoint": true, "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-keyboard-dismiss.dismiss", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "keyboardDismiss", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-keyboard-enter.return", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "keyboardReturn", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-read-text.point", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index beda344a5a..a649b436a1 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -1089,7 +1089,7 @@ export const RAW_COMMAND_DESCRIPTORS = [ frameworkTier: 'extended', recordsSessionAction: false, daemon: { route: 'generic', refFrameEffect: 'preserve' }, - timeoutPolicy: DEFAULT_TIMEOUT_POLICY, + timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, batchable: true, platformExecution: { kind: 'device-runtime', uses: [inspectPointRuntimeUse] }, }, diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index b0bfb1c905..03ece82945 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -111,7 +111,7 @@ export type PointInspectionResult = | Readonly<{ status: 'inspected'; point: Readonly<{ x: number; y: number }>; - elements: readonly PointInspectionElement[]; + elements: readonly [PointInspectionElement, ...PointInspectionElement[]]; }> | Readonly<{ status: 'no-element-at-point'; diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 7b101933e8..6f5e0d8cd7 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -375,7 +375,7 @@ export type Interactor = { ): Promise; inspectPoint?( point: Point, - options?: { appBundleId?: string; surface?: SessionSurface; signal?: AbortSignal }, + options?: { appBundleId?: string; signal?: AbortSignal }, ): Promise; /** * Native text-presence reading, when the backend has one that does not require a tree capture. diff --git a/packages/contracts/src/point-inspection-runtime.ts b/packages/contracts/src/point-inspection-runtime.ts index 504668fb50..5d02f6e53d 100644 --- a/packages/contracts/src/point-inspection-runtime.ts +++ b/packages/contracts/src/point-inspection-runtime.ts @@ -1,7 +1,5 @@ export { bindPointInspectionRuntime, pointInspectionRuntimeOperationFacts, - type InspectPointRuntimeInput, - type PointInspectionRuntimeOperationFacts, type PointInspectionRuntimeOperations, } from './selector-observation-runtime.ts'; diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index a948d85a08..7fc39cd969 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -78,7 +78,11 @@ test.each([ expect(facts.operations.captureSnapshot).toEqual({ available: true }); // uiautomator reads text at a point over the same adb transport the capture uses. expect(facts.operations.readTextAtPoint).toEqual({ available: true }); - expect(facts.operations.inspectPoint).toMatchObject({ available: false }); + expect(facts.operations.inspectPoint).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'inspect-point is currently supported on iOS Simulator targets only.', + }); expect(facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(facts.operations.setViewport).toMatchObject({ available: false }); diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 42ea3401ca..27bba280fd 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -366,7 +366,13 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor ...elementTextRuntimeOperationFacts({ readTextAtPoint: device.kind === 'simulator' ? elementTextKindUnavailable : available, }), - ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextKindUnavailable }), + ...pointInspectionRuntimeOperationFacts({ + inspectPoint: Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'inspect-point is currently supported on iOS Simulator targets only.', + } as const), + }), ...backRuntimeOperationFacts({ back: androidTouchFact(device) }), // `home` and `app-switcher` are one `input keyevent` each, admitted wherever the retired // `ANDROID_ALL` bucket admitted them. diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index f349eb9f8a..d618fbe1b5 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -128,6 +128,65 @@ test('provider-backed interactor routes runner-command methods through the injec } }); +test('inspectPoint uses its explicit runner mode without changing readTextAtPoint wire requests', async () => { + const calls: RecordedRunnerCall[] = []; + const interactor = createAppleInteractor( + IOS_SIMULATOR, + { appBundleId: 'com.example.app' }, + recordingRunnerProvider(calls), + ); + + await interactor.readTextAtPoint!({ x: 10, y: 20 }); + assert.equal(calls[0]!.command.inspectPoint, undefined); + + calls.length = 0; + await interactor.inspectPoint!({ x: 10, y: 20 }); + assert.equal(calls[0]!.command.inspectPoint, true); +}); + +test('inspectPoint rejects a malformed runner payload instead of reporting an honest miss', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + recordingRunnerProvider([], { readText: { text: 'ignored' } }), + ); + await assert.rejects( + interactor.inspectPoint!({ x: 10, y: 20 }), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); +}); + +test('inspectPoint rejects non-finite accessibility frames', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + recordingRunnerProvider([], { + readText: { + elements: [{ label: 'Bad frame', frame: { x: Number.NaN, y: 0, width: 10, height: 10 } }], + }, + }), + ); + await assert.rejects( + interactor.inspectPoint!({ x: 10, y: 20 }), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); +}); + +test('inspectPoint returns valid runner descriptors in their reported order', async () => { + const elements = [ + { label: 'Smallest', frame: { x: 10, y: 10, width: 10, height: 10 } }, + { label: 'Containing', frame: { x: 0, y: 0, width: 40, height: 40 } }, + ]; + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + recordingRunnerProvider([], { readText: { elements } }), + ); + + const result = await interactor.inspectPoint!({ x: 12, y: 12 }); + assert.deepEqual(result.elements, elements); +}); + test('provider-backed interactor rejects local Apple tooling methods with a clear error', async () => { const interactor = createAppleInteractor(IOS_SIMULATOR, {}, recordingRunnerProvider([])); for (const [method, invoke] of Object.entries(LOCAL_TOOL_METHODS)) { diff --git a/packages/platform-apple/src/__tests__/recording-runner-provider.ts b/packages/platform-apple/src/__tests__/recording-runner-provider.ts index 5c97b0745b..a814838636 100644 --- a/packages/platform-apple/src/__tests__/recording-runner-provider.ts +++ b/packages/platform-apple/src/__tests__/recording-runner-provider.ts @@ -24,7 +24,9 @@ export function recordingRunnerProvider( }; } -export function runnerResultFor(sent: Pick) { +export function runnerResultFor( + sent: Pick, +) { switch (sent.command) { case 'snapshot': return { @@ -44,6 +46,8 @@ export function runnerResultFor(sent: Pick { const result = await runAppleRunnerCommand( device, - { command: 'readText', x: point.x, y: point.y, appBundleId: options?.appBundleId }, + { + command: 'readText', + inspectPoint: true, + x: point.x, + y: point.y, + appBundleId: options?.appBundleId, + }, options?.signal ? { ...runnerOpts, signal: options.signal } : runnerOpts, ); - const elements = Array.isArray(result.elements) - ? result.elements.flatMap((value) => { - const element = readPointInspectionElement(value); - return element ? [element] : []; - }) - : []; + if (!Array.isArray(result.elements)) { + throw new AppError( + 'COMMAND_FAILED', + 'iOS runner returned an invalid point-inspection payload: missing elements', + ); + } + const elements = result.elements.map((value) => { + const element = readPointInspectionElement(value); + if (!element) { + throw new AppError( + 'COMMAND_FAILED', + 'iOS runner returned an invalid point-inspection element', + ); + } + return element; + }); return { ...(typeof result.text === 'string' ? { text: result.text } : {}), elements, @@ -498,7 +514,8 @@ function readPointInspectionFrame(value: unknown): PointInspectionElement['frame if (typeof value !== 'object' || value === null) return undefined; const frame = value as Record; const dimensions = [frame.x, frame.y, frame.width, frame.height]; - if (!dimensions.every((dimension) => typeof dimension === 'number')) return undefined; + if (!dimensions.every((dimension) => typeof dimension === 'number' && Number.isFinite(dimension))) + return undefined; const [x, y, width, height] = dimensions as [number, number, number, number]; return { x, y, width, height }; } diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index 27522d3c1b..67dacb75ed 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -40,6 +40,7 @@ export const MAIN_THREAD_TIMEOUT_RUNNER_CODE = 'MAIN_THREAD_TIMEOUT'; const APP_NOT_RUNNING_RUNNER_CODE = 'APP_NOT_RUNNING'; export type RunnerCommand = { + inspectPoint?: boolean; command: | 'tap' | 'mouseClick' diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index 94ca13835e..9647469ba9 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -25,6 +25,18 @@ const presentIosAcquisition: SnapshotRuntimeHost['presentIosAcquisition'] = asyn nodes: [], }); +function expectPointInspectionBinding( + binding: DeviceBinding, + available: boolean, +): void { + expect(binding.facts.operations.inspectPoint.available).toBe(available); + if (available) { + expect(binding.operations.inspectPoint).toBeTypeOf('function'); + } else { + expect(binding.operations.inspectPoint).toBeUndefined(); + } +} + function appleDevice(overrides: Partial = {}): DeviceInfo { return { platform: 'apple', @@ -90,91 +102,92 @@ function expectAppStateFact( expect(binding.operations.appState).toBeTypeOf('function'); } -function expectsPointInspection(device: DeviceInfo): boolean { - return device.appleOs === 'ios' && device.kind === 'simulator'; -} - test.each([ - ['iOS simulator', leaves.ios, true, undefined], + ['iOS simulator', leaves.ios, true, undefined, true], [ 'iOS physical CoreDevice', appleDevice({ kind: 'device', iosPhysicalDeviceBackend: 'coredevice' }), true, undefined, + false, ], [ 'iOS physical XCTest', appleDevice({ kind: 'device', iosPhysicalDeviceBackend: 'xctest' }), false, 'CoreDevice-backed physical iOS device', + false, ], - ['iPadOS simulator', leaves.ipados, true, undefined], - ['tvOS simulator', leaves.tvos, true, undefined], - ['macOS host', leaves.macos, true, undefined], - ['visionOS simulator', leaves.visionos, true, undefined], - ['watchOS sentinel', leaves.watchos, false, 'watchOS app logs are not supported'], -])('classifies the %s leaf explicitly', async (_name, device, available, hint) => { - const binding = await createApplePlatformRuntime(platformRuntimeHostFixture()).bind({ - device, - intent: { kind: 'ordinary' }, - scope: { - signal: new AbortController().signal, - diagnostics: { emit: () => {} }, - progress: { report: () => {} }, - }, - }); - const { facts } = binding; - expect(facts.device.providerMode).toBe('local'); - expectAppStateFact(device, binding); - expect(facts.operations.networkDump).toEqual({ available: true }); - expect(facts.operations.listApps.available).toBe( - device.appleOs !== 'watchos' && device.iosPhysicalDeviceBackend !== 'xctest', - ); - // R40/R41: touch and text ride the Apple interactor, which exists for the simulator and - // physical device kinds — every leaf in this table is one of those two, so both cells are - // available across it (parity with the retired buckets). - expect(facts.operations.focusPoint).toEqual({ available: true }); - expect(facts.operations.typeText).toEqual({ available: true }); - expect(facts.operations.inspectPoint.available).toBe(expectsPointInspection(device)); - expect(binding.operations.focusPoint).toBeTypeOf('function'); - expect(binding.operations.typeText).toBeTypeOf('function'); - for (const operation of ['appLogInspect', 'appLogDoctor', 'appLogStart'] as const) { - const fact = facts.operations[operation]; - expect(fact.available).toBe(available); - if (!available && hint) expect(fact).toHaveProperty('hint', expect.stringContaining(hint)); - } - for (const operation of [ - 'screenRecordingStart', - 'screenRecordingReattach', - 'screenRecordingCleanup', - ] as const) { - expect(facts.operations[operation].available).toBe(available); - } - expectApplePerfAvailability(binding, available); - if (device.iosPhysicalDeviceBackend === 'xctest') { - expect(facts.operations.screenRecordingStart).toMatchObject({ - hint: expect.stringContaining('CoreDevice-backed physical iOS device'), + ['iPadOS simulator', leaves.ipados, true, undefined, false], + ['tvOS simulator', leaves.tvos, true, undefined, false], + ['macOS host', leaves.macos, true, undefined, false], + ['visionOS simulator', leaves.visionos, true, undefined, false], + ['watchOS sentinel', leaves.watchos, false, 'watchOS app logs are not supported', false], +])( + 'classifies the %s leaf explicitly', + async (_name, device, available, hint, inspectPointAvailable) => { + const binding = await createApplePlatformRuntime(platformRuntimeHostFixture()).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, }); - } - if (device.appleOs === 'watchos') { - expect(facts.operations.screenRecordingStart).toMatchObject({ - hint: 'watchOS recording is not supported.', + const { facts } = binding; + expect(facts.device.providerMode).toBe('local'); + expectAppStateFact(device, binding); + expect(facts.operations.networkDump).toEqual({ available: true }); + expect(facts.operations.listApps.available).toBe( + device.appleOs !== 'watchos' && device.iosPhysicalDeviceBackend !== 'xctest', + ); + // R40/R41: touch and text ride the Apple interactor, which exists for the simulator and + // physical device kinds — every leaf in this table is one of those two, so both cells are + // available across it (parity with the retired buckets). + expect(facts.operations.focusPoint).toEqual({ available: true }); + expect(facts.operations.typeText).toEqual({ available: true }); + expectPointInspectionBinding(binding, inspectPointAvailable); + expect(binding.operations.focusPoint).toBeTypeOf('function'); + expect(binding.operations.typeText).toBeTypeOf('function'); + for (const operation of ['appLogInspect', 'appLogDoctor', 'appLogStart'] as const) { + const fact = facts.operations[operation]; + expect(fact.available).toBe(available); + if (!available && hint) expect(fact).toHaveProperty('hint', expect.stringContaining(hint)); + } + for (const operation of [ + 'screenRecordingStart', + 'screenRecordingReattach', + 'screenRecordingCleanup', + ] as const) { + expect(facts.operations[operation].available).toBe(available); + } + expectApplePerfAvailability(binding, available); + if (device.iosPhysicalDeviceBackend === 'xctest') { + expect(facts.operations.screenRecordingStart).toMatchObject({ + hint: expect.stringContaining('CoreDevice-backed physical iOS device'), + }); + } + if (device.appleOs === 'watchos') { + expect(facts.operations.screenRecordingStart).toMatchObject({ + hint: 'watchOS recording is not supported.', + }); + } + expect(facts.operations.ensureReady.available).toBe(device.appleOs !== 'watchos'); + expect(facts.operations.bootTarget.available).toBe( + device.appleOs !== 'macos' && device.appleOs !== 'watchos', + ); + expect(facts.operations.bootTargetHeadless.available).toBe(false); + expect(facts.operations.setViewport).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'viewport resizes web targets only (--platform web). Apple screen geometry is fixed by the selected simulator or device type — open a different simulator to test another screen size.', }); - } - expect(facts.operations.ensureReady.available).toBe(device.appleOs !== 'watchos'); - expect(facts.operations.bootTarget.available).toBe( - device.appleOs !== 'macos' && device.appleOs !== 'watchos', - ); - expect(facts.operations.bootTargetHeadless.available).toBe(false); - expect(facts.operations.setViewport).toEqual({ - available: false, - reason: 'unsupported-platform-leaf', - hint: 'viewport resizes web targets only (--platform web). Apple screen geometry is fixed by the selected simulator or device type — open a different simulator to test another screen size.', - }); - expect(binding.operations.setViewport).toBeUndefined(); - expectAppleCaptureAvailability(binding, device); - expectAppleSnapshotAvailability(binding, device); -}); + expect(binding.operations.setViewport).toBeUndefined(); + expectAppleCaptureAvailability(binding, device); + expectAppleSnapshotAvailability(binding, device); + }, +); function expectApplePerfAvailability( binding: DeviceBinding, diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index 0117b1b392..4250d3c166 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -1167,10 +1167,7 @@ function createStubClient(params: { const command = createThrowingMethodGroup(); return { command, - devices: { - ...createThrowingMethodGroup(), - list: async () => [], - }, + devices: createThrowingMethodGroup({ list: async () => [] }), sessions: { list: async () => [], stateDir: async () => '/tmp/agent-device-state', @@ -1248,8 +1245,7 @@ function createStubClient(params: { transport: 'http' as const, })), }, - capture: { - ...createThrowingMethodGroup(), + capture: createThrowingMethodGroup({ snapshot: async () => ({ nodes: [], truncated: false, @@ -1261,14 +1257,13 @@ function createStubClient(params: { path: '/tmp/screenshot.png', identifiers: { session: 'default' }, })), - }, + }), interactions: createThrowingMethodGroup(), replay: createThrowingMethodGroup(), batch: createThrowingMethodGroup(), - observability: { - ...createThrowingMethodGroup(), + observability: createThrowingMethodGroup({ events: params.events ?? unexpectedCommandCall, - }, + }), debug: createThrowingMethodGroup(), recording: createThrowingMethodGroup(), settings: { @@ -1277,11 +1272,11 @@ function createStubClient(params: { }; } -function createThrowingMethodGroup(): T { +function createThrowingMethodGroup(methods: Partial = {}): T { const unexpectedCommandCall = async (): Promise => { throw new Error('unexpected command call'); }; - return new Proxy({} as Partial, { + return new Proxy(methods, { get: (target, property) => target[property as keyof T] ?? unexpectedCommandCall, }) as T; } diff --git a/src/__tests__/command-descriptor-timeout-policy.test.ts b/src/__tests__/command-descriptor-timeout-policy.test.ts index d8d444d278..cf2e33e579 100644 --- a/src/__tests__/command-descriptor-timeout-policy.test.ts +++ b/src/__tests__/command-descriptor-timeout-policy.test.ts @@ -61,6 +61,8 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { // Interaction commands joined in #1105: their target resolution runs the // same capture as snapshot, and resetting the daemon on a wedged capture // destroyed healthy app sessions. + // inspect-point joins because its XCTest accessibility read has the same + // blocked-bridge failure mode and must not reset unrelated sessions. // scroll/back joined in #1638: `--settle` gives them the same post-action // capture loop, so a wedged bridge is now their dominant hang mode too. // The lease route joined in #1774: those commands act on BILLED provider @@ -77,6 +79,7 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { 'find', 'get', 'hover', + 'inspect-point', 'is', 'lease_allocate', 'lease_heartbeat', diff --git a/src/cli/parser/__tests__/args-parse-session.test.ts b/src/cli/parser/__tests__/args-parse-session.test.ts index 14c87a98f7..50caefe688 100644 --- a/src/cli/parser/__tests__/args-parse-session.test.ts +++ b/src/cli/parser/__tests__/args-parse-session.test.ts @@ -252,6 +252,17 @@ test('parseArgs recognizes command-specific flag combinations', async () => { } }); +test('parseArgs rejects empty inline numeric values', () => { + assert.throws( + () => parseArgs(['inspect-point', '--x=', '--y=12'], { strictFlags: true }), + (error: unknown) => error instanceof Error && /Invalid x/.test(error.message), + ); + assert.throws( + () => parseArgs(['inspect-point', '--x=12', '--y='], { strictFlags: true }), + (error: unknown) => error instanceof Error && /Invalid y/.test(error.message), + ); +}); + test('parseArgs recognizes device isolation flags', () => { const parsed = parseArgs( [ diff --git a/src/cli/parser/args.ts b/src/cli/parser/args.ts index cf7cc19230..23690f3a41 100644 --- a/src/cli/parser/args.ts +++ b/src/cli/parser/args.ts @@ -308,7 +308,7 @@ function parseFlagValue( return { value, consumeNext: inlineValue === undefined }; } const parsed = Number(value); - if (!Number.isFinite(parsed)) { + if (value.trim().length === 0 || !Number.isFinite(parsed)) { throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); } if (typeof definition.min === 'number' && parsed < definition.min) { diff --git a/src/commands/capture/index.test.ts b/src/commands/capture/index.test.ts index 814ab32be4..e899e4b18e 100644 --- a/src/commands/capture/index.test.ts +++ b/src/commands/capture/index.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'vitest'; import type { CliFlags } from '@agent-device/contracts/command'; import { alertCliReader, alertDaemonWriter } from './alert.ts'; import { diffCliReader } from './diff.ts'; -import { inspectPointCliReader, inspectPointDaemonWriter } from './inspect-point.ts'; +import { captureCommandFamily } from './index.ts'; import { snapshotCliOutput } from './output.ts'; import { screenshotCliReader, screenshotDaemonWriter } from './screenshot.ts'; import { snapshotCliReader } from './snapshot.ts'; @@ -23,16 +23,22 @@ function expectInvalidArgs(fn: () => unknown, messageFragment: string) { describe('capture command interface', () => { test('reads inspect-point coordinate flags and writes its daemon request', () => { - const input = inspectPointCliReader([], flags({ pointX: 14.5, pointY: 29 })); + const input = captureCommandFamily.cliReaders['inspect-point']( + [], + flags({ pointX: 14.5, pointY: 29 }), + ); expect(input).toMatchObject({ point: { x: 14.5, y: 29 } }); - expect(inspectPointDaemonWriter(input)).toMatchObject({ + expect(captureCommandFamily.daemonWriters?.['inspect-point']?.(input)).toMatchObject({ command: 'inspect-point', positionals: ['14.5', '29'], }); }); test('rejects inspect-point when either coordinate is missing', () => { - expectInvalidArgs(() => inspectPointCliReader([], flags({ pointX: 1 })), '--x and --y'); + expectInvalidArgs( + () => captureCommandFamily.cliReaders['inspect-point']([], flags({ pointX: 1 })), + '--x and --y', + ); }); test('reads snapshot flags', () => { diff --git a/src/commands/capture/index.ts b/src/commands/capture/index.ts index 219bdeffbd..c7d0b3aadc 100644 --- a/src/commands/capture/index.ts +++ b/src/commands/capture/index.ts @@ -1,11 +1,51 @@ -import { defineCommandFamilyFromFacets } from '../family/types.ts'; +import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/types.ts'; +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { InspectPointOptions, PointInspectionResult } from '@agent-device/contracts/client'; +import { AppError } from '@agent-device/kernel/errors'; +import { pointField, requiredField } from '../command-input.ts'; +import { commonInputFromFlags, direct } from '../cli-grammar/common.ts'; +import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; +import { defineFieldCommandMetadata } from '../field-command-contract.ts'; +import { resultOutput } from '../output-common.ts'; import { alertCommandFacet } from './alert.ts'; import { diffCommandFacet } from './diff.ts'; import { screenshotCommandFacet } from './screenshot.ts'; import { settingsCommandFacet } from './settings.ts'; -import { inspectPointCommandFacet, snapshotCommandFacet } from './snapshot.ts'; +import { snapshotCommandFacet } from './snapshot.ts'; import { waitCommandFacet } from './wait.ts'; +const inspectPointMetadata = defineFieldCommandMetadata( + 'inspect-point', + 'Inspect the live accessibility elements containing one screen coordinate.', + { point: requiredField(pointField('Screen coordinate to inspect.')) }, +); + +const inspectPointCliReader: CliReader = (_positionals, flags) => { + if (typeof flags.pointX !== 'number' || typeof flags.pointY !== 'number') { + throw new AppError('INVALID_ARGS', 'inspect-point requires --x and --y'); + } + return { ...commonInputFromFlags(flags), point: { x: flags.pointX, y: flags.pointY } }; +}; + +const inspectPointDaemonWriter: DaemonWriter = direct(PUBLIC_COMMANDS['inspect-point'], (input) => { + const { point } = input as InspectPointOptions; + return [String(point.x), String(point.y)]; +}); + +const inspectPointCommandFacet = defineCommandFacet({ + name: 'inspect-point', + text: { summary: 'Inspect elements at a screen coordinate' }, + metadata: inspectPointMetadata, + run: (client, input) => client.capture.inspectPoint(input), + cliSchema: { allowedFlags: ['pointX', 'pointY'] }, + cliReader: inspectPointCliReader, + daemonWriter: inspectPointDaemonWriter, + cliOutputFormatter: resultOutput((result: PointInspectionResult) => ({ + data: result, + text: JSON.stringify(result, null, 2), + })), +}); + const captureCommandFacets = [ snapshotCommandFacet, inspectPointCommandFacet, diff --git a/src/commands/capture/inspect-point.ts b/src/commands/capture/inspect-point.ts deleted file mode 100644 index c89670ec84..0000000000 --- a/src/commands/capture/inspect-point.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { - inspectPointCliReader, - inspectPointCommandFacet, - inspectPointDaemonWriter, -} from './snapshot.ts'; diff --git a/src/commands/capture/snapshot.ts b/src/commands/capture/snapshot.ts index 1cfaab7170..093a9c2056 100644 --- a/src/commands/capture/snapshot.ts +++ b/src/commands/capture/snapshot.ts @@ -1,20 +1,11 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import type { InspectPointOptions, PointInspectionResult } from '@agent-device/contracts/client'; import { SNAPSHOT_BACKEND_CAPABILITIES } from '@agent-device/capture-kit/snapshot-quality-backend-capabilities'; import { SNAPSHOT_COMMAND_OPTION_KEYS, snapshotOptionsFromFlags, } from '@agent-device/kernel/snapshot'; import { SNAPSHOT_FLAGS } from '@agent-device/command-registry/flag-groups'; -import { AppError } from '@agent-device/kernel/errors'; -import { - booleanField, - integerField, - optionField, - pointField, - requiredField, - stringField, -} from '../command-input.ts'; +import { booleanField, integerField, optionField, stringField } from '../command-input.ts'; import { commonInputFromFlags, direct, @@ -24,7 +15,6 @@ import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; import { defineCommandFacet } from '../family/types.ts'; import { defineFieldCommandMetadata } from '../field-command-contract.ts'; import { captureCliOutputFormatters } from './output.ts'; -import { resultOutput } from '../output-common.ts'; const SNAPSHOT_COMMAND_NAME = 'snapshot'; @@ -93,38 +83,3 @@ export const snapshotCommandFacet = defineCommandFacet({ daemonWriter: snapshotDaemonWriter, cliOutputFormatter: captureCliOutputFormatters.snapshot, }); - -const inspectPointMetadata = defineFieldCommandMetadata( - 'inspect-point', - 'Inspect the live accessibility elements containing one screen coordinate.', - { point: requiredField(pointField('Screen coordinate to inspect.')) }, -); - -export const inspectPointCliReader: CliReader = (_positionals, flags) => { - if (typeof flags.pointX !== 'number' || typeof flags.pointY !== 'number') { - throw new AppError('INVALID_ARGS', 'inspect-point requires --x and --y'); - } - return { ...commonInputFromFlags(flags), point: { x: flags.pointX, y: flags.pointY } }; -}; - -export const inspectPointDaemonWriter: DaemonWriter = direct( - PUBLIC_COMMANDS['inspect-point'], - (input) => { - const { point } = input as InspectPointOptions; - return [String(point.x), String(point.y)]; - }, -); - -export const inspectPointCommandFacet = defineCommandFacet({ - name: 'inspect-point', - text: { summary: 'Inspect elements at a screen coordinate' }, - metadata: inspectPointMetadata, - run: (client, input) => client.capture.inspectPoint(input), - cliSchema: { allowedFlags: ['pointX', 'pointY'] }, - cliReader: inspectPointCliReader, - daemonWriter: inspectPointDaemonWriter, - cliOutputFormatter: resultOutput((result: PointInspectionResult) => ({ - data: result, - text: JSON.stringify(result, null, 2), - })), -}); diff --git a/src/daemon/inspect-point-runtime.ts b/src/daemon/inspect-point-runtime.ts index 6675049db2..ca93681d21 100644 --- a/src/daemon/inspect-point-runtime.ts +++ b/src/daemon/inspect-point-runtime.ts @@ -38,9 +38,10 @@ export async function resolveBoundInspectPointRuntime( ...(context.appBundleId ? { options: { appBundleId: context.appBundleId } } : {}), execution: runtimeExecutionFromContext(context), }); - return result.elements.length === 0 + const [first, ...rest] = result.elements; + return first === undefined ? { status: 'no-element-at-point', point, elements: [] } - : { status: 'inspected', point, elements: result.elements }; + : { status: 'inspected', point, elements: [first, ...rest] }; }, ); } diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index f4228571fd..4f14a98c0d 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -1395,8 +1395,8 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), iosSimulator: iosSimulator.contract( 'packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts', - 'inspectPoint', - 'the Apple runner returns ordered accessibility descriptors for a screen coordinate', + 'inspectPoint returns valid runner descriptors in their reported order', + 'the Apple interactor preserves valid ordered point-inspection descriptors returned by the runner', ), macos: macos.contract( 'packages/platform-apple/src/runtime.test.ts', diff --git a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts index 76eaf8b96e..61724973ac 100644 --- a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts +++ b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts @@ -74,6 +74,7 @@ const DRIVEN_COMMANDS: Record = { [PUBLIC_COMMANDS.boot]: () => one(), [PUBLIC_COMMANDS.prepare]: () => one(['ios-runner']), [PUBLIC_COMMANDS.snapshot]: () => one([], { snapshotInteractiveOnly: true }), + [PUBLIC_COMMANDS['inspect-point']]: () => one(['10', '10']), [PUBLIC_COMMANDS.perf]: () => [{ positionals: [] }, { positionals: ['frames'] }], [PUBLIC_COMMANDS.record]: ({ world, tmpDir }) => world === 'ios' @@ -146,10 +147,7 @@ const DRIVEN_COMMANDS: Record = { // undriveable command has a home and the partition test keeps a new command from escaping the // guard silently. // --------------------------------------------------------------------------- -const SKIPPED_COMMANDS: Record = { - [PUBLIC_COMMANDS['inspect-point']]: - 'requires a local iOS Simulator XCTest runner; provider worlds do not expose that runtime', -}; +const SKIPPED_COMMANDS: Record = {}; // Commands driven last so the priority commands run against a live session. const DRIVE_LAST = new Set([PUBLIC_COMMANDS.shutdown, PUBLIC_COMMANDS.close]); @@ -249,6 +247,8 @@ function permissiveRunner(): AppleRunnerProvider { return { found: true, nodes: [richNodes()[2]] }; case 'findText': return { found: true }; + case 'readText': + return command.inspectPoint === true ? { elements: [] } : { done: true }; default: return { done: true }; } diff --git a/test/integration/provider-scenarios/ios-lifecycle.test.ts b/test/integration/provider-scenarios/ios-lifecycle.test.ts index 65ef25c939..04a92b300c 100644 --- a/test/integration/provider-scenarios/ios-lifecycle.test.ts +++ b/test/integration/provider-scenarios/ios-lifecycle.test.ts @@ -74,7 +74,6 @@ test('Provider-backed integration iOS Settings flow uses scripted simctl and run name: 'inspect the accessibility elements at a settings coordinate', command: 'inspect-point', positionals: ['196', '122'], - flags: { pointX: 196, pointY: 122 }, expectData: { status: 'inspected', point: { x: 196, y: 122 }, @@ -84,7 +83,6 @@ test('Provider-backed integration iOS Settings flow uses scripted simctl and run label: 'General', identifier: 'General', type: 'Button', - role: 'button', value: 'General', frame: { x: 146, y: 102, width: 100, height: 40 }, hittable: true, diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index 1f01207043..612ae4820f 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -46,6 +46,7 @@ export async function createIosSettingsWorld(): Promise { platform: 'apple', request: { command: 'readText', + inspectPoint: true, x: 196, y: 122, appBundleId: 'com.apple.Preferences', @@ -58,7 +59,6 @@ export async function createIosSettingsWorld(): Promise { label: 'General', identifier: 'General', type: 'Button', - role: 'button', value: 'General', frame: { x: 146, y: 102, width: 100, height: 40 }, hittable: true, From c29d2c44ac6d6e9a482710a0a8e43ac9c50941d3 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 13:15:45 -0700 Subject: [PATCH 07/21] fix: update inspect-point owner attribution --- packages/command-registry/src/registry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index a649b436a1..b5b8d1ef27 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -1083,7 +1083,7 @@ export const RAW_COMMAND_DESCRIPTORS = [ name: 'inspect-point', deviceClaimPolicy: 'require-owner', ...(ownerFilesEnabled - ? { ownerFiles: ['src/commands/capture/inspect-point.ts'] as const } + ? { ownerFiles: ['src/commands/capture/index.ts'] as const } : {}), catalog: { group: 'public' }, frameworkTier: 'extended', From 9818f101e8538340a8e54029e9eb02ab3dac0798 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 13:16:11 -0700 Subject: [PATCH 08/21] style: format command registry owner path --- packages/command-registry/src/registry.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index b5b8d1ef27..0096841628 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -1082,9 +1082,7 @@ export const RAW_COMMAND_DESCRIPTORS = [ { name: 'inspect-point', deviceClaimPolicy: 'require-owner', - ...(ownerFilesEnabled - ? { ownerFiles: ['src/commands/capture/index.ts'] as const } - : {}), + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/capture/index.ts'] as const } : {}), catalog: { group: 'public' }, frameworkTier: 'extended', recordsSessionAction: false, From f92d8195de54344440b93120c49da2e3b5bb55dc Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 13:17:14 -0700 Subject: [PATCH 09/21] fix: retain public type suppression notes --- src/client/client-types.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/client/client-types.ts b/src/client/client-types.ts index d266306584..1936faf2a8 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -33,12 +33,18 @@ export type { // these up via `export type *`, and that is their only job — every internal consumer imports // them from the declaring module instead. Fallow therefore sees no consumer, which is exactly // right and exactly not actionable: deleting them would remove names from the package's public -// types. +// types. Suppressed per name rather than baselined so the reason travels with the code. +// fallow-ignore-next-line unused-type export type { TargetShutdownResult } from '@agent-device/contracts/device'; +// fallow-ignore-next-line unused-type export type { MetroBridgeScope } from '@agent-device/contracts/remote'; +// fallow-ignore-next-line unused-type export type { AppsFilter } from '@agent-device/contracts/device'; +// fallow-ignore-next-line unused-type export type { AlertAction } from '@agent-device/contracts/alert-contract'; +// fallow-ignore-next-line unused-type export type { AppleOS } from '@agent-device/kernel/device'; +// fallow-ignore-next-line unused-type export type { JsonObject } from '@agent-device/contracts/client'; export type { BatchRunResult } from '@agent-device/command-registry/batch'; From abccb5806f0977824aadeb19191f8dc5f840c746 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 13:17:36 -0700 Subject: [PATCH 10/21] docs: clarify public type re-exports --- src/client/client-types.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/client/client-types.ts b/src/client/client-types.ts index 1936faf2a8..78cb854ba8 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -31,20 +31,13 @@ export type { // Contracts/kernel types re-exported into the PUBLISHED surface: `agent-device-client.ts` picks // these up via `export type *`, and that is their only job — every internal consumer imports -// them from the declaring module instead. Fallow therefore sees no consumer, which is exactly -// right and exactly not actionable: deleting them would remove names from the package's public -// types. Suppressed per name rather than baselined so the reason travels with the code. -// fallow-ignore-next-line unused-type +// them from the declaring module instead. They remain part of the package's public type surface +// for downstream consumers even though repository-local code does not import them here. export type { TargetShutdownResult } from '@agent-device/contracts/device'; -// fallow-ignore-next-line unused-type export type { MetroBridgeScope } from '@agent-device/contracts/remote'; -// fallow-ignore-next-line unused-type export type { AppsFilter } from '@agent-device/contracts/device'; -// fallow-ignore-next-line unused-type export type { AlertAction } from '@agent-device/contracts/alert-contract'; -// fallow-ignore-next-line unused-type export type { AppleOS } from '@agent-device/kernel/device'; -// fallow-ignore-next-line unused-type export type { JsonObject } from '@agent-device/contracts/client'; export type { BatchRunResult } from '@agent-device/command-registry/batch'; From ae66184a5c8d7484da0ea7b5e531ce3cc78a02ed Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 13:18:27 -0700 Subject: [PATCH 11/21] fix: share point positional parsing --- src/daemon/inspect-point-runtime.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/daemon/inspect-point-runtime.ts b/src/daemon/inspect-point-runtime.ts index ca93681d21..d5065f2a49 100644 --- a/src/daemon/inspect-point-runtime.ts +++ b/src/daemon/inspect-point-runtime.ts @@ -1,29 +1,18 @@ import { inspectPointRuntimeUse } from '@agent-device/contracts/platform-runtime-operations'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { AppError } from '@agent-device/kernel/errors'; +import { readPointPositionals } from '@agent-device/kernel/validation'; import type { DaemonCommandContext } from './context.ts'; import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; import { resolveBoundGenericRuntime, type RuntimeAdmissionBindings } from './runtime-admission.ts'; import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; -function coordinate(value: string | undefined, name: string): number { - const parsed = value === undefined ? Number.NaN : Number(value); - if (!Number.isFinite(parsed)) { - throw new AppError('INVALID_ARGS', `inspect-point requires a finite ${name} coordinate`); - } - return parsed; -} - export async function resolveBoundInspectPointRuntime( params: { device: DeviceInfo; positionals: string[]; } & RuntimeAdmissionBindings, ): Promise { - const point = { - x: coordinate(params.positionals[0], 'x'), - y: coordinate(params.positionals[1], 'y'), - }; + const point = readPointPositionals(params.positionals, 'inspect-point requires x y'); return await resolveBoundGenericRuntime( { command: 'inspect-point', From d661e22a9f4fea350d1cc2c74acce1d569899b82 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 13:37:23 -0700 Subject: [PATCH 12/21] fix: contain point inspection AX requests --- .../RunnerAXSnapshotBridge.h | 6 + .../RunnerAXSnapshotBridge.m | 139 ++++++++++++++++-- .../RunnerTests+Interaction.swift | 37 +++-- .../RunnerTests+AXSnapshotFallbackTests.swift | 89 +++++++++++ scripts/integration-progress-model.ts | 27 +++- 5 files changed, 263 insertions(+), 35 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h index 33493bec41..fde9c73bca 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h @@ -112,6 +112,12 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotCustomActionsBlockedKey; /// Total reads refused by single-flight admission. + (NSInteger)customActionReadBlockedCount; +/// Contained snapshot-request counters, exposed so the unit bundle can pin the +/// no-queued-work invariant for a wedged private AX snapshot request. ++ (NSInteger)snapshotReadsInFlight; ++ (NSInteger)snapshotReadDispatchCount; ++ (NSInteger)snapshotReadBlockedCount; + /// The shared AX client (`XCUIDevice.accessibilityInterface`), or nil when the /// private interface is unavailable. + (nullable id)accessibilityClient; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m index 81d1692e13..ab3f236592 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m @@ -30,6 +30,25 @@ /// capture's budget and starve every remaining candidate, turning a bounded /// enrichment into a capture-length stall. static const NSTimeInterval RunnerAXCustomActionReadTimeout = 1.0; +static const NSTimeInterval RunnerAXSnapshotReadTimeout = 2.0; + +/// Snapshot requests are synchronous private-XPC calls too. Keep them off the +/// XCTest main thread and refuse a second request while a timed-out call is +/// still outstanding; the client call itself cannot be cancelled. +static dispatch_queue_t RunnerAXSnapshotReadQueue(void) +{ + static dispatch_queue_t queue; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + queue = dispatch_queue_create("com.callstack.agentdevice.runner.ax-snapshot", + DISPATCH_QUEUE_SERIAL); + }); + return queue; +} + +static atomic_int RunnerAXSnapshotReadsInFlight = 0; +static atomic_long RunnerAXSnapshotReadDispatches = 0; +static atomic_long RunnerAXSnapshotReadBlocked = 0; /// The AX call is a synchronous XPC round trip that cannot be cancelled once /// issued, so the deadline above only frees the CALLER — the call itself keeps @@ -85,6 +104,16 @@ static dispatch_queue_t RunnerAXCustomActionReadQueue(void) @implementation RunnerAXSnapshotFrontier @end +@interface RunnerAXSnapshotBridge () ++ (nullable id)requestSnapshotContainedFromClient:(id)axClient + target:(id)target + attributes:(NSArray *)attributes + maxDepth:(NSInteger)maxDepth + maxNodes:(NSInteger)maxNodes + deadline:(nullable NSDate *)deadline + error:(NSError **)error; +@end + @implementation RunnerAXSnapshotBridge + (NSDictionary *)snapshotTreeForApplication:(XCUIApplication *)application @@ -127,12 +156,13 @@ @implementation RunnerAXSnapshotBridge @try { NSArray *attributes = [self snapshotAttributes]; NSError *error = nil; - id root = [self requestSnapshotFromClient:axClient - target:target - attributes:attributes - maxDepth:maxDepth - maxNodes:maxNodes - error:&error]; + id root = [self requestSnapshotContainedFromClient:axClient + target:target + attributes:attributes + maxDepth:maxDepth + maxNodes:maxNodes + deadline:deadline + error:&error]; if (nil == root) { return [self failure:error.localizedDescription ?: @"AX snapshot request returned nil"]; } @@ -242,12 +272,13 @@ + (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray *keyPaths = @[ @@ -649,6 +749,21 @@ + (NSInteger)customActionReadBlockedCount return (NSInteger)atomic_load(&RunnerAXCustomActionReadBlocked); } ++ (NSInteger)snapshotReadsInFlight +{ + return atomic_load(&RunnerAXSnapshotReadsInFlight); +} + ++ (NSInteger)snapshotReadDispatchCount +{ + return (NSInteger)atomic_load(&RunnerAXSnapshotReadDispatches); +} + ++ (NSInteger)snapshotReadBlockedCount +{ + return (NSInteger)atomic_load(&RunnerAXSnapshotReadBlocked); +} + + (nullable id)accessibilityClient { return [self objectFrom:XCUIDevice.sharedDevice selectorName:@"accessibilityInterface"]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 4392e7d5a3..979c1f3ff2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -357,6 +357,20 @@ extension RunnerTests { } .sorted(by: smallestElementFirst) + // Resolve the legacy text field before building optional point-inspection + // descriptors. The descriptor pass is still needed by inspect-point, but + // must not delay the primary text result with hittability AX reads. + func firstReadableText(in elements: [XCUIElement], preferredOnly: Bool) -> String? { + for element in elements { + if preferredOnly && !prefersExpandedTextRead(element) { continue } + if let text = readableText(for: element) { return text } + } + return nil + } + let text = firstReadableText(in: textInputCandidates, preferredOnly: true) + ?? firstReadableText(in: candidates, preferredOnly: true) + ?? firstReadableText(in: candidates, preferredOnly: false) + let elements = Array(candidates.prefix(24)).map { element in let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) @@ -367,29 +381,13 @@ extension RunnerTests { label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier, type: elementTypeName(element.elementType), + role: nil, value: value.isEmpty ? nil : value, frame: SnapshotRect(element.frame), - hittable: element.isHittable + hittable: nil ) } - - for element in textInputCandidates where prefersExpandedTextRead(element) { - if let text = readableText(for: element) { - return (text, elements, true) - } - } - - for element in candidates where prefersExpandedTextRead(element) { - if let text = readableText(for: element) { - return (text, elements, true) - } - } - for element in candidates { - if let text = readableText(for: element) { - return (text, elements, true) - } - } - return (nil, elements, true) + return (text, elements, true) } #if os(iOS) && targetEnvironment(simulator) @@ -455,6 +453,7 @@ extension RunnerTests { label: label, identifier: identifier, type: type, + role: nil, value: value, frame: SnapshotRect(frame), hittable: nil diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index 4e4fd9ff44..724606aafa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -291,6 +291,65 @@ extension RunnerTests { RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 2) } + func testHungInitialSnapshotRequestIsContainedAndRecovers() { + let client = HungSnapshotAXClientForTesting() + let dispatchesBefore = RunnerAXSnapshotBridge.snapshotReadDispatchCount() + let blockedBefore = RunnerAXSnapshotBridge.snapshotReadBlockedCount() + defer { client.release() } + + let firstStarted = Date() + let first = RunnerAXSnapshotBridge.snapshotTree( + withClient: client, + target: NSObject(), + maxDepth: 4, + maxNodes: 16, + deepExtensionCallLimit: 0, + customActionLimit: 0, + deadline: Date().addingTimeInterval(0.25) + ) + XCTAssertEqual(first["ok"] as? Bool, false) + XCTAssertGreaterThanOrEqual(-firstStarted.timeIntervalSinceNow, 0.2) + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadsInFlight(), 1) + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadDispatchCount(), dispatchesBefore + 1) + + // Repeated captures fail closed without adding work behind the wedged XPC call. + for _ in 0..<4 { + let repeated = RunnerAXSnapshotBridge.snapshotTree( + withClient: client, + target: NSObject(), + maxDepth: 4, + maxNodes: 16, + deepExtensionCallLimit: 0, + customActionLimit: 0, + deadline: Date().addingTimeInterval(1) + ) + XCTAssertEqual(repeated["ok"] as? Bool, false) + } + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadDispatchCount(), dispatchesBefore + 1) + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadBlockedCount(), blockedBefore + 4) + + client.release() + let drained = expectation(description: "wedged initial request drains") + DispatchQueue.global().async { + while RunnerAXSnapshotBridge.snapshotReadsInFlight() > 0 { + usleep(20_000) + } + drained.fulfill() + } + wait(for: [drained], timeout: 5) + + _ = RunnerAXSnapshotBridge.snapshotTree( + withClient: client, + target: NSObject(), + maxDepth: 4, + maxNodes: 16, + deepExtensionCallLimit: 0, + customActionLimit: 0, + deadline: Date().addingTimeInterval(1) + ) + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadDispatchCount(), dispatchesBefore + 2) + } + /// The element budget bounds how many elements we read; these caps bound what /// any ONE element can put in the response. Clipping must be reported, since /// a clipped list looks exactly like a complete one. @@ -487,6 +546,36 @@ extension RunnerTests { /// Stands in for an AX client whose `attributesForElement:` never returns — /// the wedged-server case the containment exists for. `release()` lets the /// hung call finish so recovery is observable. +private final class HungSnapshotAXClientForTesting: NSObject { + private let gate = DispatchSemaphore(value: 0) + private let lock = NSLock() + private var released = false + + @objc(requestSnapshotForElement:attributes:parameters:error:) + func requestSnapshot( + forElement element: Any, + attributes: Any, + parameters: Any, + error: NSErrorPointer + ) -> Any? { + lock.lock() + let alreadyReleased = released + lock.unlock() + if !alreadyReleased { + gate.wait() + } + return nil + } + + func release() { + lock.lock() + defer { lock.unlock() } + guard !released else { return } + released = true + gate.signal() + } +} + private final class HungAXClientForTesting: NSObject { private let gate = DispatchSemaphore(value: 0) private let releasedOnce = NSLock() diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 1c39f215e0..f1ada61470 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -202,14 +202,33 @@ function summarizeProviderScenarioFlagCoverage(files) { ]; const sources = files.map((file) => fs.readFileSync(file, 'utf8')).join('\n'); return flagTargets.map(([key, reason, aliases = []]) => { - const references = [key, ...aliases].reduce( - (count, candidate) => count + countFlagReferences(sources, candidate), - 0, - ); + // inspect-point takes x/y as positional arguments, unlike the named + // workflow flags counted below. Attribute those coordinates only to an + // inspect-point provider scenario so an unrelated geometry literal cannot + // satisfy this coverage row. + const positionalCoordinate = key === 'pointX' ? 0 : key === 'pointY' ? 1 : null; + const references = + positionalCoordinate === null + ? [key, ...aliases].reduce( + (count, candidate) => count + countFlagReferences(sources, candidate), + 0, + ) + : countInspectPointCoordinateReferences(sources, positionalCoordinate); return { key, reason, references }; }); } +function countInspectPointCoordinateReferences(text, coordinateIndex) { + let count = 0; + const scenarios = + /command\s*:\s*['"]inspect-point['"][\s\S]{0,240}?positionals\s*:\s*\[([^\]]*)\]/g; + for (const match of text.matchAll(scenarios)) { + const positionals = match[1]?.match(/['"][^'"]*['"]/g) ?? []; + if (positionals.length > coordinateIndex) count += 1; + } + return count; +} + function countFlagReferences(text, key) { const escaped = key.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); return text.match(new RegExp(`\\b${escaped}\\s*:`, 'g'))?.length ?? 0; From 5eae2466e4b72adb7780cd76239fe4731c45db71 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 13:58:57 -0700 Subject: [PATCH 13/21] fix: reject incomplete point inspection captures --- .../RunnerTests+Interaction.swift | 50 ++++--- .../RunnerTests+InteractionTests.swift | 21 +++ scripts/integration-progress-model.test.ts | 25 +++- scripts/integration-progress-model.ts | 127 +++++++++++++++++- 4 files changed, 196 insertions(+), 27 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 979c1f3ff2..f39ecfd32e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -320,16 +320,18 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) let textInputCandidates = textInputCandidatesAt(app: app, point: point) - for element in textInputCandidates where prefersExpandedTextRead(element) { - if let text = readableText(for: element) { return text } - } let candidates = app.descendants(matching: .any).allElementsBoundByIndex .filter { $0.exists && !$0.frame.isEmpty && $0.frame.contains(point) } .sorted(by: smallestElementFirst) - for element in candidates where prefersExpandedTextRead(element) { - if let text = readableText(for: element) { return text } - } - for element in candidates { + return firstReadableText(in: textInputCandidates, preferredOnly: true) + ?? firstReadableText(in: candidates, preferredOnly: true) + ?? firstReadableText(in: candidates, preferredOnly: false) + } + + /// Keep live point text resolution identical for legacy `get text` and point inspection. + func firstReadableText(in elements: [XCUIElement], preferredOnly: Bool) -> String? { + for element in elements { + if preferredOnly && !prefersExpandedTextRead(element) { continue } if let text = readableText(for: element) { return text } } return nil @@ -357,16 +359,7 @@ extension RunnerTests { } .sorted(by: smallestElementFirst) - // Resolve the legacy text field before building optional point-inspection - // descriptors. The descriptor pass is still needed by inspect-point, but - // must not delay the primary text result with hittability AX reads. - func firstReadableText(in elements: [XCUIElement], preferredOnly: Bool) -> String? { - for element in elements { - if preferredOnly && !prefersExpandedTextRead(element) { continue } - if let text = readableText(for: element) { return text } - } - return nil - } + // Resolve the legacy text semantics before materializing inspection payloads. let text = firstReadableText(in: textInputCandidates, preferredOnly: true) ?? firstReadableText(in: candidates, preferredOnly: true) ?? firstReadableText(in: candidates, preferredOnly: false) @@ -409,19 +402,36 @@ extension RunnerTests { else { return nil } + let deepExtension = response[RunnerAXSnapshotDeepExtensionKey] as? [String: Any] + let completeDeepExtension: Bool + if let deepExtension { + if let pending = deepExtension[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, + let missed = deepExtension[RunnerAXSnapshotDeepExtensionMissedKey] as? Int + { + completeDeepExtension = pending == 0 && missed == 0 + } else { + completeDeepExtension = false + } + } else { + // The bridge omits this field only when the initial tree had no capped + // frontiers to extend. + completeDeepExtension = true + } return privateAXPointInspection( root: root, point: CGPoint(x: x, y: y), - truncated: (response["truncated"] as? NSNumber)?.boolValue == true + truncated: (response["truncated"] as? NSNumber)?.boolValue == true, + completeDeepExtension: completeDeepExtension ) } func privateAXPointInspection( root: [String: Any], point: CGPoint, - truncated: Bool = false + truncated: Bool = false, + completeDeepExtension: Bool = true ) -> (text: String?, elements: [PointInspectionElementPayload], complete: Bool) { - guard !truncated else { return (nil, [], false) } + guard !truncated && completeDeepExtension else { return (nil, [], false) } var candidates: [(payload: PointInspectionElementPayload, area: CGFloat)] = [] func visit(_ raw: [String: Any]) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index e318c8bc18..4676534e16 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -89,6 +89,27 @@ extension RunnerTests { XCTAssertTrue(inspection.elements.isEmpty) } + func testPrivateAXPointInspectionDoesNotReportMissForIncompleteDeepExtension() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "Root", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 100, "height": 100], + "children": [], + ] + + let inspection = privateAXPointInspection( + root: root, + point: CGPoint(x: 200, y: 200), + completeDeepExtension: false + ) + + XCTAssertFalse(inspection.complete) + XCTAssertNil(inspection.text) + XCTAssertTrue(inspection.elements.isEmpty) + } + func testPrivateAXPointInspectionOmitsOwningApplicationLabel() { let root: [String: Any] = [ "type": NSNumber(value: XCUIElement.ElementType.application.rawValue), diff --git a/scripts/integration-progress-model.test.ts b/scripts/integration-progress-model.test.ts index dc90be093a..560ed4e7ab 100644 --- a/scripts/integration-progress-model.test.ts +++ b/scripts/integration-progress-model.test.ts @@ -3,7 +3,30 @@ import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { test } from 'node:test'; import os from 'node:os'; import path from 'node:path'; -import { buildIntegrationProgressModel } from './integration-progress-model.ts'; +import { + buildIntegrationProgressModel, + countInspectPointCoordinateReferences, +} from './integration-progress-model.ts'; + +test('integration progress scopes point coordinates to each inspect-point step object', () => { + const source = ` + const steps = [ + { + flags: { label: 'unrelated object with braces', nested: { positionals: ['wrong'] } }, + command: 'inspect-point', + assert: (response) => response.json, + positionals: ['196', '122'], + }, + { command: 'tap', positionals: ['900', '901'] }, + { command: 'inspect-point', positionals: ['196'] }, + { command: 'inspect-point', positionals: [] }, + ]; + // { command: 'inspect-point', positionals: ['unrelated', 'literal'] } + `; + + assert.equal(countInspectPointCoordinateReferences(source, 0), 2); + assert.equal(countInspectPointCoordinateReferences(source, 1), 1); +}); test('integration progress counts explicit generic Apple host-tool usage only', async () => { const root = await mkdtemp(path.join(os.tmpdir(), 'agent-device-progress-')); diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index f1ada61470..53bfa35279 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -218,17 +218,132 @@ function summarizeProviderScenarioFlagCoverage(files) { }); } -function countInspectPointCoordinateReferences(text, coordinateIndex) { +export function countInspectPointCoordinateReferences(text, coordinateIndex) { + const code = maskScenarioStringsAndComments(text); let count = 0; - const scenarios = - /command\s*:\s*['"]inspect-point['"][\s\S]{0,240}?positionals\s*:\s*\[([^\]]*)\]/g; - for (const match of text.matchAll(scenarios)) { - const positionals = match[1]?.match(/['"][^'"]*['"]/g) ?? []; - if (positionals.length > coordinateIndex) count += 1; + const commandPattern = /\bcommand\s*:\s*(['"])inspect-point\1/g; + for (const match of text.matchAll(commandPattern)) { + const commandCodePrefix = match[0].slice(0, match[0].indexOf(match[1])); + if (code.slice(match.index, match.index + commandCodePrefix.length) !== commandCodePrefix) { + continue; + } + + const openBrace = findEnclosingBrace(code, match.index); + if (openBrace < 0) continue; + const closeBrace = findMatchingDelimiter(code, openBrace, '{', '}'); + if (closeBrace < 0) continue; + const positionals = findDirectArrayProperty(text, code, openBrace, closeBrace, 'positionals'); + if (positionals !== undefined && countArrayTokens(text, code, positionals) > coordinateIndex) { + count += 1; + } } return count; } +function maskScenarioStringsAndComments(text) { + const chars = text.split(''); + let index = 0; + while (index < chars.length) { + const char = chars[index]; + const next = chars[index + 1]; + if (char === '/' && next === '/') { + while (index < chars.length && chars[index] !== '\n') chars[index++] = ' '; + continue; + } + if (char === '/' && next === '*') { + chars[index++] = ' '; + chars[index++] = ' '; + while (index < chars.length && !(chars[index] === '*' && chars[index + 1] === '/')) { + if (chars[index] !== '\n') chars[index] = ' '; + index += 1; + } + if (index < chars.length) { + chars[index++] = ' '; + chars[index++] = ' '; + } + continue; + } + if (char === '"' || char === "'" || char === '`') { + const quote = char; + chars[index++] = ' '; + while (index < chars.length) { + const current = chars[index]; + if (current === '\\') { + if (chars[index] !== '\n') chars[index] = ' '; + index += 2; + continue; + } + if (current === quote) { + chars[index++] = ' '; + break; + } + if (current !== '\n') chars[index] = ' '; + index += 1; + } + continue; + } + index += 1; + } + return chars.join(''); +} + +function findEnclosingBrace(code, position) { + const stack = []; + for (let index = 0; index < position; index += 1) { + if (code[index] === '{') stack.push(index); + else if (code[index] === '}') stack.pop(); + } + return stack.at(-1) ?? -1; +} + +function findMatchingDelimiter(code, openIndex, open, close) { + let depth = 0; + for (let index = openIndex; index < code.length; index += 1) { + if (code[index] === open) depth += 1; + else if (code[index] === close && --depth === 0) return index; + } + return -1; +} + +function findDirectArrayProperty(text, code, openBrace, closeBrace, name) { + const propertyPattern = new RegExp(`\\b${name}\\s*:\\s*\\[`, 'g'); + const body = code.slice(openBrace + 1, closeBrace); + for (const match of body.matchAll(propertyPattern)) { + const position = openBrace + 1 + match.index; + if (!isTopLevelObjectPosition(code, openBrace, position)) continue; + const openBracket = position + match[0].lastIndexOf('['); + const closeBracket = findMatchingDelimiter(code, openBracket, '[', ']'); + if (closeBracket >= 0 && closeBracket < closeBrace) return [openBracket, closeBracket]; + } + return undefined; +} + +function isTopLevelObjectPosition(code, openBrace, position) { + let depth = 0; + for (let index = openBrace + 1; index < position; index += 1) { + if (code[index] === '{' || code[index] === '[' || code[index] === '(') depth += 1; + else if (code[index] === '}' || code[index] === ']' || code[index] === ')') depth -= 1; + } + return depth === 0; +} + +function countArrayTokens(text, code, [openBracket, closeBracket]) { + const tokens = []; + let start = openBracket + 1; + let depth = 0; + for (let index = start; index < closeBracket; index += 1) { + if (code[index] === '{' || code[index] === '[' || code[index] === '(') depth += 1; + else if (code[index] === '}' || code[index] === ']' || code[index] === ')') depth -= 1; + else if (code[index] === ',' && depth === 0) { + tokens.push(text.slice(start, index)); + start = index + 1; + } + } + tokens.push(text.slice(start, closeBracket)); + return tokens.filter((token) => token.trim() !== "''" && token.trim() !== '""' && token.trim()) + .length; +} + function countFlagReferences(text, key) { const escaped = key.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); return text.match(new RegExp(`\\b${escaped}\\s*:`, 'g'))?.length ?? 0; From aaf82e133c37d1213d69dcfd07f6b5bde4d85b43 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 14:03:45 -0700 Subject: [PATCH 14/21] fix: distinguish blocked private AX snapshot reads --- .../RunnerAXSnapshotBridge.h | 1 + .../RunnerAXSnapshotBridge.m | 22 +++++++--- .../RunnerTests+AXSnapshotFallback.swift | 16 +++++++- .../RunnerTests+Interaction.swift | 3 +- .../RunnerTests+SnapshotCapturePlan.swift | 2 +- .../RunnerTests+AXSnapshotFallbackTests.swift | 41 +++++++++++-------- .../provider-scenarios/ios-lifecycle.test.ts | 10 +++++ .../provider-scenarios/ios-world.ts | 13 ++++++ 8 files changed, 83 insertions(+), 25 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h index fde9c73bca..6584cc2849 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h @@ -11,6 +11,7 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionCallsKey; FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionNodesAddedKey; FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionPendingKey; FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionMissedKey; +FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionBlockedKey; /// Keys of the `customActions` dictionary in the snapshot response: how many /// merged elements were eligible for an action read, and how many the bounded diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m index ab3f236592..f8ef7cc35e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m @@ -14,6 +14,7 @@ NSString *const RunnerAXSnapshotDeepExtensionNodesAddedKey = @"nodesAdded"; NSString *const RunnerAXSnapshotDeepExtensionPendingKey = @"pendingFrontiers"; NSString *const RunnerAXSnapshotDeepExtensionMissedKey = @"missedFrontiers"; +NSString *const RunnerAXSnapshotDeepExtensionBlockedKey = @"blockedFrontiers"; NSString *const RunnerAXSnapshotCustomActionsKey = @"customActions"; NSString *const RunnerAXSnapshotCustomActionsReadKey = @"read"; @@ -252,10 +253,11 @@ + (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray 0) { if (callsUsed >= callsAllowed || *nodeCount >= maxNodes || (nil != deadline && deadline.timeIntervalSinceNow <= 0)) { @@ -280,6 +282,14 @@ + (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray 0 || missedFrontiers > 0 } + + static func privateAXContainmentFailure(blockedFrontiers: Int?) -> SnapshotCaptureFailure? { + guard let blockedFrontiers, blockedFrontiers > 0 else { return nil } + return SnapshotCaptureFailure( + code: "IOS_SNAPSHOT_AX_CONTAINMENT_FAILED", + message: "The private AX snapshot could not complete a contained accessibility read.", + hint: "Retry after the accessibility server responds; no partial tree was treated as complete." + ) + } /// Deep React Native trees make the AX server reject bulk snapshot requests outright with /// kAXErrorIllegalArgument once the requested depth crosses a tree-size-dependent limit /// (observed between depth 56 and 64 on the Bluesky Home feed; the limit moves with live @@ -165,7 +174,7 @@ extension RunnerTests { target: SnapshotCaptureTarget, hint: CaptureHint, deadline: Date = .distantFuture - ) -> SnapshotAcquisition? { + ) throws -> SnapshotAcquisition? { #if os(iOS) && targetEnvironment(simulator) let app = target.app let requestedDepth = hint.rawTraversalDepth ?? 64 @@ -238,6 +247,11 @@ extension RunnerTests { // depth-limited would send agents chasing deeper content that is not // there. Pending or missed frontiers keep the depth-limited verdict. let deepExtension = response[RunnerAXSnapshotDeepExtensionKey] as? [String: Any] + if let failure = Self.privateAXContainmentFailure( + blockedFrontiers: deepExtension?[RunnerAXSnapshotDeepExtensionBlockedKey] as? Int + ) { + throw failure + } let depthLimited = Self.privateAXDepthLimited( effectiveDepth: effectiveDepth, requestedDepth: requestedDepth, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index f39ecfd32e..d33b6e3267 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -408,7 +408,8 @@ extension RunnerTests { if let pending = deepExtension[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, let missed = deepExtension[RunnerAXSnapshotDeepExtensionMissedKey] as? Int { - completeDeepExtension = pending == 0 && missed == 0 + let blocked = deepExtension[RunnerAXSnapshotDeepExtensionBlockedKey] as? Int ?? 0 + completeDeepExtension = pending == 0 && missed == 0 && blocked == 0 } else { completeDeepExtension = false } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 8ef6bc8ca7..085975dbab 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -492,7 +492,7 @@ extension RunnerTests { return (sweep.acquisition, sweep.outcome) case .privateAX: return ( - self.privateAXSnapshotAcquisition( + try self.privateAXSnapshotAcquisition( target: target, hint: hint, deadline: deadline diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index 724606aafa..56115355c9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -5,6 +5,15 @@ import AgentDeviceSnapshotPresentation // MARK: - In-bundle unit tests extension RunnerTests { + func testPrivateAXContainmentFailureIsTypedAndNotADepthLimit() { + XCTAssertNil(Self.privateAXContainmentFailure(blockedFrontiers: nil)) + XCTAssertNil(Self.privateAXContainmentFailure(blockedFrontiers: 0)) + XCTAssertEqual( + Self.privateAXContainmentFailure(blockedFrontiers: 1)?.code, + "IOS_SNAPSHOT_AX_CONTAINMENT_FAILED" + ) + } + func testPrivateAXAttemptDepthsAppliesRememberedDepth() { XCTAssertEqual( Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), @@ -24,12 +33,10 @@ extension RunnerTests { XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 24, rememberedDepth: 56), [24, 12]) } - /// Executed producer contract for the #1627 review blocker: a frontier whose - /// live element vanished, and one whose re-rooted request fails, must BOTH - /// count as missed — an all-miss extension reporting itself drained would - /// present a capped capture as complete. Goes red if either miss-path - /// increment in extendSnapshotFrontiers is removed. - func testDeepExtensionCountsMissedFrontiers() { + /// A vanished element is a missed frontier; a contained request failure is + /// a blocked frontier and must fail the backend instead of degrading to a + /// benign depth-limit verdict. + func testDeepExtensionSeparatesMissedAndBlockedFrontiers() { // Element vanished (list churn between serialization and extension): the // fabricated snapshot answers nil for accessibilityElement — missed, and // no request call is consumed. (An explicit nil property: bare NSObject @@ -38,7 +45,7 @@ extension RunnerTests { orphan.snapshot = FrontierSnapshotWithoutElementForTesting() orphan.node = NSMutableDictionary() // Re-rooted request fails: the element resolves but the client cannot - // serve requestSnapshotForElement — one consumed call AND a miss. + // serve requestSnapshotForElement — one consumed call AND a blocked read. let unreachable = RunnerAXSnapshotFrontier() unreachable.snapshot = FrontierSnapshotWithElementForTesting() unreachable.node = NSMutableDictionary() @@ -58,15 +65,13 @@ extension RunnerTests { deadline: nil ) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 2) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 1) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionBlockedKey] as? Int, 1) XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int, 1) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 0) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 1) XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int, 0) - XCTAssertFalse(truncated.boolValue) - // And the consumer verdict over exactly this outcome: still depth-limited. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 2)) + XCTAssertTrue(truncated.boolValue) + XCTAssertNotNil(Self.privateAXContainmentFailure(blockedFrontiers: 1)) } func testPrivateAXDepthLimitedRequiresEveryFrontierResolved() { @@ -543,9 +548,8 @@ extension RunnerTests { } } -/// Stands in for an AX client whose `attributesForElement:` never returns — -/// the wedged-server case the containment exists for. `release()` lets the -/// hung call finish so recovery is observable. +/// Stands in for a snapshot AX client whose request never returns. `release()` +/// lets the contained request finish so timeout recovery is observable. private final class HungSnapshotAXClientForTesting: NSObject { private let gate = DispatchSemaphore(value: 0) private let lock = NSLock() @@ -576,6 +580,9 @@ private final class HungSnapshotAXClientForTesting: NSObject { } } +/// Stands in for an AX client whose `attributesForElement:` never returns — +/// the wedged-server case the containment exists for. `release()` lets the +/// hung call finish so recovery is observable. private final class HungAXClientForTesting: NSObject { private let gate = DispatchSemaphore(value: 0) private let releasedOnce = NSLock() diff --git a/test/integration/provider-scenarios/ios-lifecycle.test.ts b/test/integration/provider-scenarios/ios-lifecycle.test.ts index 04a92b300c..957375f393 100644 --- a/test/integration/provider-scenarios/ios-lifecycle.test.ts +++ b/test/integration/provider-scenarios/ios-lifecycle.test.ts @@ -90,6 +90,16 @@ test('Provider-backed integration iOS Settings flow uses scripted simctl and run ], }, }, + { + name: 'reports an honest miss for an empty settings coordinate', + command: 'inspect-point', + positionals: ['0', '0'], + expectData: { + status: 'no-element-at-point', + point: { x: 0, y: 0 }, + elements: [], + }, + }, { name: 'reopen existing session app', command: 'open', diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index 612ae4820f..21eba2ab5c 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -66,6 +66,19 @@ export async function createIosSettingsWorld(): Promise { ], }, }, + { + command: 'ios.runner.readText', + deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + platform: 'apple', + request: { + command: 'readText', + inspectPoint: true, + x: 0, + y: 0, + appBundleId: 'com.apple.Preferences', + }, + result: { elements: [] }, + }, { command: 'ios.runner.tap', deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, From f258c2ae98d349f13062db0de3153d5e9bd1275d Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 14:06:24 -0700 Subject: [PATCH 15/21] test: harden point inspection coverage --- .../interactor-runner-provider.test.ts | 11 ++++++++ scripts/integration-progress-model.test.ts | 12 +++++++++ scripts/integration-progress-model.ts | 27 +++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index d618fbe1b5..56f319c93c 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -156,6 +156,17 @@ test('inspectPoint rejects a malformed runner payload instead of reporting an ho ); }); +test('inspectPoint preserves an empty elements payload as an honest miss', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + recordingRunnerProvider([], { readText: { elements: [] } }), + ); + + const result = await interactor.inspectPoint!({ x: 10, y: 20 }); + assert.deepEqual(result, { elements: [] }); +}); + test('inspectPoint rejects non-finite accessibility frames', async () => { const interactor = createAppleInteractor( IOS_SIMULATOR, diff --git a/scripts/integration-progress-model.test.ts b/scripts/integration-progress-model.test.ts index 560ed4e7ab..8a19c05acc 100644 --- a/scripts/integration-progress-model.test.ts +++ b/scripts/integration-progress-model.test.ts @@ -28,6 +28,18 @@ test('integration progress scopes point coordinates to each inspect-point step o assert.equal(countInspectPointCoordinateReferences(source, 1), 1); }); +test('integration progress ignores regex literals while finding inspect-point steps', () => { + const source = String.raw` + const steps = [ + { command: 'inspect-point', positionals: ['1', '2'], assert: (response) => /[{]/.test(response.json) }, + { command: 'inspect-point', positionals: ['3', '4'] }, + ]; + `; + + assert.equal(countInspectPointCoordinateReferences(source, 0), 2); + assert.equal(countInspectPointCoordinateReferences(source, 1), 2); +}); + test('integration progress counts explicit generic Apple host-tool usage only', async () => { const root = await mkdtemp(path.join(os.tmpdir(), 'agent-device-progress-')); try { diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 53bfa35279..a523b49837 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -263,6 +263,27 @@ function maskScenarioStringsAndComments(text) { } continue; } + if (char === '/' && beginsRegexLiteral(chars, index)) { + chars[index++] = ' '; + let inCharacterClass = false; + while (index < chars.length && chars[index] !== '\n') { + const current = chars[index]; + if (current === '\\') { + chars[index++] = ' '; + if (index < chars.length && chars[index] !== '\n') chars[index++] = ' '; + continue; + } + if (current === '[') inCharacterClass = true; + if (current === ']') inCharacterClass = false; + if (current === '/' && !inCharacterClass) { + chars[index++] = ' '; + while (index < chars.length && /[a-z]/i.test(chars[index])) chars[index++] = ' '; + break; + } + chars[index++] = ' '; + } + continue; + } if (char === '"' || char === "'" || char === '`') { const quote = char; chars[index++] = ' '; @@ -287,6 +308,12 @@ function maskScenarioStringsAndComments(text) { return chars.join(''); } +function beginsRegexLiteral(chars, index) { + let previous = index - 1; + while (previous >= 0 && /\s/.test(chars[previous])) previous -= 1; + return previous < 0 || /[(:,=!?&|;{[>]/.test(chars[previous]); +} + function findEnclosingBrace(code, position) { const stack = []; for (let index = 0; index < position; index += 1) { From eb2bef05edf572afeb3e74a723ca98b3819a4590 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 15:47:45 -0700 Subject: [PATCH 16/21] fix(apple): preserve point inspection text priority --- .../RunnerTests+Interaction.swift | 21 +++++----- .../RunnerTests+PrivateAXPresentation.swift | 4 +- .../RunnerTests+InteractionTests.swift | 38 +++++++++++++++++++ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index d33b6e3267..e1ddcdfc94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -438,7 +438,8 @@ extension RunnerTests { func visit(_ raw: [String: Any]) { let frame = privateAXRect(raw["frame"]) if !frame.isEmpty && frame.contains(point) { - let rawType = (raw["type"] as? NSNumber)?.intValue ?? 0 + let fields = privateAXFields(raw) + let rawType = fields.rawType // The private tree includes the owning application at the full screen // frame. Its label is the app name, not a control under the point, so // returning it can make point-based system-UI probes tap a false match. @@ -448,10 +449,10 @@ extension RunnerTests { } return } - let type = Self.elementTypeNamesByRawValue[UInt(rawType)] ?? "Element(\(rawType))" - let label = pointInspectionText(raw["label"]) - let identifier = pointInspectionText(raw["identifier"]) - let value = pointInspectionText(raw["value"]) + let type = fields.elementType.map(elementTypeName) ?? "Element(\(rawType))" + let label = fields.label.isEmpty ? nil : fields.label + let identifier = fields.identifier.isEmpty ? nil : fields.identifier + let value = fields.value.isEmpty ? nil : fields.value let text = pointInspectionReadableText( type: type, label: label, @@ -491,12 +492,10 @@ extension RunnerTests { } .prefix(24) .map(\.payload) - return (elements.compactMap(\.text).first, elements, true) - } - - private func pointInspectionText(_ value: Any?) -> String? { - let text = (value as? String ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - return text.isEmpty ? nil : text + let textInputTypes: Set = ["TextField", "SecureTextField", "SearchField", "TextView"] + let text = elements.first(where: { textInputTypes.contains($0.type ?? "") && $0.text != nil })?.text + ?? elements.compactMap(\.text).first + return (text, elements, true) } private func pointInspectionReadableText( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index f6365d592c..e08b229b2d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -46,7 +46,7 @@ extension RunnerTests { } } - private func privateAXFields(_ raw: [String: Any]) -> PrivateAXFields { + func privateAXFields(_ raw: [String: Any]) -> PrivateAXFields { let rawType = privateAXPresentationInt(raw["type"]) ?? 0 return PrivateAXFields( rect: privateAXRect(raw["frame"]), @@ -80,7 +80,7 @@ extension RunnerTests { actions: fields.actions) } - private func privateAXPresentationString(_ value: Any?) -> String { + func privateAXPresentationString(_ value: Any?) -> String { guard let value else { return "" } return (value as? String ?? String(describing: value)) .trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index 4676534e16..9843bc450b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -52,6 +52,44 @@ extension RunnerTests { XCTAssertNil(inspection.elements.first?.hittable) } + func testPrivateAXPointInspectionPrefersTextInputValueAndDescribesNumericValues() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 400, "height": 800], + "children": [[ + "type": NSNumber(value: XCUIElement.ElementType.textField.rawValue), + "label": "Search", + "identifier": "search-field", + "value": "long live field value", + "frame": ["x": 100, "y": 500, "width": 200, "height": 48], + "children": [[ + "type": NSNumber(value: XCUIElement.ElementType.button.rawValue), + "label": "Clear", + "identifier": "clear-button", + "value": "", + "frame": ["x": 250, "y": 510, "width": 24, "height": 24], + "children": [], + ]], + ], [ + "type": NSNumber(value: XCUIElement.ElementType.switch.rawValue), + "label": "Enabled", + "identifier": "enabled-switch", + "value": NSNumber(value: 1), + "frame": ["x": 255, "y": 515, "width": 10, "height": 10], + "children": [], + ]], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 260, y: 520)) + + XCTAssertEqual(inspection.text, "long live field value") + XCTAssertEqual(inspection.elements.first?.identifier, "clear-button") + XCTAssertEqual(inspection.elements.last(where: { $0.identifier == "enabled-switch" })?.value, "1") + } + func testPrivateAXPointInspectionReturnsNoElementForHonestMiss() { let root: [String: Any] = [ "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), From 2d12842ab784c0716eb00c053fa98b37e5ccfd8d Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 16:10:04 -0700 Subject: [PATCH 17/21] test: fix point inspection ordering assertion --- .../UnitTests/RunnerTests+InteractionTests.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index 9843bc450b..4cccb98361 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -86,7 +86,8 @@ extension RunnerTests { let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 260, y: 520)) XCTAssertEqual(inspection.text, "long live field value") - XCTAssertEqual(inspection.elements.first?.identifier, "clear-button") + XCTAssertEqual(inspection.elements.first?.identifier, "enabled-switch") + XCTAssertTrue(inspection.elements.contains { $0.identifier == "clear-button" }) XCTAssertEqual(inspection.elements.last(where: { $0.identifier == "enabled-switch" })?.value, "1") } From d5ab5ccd33ccb14b115b52528be20de11e640734 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 17:22:35 -0700 Subject: [PATCH 18/21] fix: pass inspect-point validation gates --- scripts/integration-progress-model.ts | 167 +++++++---- src/cli/parser/args.ts | 296 +++++++++++++------- test/integration/smoke-web-platform.test.ts | 20 +- test/wire-compat/ledger.json | 40 +++ 4 files changed, 351 insertions(+), 172 deletions(-) diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index a523b49837..c63513951f 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -244,68 +244,102 @@ function maskScenarioStringsAndComments(text) { const chars = text.split(''); let index = 0; while (index < chars.length) { - const char = chars[index]; - const next = chars[index + 1]; - if (char === '/' && next === '/') { - while (index < chars.length && chars[index] !== '\n') chars[index++] = ' '; - continue; - } - if (char === '/' && next === '*') { - chars[index++] = ' '; - chars[index++] = ' '; - while (index < chars.length && !(chars[index] === '*' && chars[index + 1] === '/')) { - if (chars[index] !== '\n') chars[index] = ' '; - index += 1; - } - if (index < chars.length) { - chars[index++] = ' '; - chars[index++] = ' '; - } - continue; + const end = + maskComment(chars, index) ?? maskRegex(chars, index) ?? maskQuotedText(chars, index); + if (end === null) { + index += 1; + } else { + index = end; } - if (char === '/' && beginsRegexLiteral(chars, index)) { - chars[index++] = ' '; - let inCharacterClass = false; - while (index < chars.length && chars[index] !== '\n') { - const current = chars[index]; - if (current === '\\') { - chars[index++] = ' '; - if (index < chars.length && chars[index] !== '\n') chars[index++] = ' '; - continue; - } - if (current === '[') inCharacterClass = true; - if (current === ']') inCharacterClass = false; - if (current === '/' && !inCharacterClass) { - chars[index++] = ' '; - while (index < chars.length && /[a-z]/i.test(chars[index])) chars[index++] = ' '; - break; - } - chars[index++] = ' '; - } + } + return chars.join(''); +} + +function maskComment(chars, index) { + if (chars[index] !== '/') return null; + if (chars[index + 1] === '/') return maskLineComment(chars, index); + if (chars[index + 1] === '*') return maskBlockComment(chars, index); + return null; +} + +function maskLineComment(chars, index) { + while (index < chars.length && chars[index] !== '\n') chars[index++] = ' '; + return index; +} + +function maskBlockComment(chars, index) { + chars[index++] = ' '; + chars[index++] = ' '; + while (index < chars.length && !(chars[index] === '*' && chars[index + 1] === '/')) { + if (chars[index] !== '\n') chars[index] = ' '; + index += 1; + } + return index < chars.length ? maskCommentEnd(chars, index) : index; +} + +function maskCommentEnd(chars, index) { + chars[index++] = ' '; + chars[index++] = ' '; + return index; +} + +function maskRegex(chars, index) { + if (chars[index] !== '/' || !beginsRegexLiteral(chars, index)) return null; + chars[index++] = ' '; + return scanRegexBody(chars, index); +} + +function scanRegexBody(chars, index) { + let inCharacterClass = false; + while (index < chars.length && chars[index] !== '\n') { + const current = chars[index]; + if (current === '\\') { + index = maskEscapedCharacter(chars, index); continue; } - if (char === '"' || char === "'" || char === '`') { - const quote = char; - chars[index++] = ' '; - while (index < chars.length) { - const current = chars[index]; - if (current === '\\') { - if (chars[index] !== '\n') chars[index] = ' '; - index += 2; - continue; - } - if (current === quote) { - chars[index++] = ' '; - break; - } - if (current !== '\n') chars[index] = ' '; - index += 1; - } - continue; + inCharacterClass = updateCharacterClass(current, inCharacterClass); + if (current === '/' && !inCharacterClass) return maskRegexEnd(chars, index); + chars[index++] = ' '; + } + return index; +} + +function updateCharacterClass(character, inCharacterClass) { + if (character === '[') return true; + if (character === ']') return false; + return inCharacterClass; +} + +function maskEscapedCharacter(chars, index) { + chars[index++] = ' '; + if (index < chars.length && chars[index] !== '\n') chars[index++] = ' '; + return index; +} + +function maskRegexEnd(chars, index) { + chars[index++] = ' '; + while (index < chars.length && /[a-z]/i.test(chars[index])) chars[index++] = ' '; + return index; +} + +function maskQuotedText(chars, index) { + const quote = chars[index]; + if (quote !== '"' && quote !== "'" && quote !== '`') return null; + chars[index++] = ' '; + while (index < chars.length && chars[index] !== quote) { + if (chars[index] === '\\') { + index = maskEscapedCharacter(chars, index); + } else { + if (chars[index] !== '\n') chars[index] = ' '; + index += 1; } - index += 1; } - return chars.join(''); + return index < chars.length ? index + maskQuoteEnd(chars, index) : index; +} + +function maskQuoteEnd(chars, index) { + chars[index] = ' '; + return 1; } function beginsRegexLiteral(chars, index) { @@ -359,16 +393,25 @@ function countArrayTokens(text, code, [openBracket, closeBracket]) { let start = openBracket + 1; let depth = 0; for (let index = start; index < closeBracket; index += 1) { - if (code[index] === '{' || code[index] === '[' || code[index] === '(') depth += 1; - else if (code[index] === '}' || code[index] === ']' || code[index] === ')') depth -= 1; - else if (code[index] === ',' && depth === 0) { + depth = updateDelimiterDepth(code[index], depth); + if (code[index] === ',' && depth === 0) { tokens.push(text.slice(start, index)); start = index + 1; } } tokens.push(text.slice(start, closeBracket)); - return tokens.filter((token) => token.trim() !== "''" && token.trim() !== '""' && token.trim()) - .length; + return tokens.filter(isPresentArrayToken).length; +} + +function updateDelimiterDepth(character, depth) { + if (character === '{' || character === '[' || character === '(') return depth + 1; + if (character === '}' || character === ']' || character === ')') return depth - 1; + return depth; +} + +function isPresentArrayToken(token) { + const trimmed = token.trim(); + return trimmed.length > 0 && trimmed !== "''" && trimmed !== '""'; } function countFlagReferences(text, key) { diff --git a/src/cli/parser/args.ts b/src/cli/parser/args.ts index 23690f3a41..80fc1ccaf7 100644 --- a/src/cli/parser/args.ts +++ b/src/cli/parser/args.ts @@ -51,77 +51,92 @@ export function parseArgs(argv: string[], options?: FinalizeArgsOptions): Parsed } export function parseRawArgs(argv: string[]): RawParsedArgs { - const flags: CliFlags = { json: false, help: false, version: false }; - let command: string | null = null; - let rawCommand: string | null = null; - const positionals: string[] = []; - const warnings: string[] = []; - const providedFlags: ParsedFlagRecord[] = []; - let parseFlags = true; - - for (let i = 0; i < argv.length; i += 1) { - const arg = argv[i]!; - if (parseFlags && arg === '--') { - parseFlags = false; - continue; - } - if (!parseFlags) { - if (!command) { - rawCommand = arg; - command = normalizeCommandAlias(arg); - } else positionals.push(arg); - continue; - } - if (shouldPreservePostCommandArgs(command)) { - positionals.push(arg); - continue; - } - const isLongFlag = arg.startsWith('--'); - const isShortFlag = arg.startsWith('-') && arg.length > 1; - if (!isLongFlag && !isShortFlag) { - if (!command) { - rawCommand = arg; - command = normalizeCommandAlias(arg); - } else positionals.push(arg); - continue; - } + const state: RawParseState = { + command: null, + flags: { json: false, help: false, version: false }, + parseFlags: true, + positionals: [], + providedFlags: [], + rawCommand: null, + warnings: [], + }; + for (let index = 0; index < argv.length; index += parseRawArgument(state, argv, index) + 1) { + // parseRawArgument returns one extra consumed argument when a value follows its flag. + } + applyAliasImpliedFlags(state.rawCommand, state.flags); + return state; +} - const [token, inlineValue] = isLongFlag ? splitLongFlag(arg) : [arg, undefined]; - if (isLegacyIgnoredSnapshotShortFlag(command, token)) { - continue; - } - const definition = resolveFlagDefinition(token, command); - if (shouldPassThroughLocalToolFlag(command, definition)) { - positionals.push(arg); - continue; - } - if (!definition) { - if (shouldTreatUnknownDashTokenAsPositional(command, positionals, arg)) { - if (!command) command = arg; - else positionals.push(arg); - continue; - } - throw new AppError('INVALID_ARGS', formatUnknownFlagMessage(token, command)); - } +type RawParseState = RawParsedArgs & { rawCommand: string | null; parseFlags: boolean }; + +function parseRawArgument(state: RawParseState, argv: string[], index: number): number { + const arg = argv[index]!; + if (arg === '--' && state.parseFlags) { + state.parseFlags = false; + return 0; + } + if (appendRawPositional(state, arg)) return 0; + return parseRawFlag(state, arg, argv[index + 1]); +} - const parsed = parseFlagValue(definition, token, inlineValue, argv[i + 1]); - if (parsed.consumeNext) i += 1; - const existingValue = (flags as Record)[definition.key]; - if (definition.multiple) { - const values = Array.isArray(existingValue) - ? [...existingValue, parsed.value] - : existingValue === undefined - ? [parsed.value] - : [existingValue, parsed.value]; - (flags as Record)[definition.key] = values; +function appendRawPositional(state: RawParseState, arg: string): boolean { + if (!state.parseFlags || shouldPreservePostCommandArgs(state.command) || !isFlagToken(arg)) { + if (!state.command) { + state.rawCommand = arg; + state.command = normalizeCommandAlias(arg); } else { - (flags as Record)[definition.key] = parsed.value; + state.positionals.push(arg); } - providedFlags.push({ key: definition.key, token }); + return true; + } + return false; +} + +function isFlagToken(arg: string): boolean { + return arg.startsWith('--') || (arg.startsWith('-') && arg.length > 1); +} + +function parseRawFlag(state: RawParseState, arg: string, nextArg: string | undefined): number { + const [token, inlineValue] = arg.startsWith('--') ? splitLongFlag(arg) : [arg, undefined]; + if (isLegacyIgnoredSnapshotShortFlag(state.command, token)) return 0; + const definition = resolveFlagDefinition(token, state.command); + if (shouldPassThroughLocalToolFlag(state.command, definition)) { + state.positionals.push(arg); + return 0; + } + if (!definition) return parseUnknownRawFlag(state, token, arg); + const parsed = parseFlagValue(definition, token, inlineValue, nextArg); + appendParsedFlag(state, definition, token, parsed.value); + return Number(parsed.consumeNext); +} + +function parseUnknownRawFlag(state: RawParseState, token: string, arg: string): number { + if (shouldTreatUnknownDashTokenAsPositional(state.command, state.positionals, arg)) { + if (!state.command) state.command = arg; + else state.positionals.push(arg); + return 0; } + throw new AppError('INVALID_ARGS', formatUnknownFlagMessage(token, state.command)); +} + +function appendParsedFlag( + state: RawParseState, + definition: FlagDefinition, + token: string, + value: unknown, +): void { + const flags = state.flags as Record; + const existingValue = flags[definition.key]; + flags[definition.key] = definition.multiple + ? appendMultipleFlagValue(existingValue, value) + : value; + state.providedFlags.push({ key: definition.key, token }); +} - applyAliasImpliedFlags(rawCommand, flags); - return { command, positionals, flags, warnings, providedFlags }; +function appendMultipleFlagValue(existingValue: unknown, value: unknown): unknown[] { + if (Array.isArray(existingValue)) return [...existingValue, value]; + if (existingValue === undefined) return [value]; + return [existingValue, value]; } function applyAliasImpliedFlags(rawCommand: string | null, flags: CliFlags): void { @@ -259,68 +274,131 @@ function parseFlagValue( inlineValue: string | undefined, nextArg: string | undefined, ): { value: unknown; consumeNext: boolean } { - if (definition.setValue !== undefined) { - if (inlineValue !== undefined) { - throw new AppError('INVALID_ARGS', `Flag ${token} does not take a value.`); - } - return { value: definition.setValue, consumeNext: false }; + if (definition.setValue !== undefined) return parseSetValue(definition, token, inlineValue); + if (definition.type === 'boolean') return parseBooleanValue(token, inlineValue); + if (definition.type === 'booleanOrString') { + return parseBooleanOrStringValue(token, inlineValue, nextArg); } - if (definition.type === 'boolean') { - if (inlineValue !== undefined) { - throw new AppError('INVALID_ARGS', `Flag ${token} does not take a value.`); - } - return { value: true, consumeNext: false }; + return parseRequiredFlagValue(definition, token, inlineValue, nextArg); +} + +function parseSetValue( + definition: FlagDefinition, + token: string, + inlineValue: string | undefined, +): { value: unknown; consumeNext: boolean } { + assertNoInlineValue(token, inlineValue); + return { value: definition.setValue, consumeNext: false }; +} + +function parseBooleanValue( + token: string, + inlineValue: string | undefined, +): { value: unknown; consumeNext: boolean } { + assertNoInlineValue(token, inlineValue); + return { value: true, consumeNext: false }; +} + +function assertNoInlineValue(token: string, inlineValue: string | undefined): void { + if (inlineValue !== undefined) { + throw new AppError('INVALID_ARGS', `Flag ${token} does not take a value.`); } - if (definition.type === 'booleanOrString') { - if (inlineValue !== undefined) { - if (inlineValue.trim().length === 0) { - throw new AppError( - 'INVALID_ARGS', - `Flag ${token} requires a non-empty value when provided.`, - ); - } - return { value: inlineValue, consumeNext: false }; - } - if (nextArg === undefined || looksLikeFlagToken(nextArg)) { - return { value: true, consumeNext: false }; - } - if (shouldConsumeOptionalPathValue(nextArg)) { - return { value: nextArg, consumeNext: true }; - } +} + +function parseBooleanOrStringValue( + token: string, + inlineValue: string | undefined, + nextArg: string | undefined, +): { value: unknown; consumeNext: boolean } { + if (inlineValue !== undefined) return parseInlineBooleanOrString(token, inlineValue); + if (nextArg === undefined || looksLikeFlagToken(nextArg)) { return { value: true, consumeNext: false }; } + return shouldConsumeOptionalPathValue(nextArg) + ? { value: nextArg, consumeNext: true } + : { value: true, consumeNext: false }; +} +function parseInlineBooleanOrString( + token: string, + value: string, +): { value: unknown; consumeNext: boolean } { + if (value.trim().length === 0) { + throw new AppError('INVALID_ARGS', `Flag ${token} requires a non-empty value when provided.`); + } + return { value, consumeNext: false }; +} + +function parseRequiredFlagValue( + definition: FlagDefinition, + token: string, + inlineValue: string | undefined, + nextArg: string | undefined, +): { value: unknown; consumeNext: boolean } { const value = inlineValue ?? nextArg; - if (value === undefined) { - throw new AppError('INVALID_ARGS', `Flag ${token} requires a value.`); + assertRequiredFlagValue(token, value, inlineValue); + if (definition.type === 'string') return { value, consumeNext: inlineValue === undefined }; + if (definition.type === 'enum') { + return parseEnumFlagValue(definition, token, value, inlineValue === undefined); } - if (inlineValue === undefined && looksLikeFlagToken(value)) { + return parseNumericFlagValue(definition, token, value, inlineValue === undefined); +} + +function assertRequiredFlagValue( + token: string, + value: string | undefined, + inlineValue: string | undefined, +): asserts value is string { + if (value === undefined || (inlineValue === undefined && looksLikeFlagToken(value))) { throw new AppError('INVALID_ARGS', `Flag ${token} requires a value.`); } +} - if (definition.type === 'string') { - return { value, consumeNext: inlineValue === undefined }; - } - if (definition.type === 'enum') { - if (!definition.enumValues?.includes(value)) { - throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); - } - return { value, consumeNext: inlineValue === undefined }; +function parseEnumFlagValue( + definition: FlagDefinition, + token: string, + value: string, + consumeNext: boolean, +): { value: unknown; consumeNext: boolean } { + if (!definition.enumValues?.includes(value)) { + throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); } + return { value, consumeNext }; +} + +function parseNumericFlagValue( + definition: FlagDefinition, + token: string, + value: string, + consumeNext: boolean, +): { value: unknown; consumeNext: boolean } { const parsed = Number(value); if (value.trim().length === 0 || !Number.isFinite(parsed)) { - throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); + return invalidNumericFlagValue(token, value); } + assertNumericBounds(definition, token, value, parsed); + return { + value: definition.type === 'int' ? Math.floor(parsed) : parsed, + consumeNext, + }; +} + +function invalidNumericFlagValue(token: string, value: string): never { + throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); +} + +function assertNumericBounds( + definition: FlagDefinition, + token: string, + value: string, + parsed: number, +): void { if (typeof definition.min === 'number' && parsed < definition.min) { - throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); + invalidNumericFlagValue(token, value); } if (typeof definition.max === 'number' && parsed > definition.max) { - throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); + invalidNumericFlagValue(token, value); } - return { - value: definition.type === 'int' ? Math.floor(parsed) : parsed, - consumeNext: inlineValue === undefined, - }; } function labelForFlag(token: string): string { diff --git a/test/integration/smoke-web-platform.test.ts b/test/integration/smoke-web-platform.test.ts index 7100ea9e6b..d08accadcf 100644 --- a/test/integration/smoke-web-platform.test.ts +++ b/test/integration/smoke-web-platform.test.ts @@ -62,7 +62,7 @@ test('web shutdown cleanup reaps the exact daemon that survived graceful shutdow const daemonPid = child.pid ?? 0; assert.ok(daemonPid > 0, 'expected the fake daemon to have a pid'); t.after(() => { - if (isProcessAlive(daemonPid)) process.kill(daemonPid, 'SIGKILL'); + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); rmSync(root, { recursive: true, force: true }); }); @@ -101,9 +101,27 @@ test('web shutdown cleanup reaps the exact daemon that survived graceful shutdow true, 'expected cleanup to escalate after the child ignored SIGTERM', ); + await waitForChildExit(child, 1_000); assert.equal(isProcessAlive(daemonPid), false); }); +async function waitForChildExit(child: ReturnType, timeoutMs: number): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + child.off('exit', onExit); + reject( + new Error(`child process ${child.pid ?? ''} did not exit within ${timeoutMs}ms`), + ); + }, timeoutMs); + const onExit = () => { + clearTimeout(timeout); + resolve(); + }; + child.once('exit', onExit); + }); +} + type StepRecord = { step: string; command: string; diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 181e04ba82..2a914c27e5 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -300,6 +300,46 @@ "declaration": "packages/kernel/src/contracts.ts#daemonRuntimeSchema", "digest": "sha256:3b99926ce9deb5d66186a55b1168011dc1ae4d010ada5ac5d79c4c7225a1f80f", "rationale": "#2266 broadens runtime-hint validation to accept the additive HarmonyOS platform value; existing protocol-2 runtime hints remain valid and unchanged." + }, + { + "declaration": "src/daemon/server/http-server.ts#writeProgressEnvelope", + "digest": "sha256:ca661f6dd2de6e517c520dbd1e20b59b24b58d925157ed06123149ed2bc0fa3b", + "rationale": "The post-v0.21.16 change adjusts only when the local HTTP response writer skips a progress event (`writableEnded` is no longer an early-return condition). Emitted events still use the same `serializeDaemonProgressEnvelope` bytes and framing; no protocol-2 payload field or interpretation changes." + }, + { + "declaration": "src/remote/upload-client.ts#finalizeDirectUpload", + "digest": "sha256:5e32c87214f9f6140535b48d05de8565cb121370fdc334c05f7f0ad476b73a07", + "rationale": "The post-v0.21.16 change removes caller cancellation plumbing from this local upload client. It still sends the same POST to `upload/finalize` with `{ uploadId }` and parses the same response; a protocol-2 daemon sees the identical request and response contract." + }, + { + "declaration": "src/remote/upload-client.ts#requestUploadPreflight", + "digest": "sha256:df42c0eaef3a54f4388c99d74bf6030885239780c2ddc78117151dbd391afedd", + "rationale": "The post-v0.21.16 change removes caller cancellation plumbing from this local upload client. The preflight URL, method, headers, JSON body, response parser, and fallback to legacy upload are unchanged, so protocol-2 peers exchange the same bytes." + }, + { + "declaration": "src/remote/upload-client.ts#tryDirectUploadWithResume", + "digest": "sha256:0f57cf9bb1eefa3b988e5d723b2eebf2c4a622c3f7c808f35eae070771acafa6", + "rationale": "The post-v0.21.16 change removes caller cancellation plumbing from the local retry orchestration. Attempt IDs, preflight/finalize requests, resume behavior, and legacy fallback retain their existing protocol-2 shapes; the daemon contract is unchanged." + }, + { + "declaration": "src/remote/upload-client.ts#uploadDirectArtifact", + "digest": "sha256:5f26f8f7b6ada2fe2089a214fb8350e3a9f65725212952c17f6787b3ca8fe232", + "rationale": "The post-v0.21.16 change removes only the caller-provided AbortSignal from this local direct-upload request. The ticket URL, PUT method, headers, streamed bytes, and server response contract are unchanged for protocol-2 peers." + }, + { + "declaration": "src/remote/upload-client.ts#uploadLegacyArtifact", + "digest": "sha256:f3ccab9c5ba18fc2f2000ea8e2a4bbb3cc97f7b50f6111192e513013f635cdd7", + "rationale": "The post-v0.21.16 change removes only the caller-provided AbortSignal from this local fallback request. The legacy upload URL, method, headers, multipart body, and response parser remain unchanged for protocol-2 peers." + }, + { + "declaration": "src/remote/upload-stream.ts#streamFileToHttpRequest", + "digest": "sha256:eaf2ea49957034f6a7e17d34092dc6cb1998812bddfd68381a0ac34d8a2de58c", + "rationale": "The post-v0.21.16 change removes optional cancellation wiring from the local HTTP upload helper; it does not alter URL construction, request method, headers, body bytes, redirect handling, or response interpretation. The upload wire remains protocol-2 compatible." + }, + { + "declaration": "src/remote/upload-stream.ts#streamFileToHttpRequestAttempt", + "digest": "sha256:da39a79fa7c1f81e55caf613eedc347c0f3a9a9711b265a4db185677532d9552", + "rationale": "The post-v0.21.16 change stops attaching the caller AbortSignal to the local Node HTTP request and removes its cancellation-specific error branch. The serialized upload request and daemon response handling are otherwise unchanged, so a protocol-2 peer parses the same contract." } ] } From c5f0dce82206c0148544bad20d9cd5d709261922 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 18:42:20 -0700 Subject: [PATCH 19/21] fix(ios): use contained AX reads for text values --- .../RunnerTests+Interaction.swift | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index e1ddcdfc94..e6eb10855e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -318,6 +318,19 @@ extension RunnerTests { } func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { +#if os(iOS) && targetEnvironment(simulator) + // Prefer the contained Simulator AX read when it can prove a value. Besides + // avoiding the unbounded XCTest descendant queries below for populated + // text fields, this uses the same text-input-first policy as that fallback. + // An incomplete capture never answers the request; it falls through to the + // legacy XCTest path so a capped tree cannot turn a real value into a miss. + if let inspection = privateAXPointInspection(app: app, x: x, y: y), + inspection.complete, + let text = inspection.text + { + return text + } +#endif let point = CGPoint(x: x, y: y) let textInputCandidates = textInputCandidatesAt(app: app, point: point) let candidates = app.descendants(matching: .any).allElementsBoundByIndex From 27d95de520290709aaa143838f22efc38f1c496c Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 18:56:44 -0700 Subject: [PATCH 20/21] fix(apple): keep bounded point text semantics --- .../RunnerTests+Interaction.swift | 44 +++++++-------- .../RunnerTests+InteractionTests.swift | 53 +++++++++++++++++++ 2 files changed, 75 insertions(+), 22 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index e6eb10855e..e84f91c9c1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -324,11 +324,11 @@ extension RunnerTests { // text fields, this uses the same text-input-first policy as that fallback. // An incomplete capture never answers the request; it falls through to the // legacy XCTest path so a capped tree cannot turn a real value into a miss. - if let inspection = privateAXPointInspection(app: app, x: x, y: y), - inspection.complete, - let text = inspection.text - { - return text + if let inspection = privateAXPointInspection(app: app, x: x, y: y), inspection.complete { + // A complete miss is authoritative too. Falling through would repeat a + // full XCTest descendant walk after the bounded AX capture already + // established that no readable element contains the point. + return inspection.text } #endif let point = CGPoint(x: x, y: y) @@ -466,7 +466,7 @@ extension RunnerTests { let label = fields.label.isEmpty ? nil : fields.label let identifier = fields.identifier.isEmpty ? nil : fields.identifier let value = fields.value.isEmpty ? nil : fields.value - let text = pointInspectionReadableText( + let text = pointReadableText( type: type, label: label, identifier: identifier, @@ -492,7 +492,7 @@ extension RunnerTests { } visit(root) - let elements = candidates + let orderedCandidates = candidates .sorted { left, right in if left.area != right.area { return left.area < right.area } if left.payload.frame.y != right.payload.frame.y { @@ -503,15 +503,19 @@ extension RunnerTests { } return (left.payload.type ?? "") < (right.payload.type ?? "") } - .prefix(24) - .map(\.payload) let textInputTypes: Set = ["TextField", "SecureTextField", "SearchField", "TextView"] - let text = elements.first(where: { textInputTypes.contains($0.type ?? "") && $0.text != nil })?.text - ?? elements.compactMap(\.text).first + // Keep text resolution independent of the bounded descriptor response. + // A large accessibility surface can have more than 24 smaller controls at + // one point while the underlying text input still owns the correct value. + let text = orderedCandidates.first(where: { + textInputTypes.contains($0.payload.type ?? "") && $0.payload.text != nil + })?.payload.text + ?? orderedCandidates.compactMap { $0.payload.text }.first + let elements = orderedCandidates.prefix(24).map(\.payload) return (text, elements, true) } - private func pointInspectionReadableText( + func pointReadableText( type: String, label: String?, identifier: String?, @@ -529,16 +533,12 @@ extension RunnerTests { let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) let valueText = String(describing: element.value ?? "") .trimmingCharacters(in: .whitespacesAndNewlines) - switch element.elementType { - case .textField, .secureTextField, .searchField, .textView: - if !valueText.isEmpty { return valueText } - if !label.isEmpty { return label } - return identifier.isEmpty ? nil : identifier - default: - if !label.isEmpty { return label } - if !valueText.isEmpty { return valueText } - return identifier.isEmpty ? nil : identifier - } + return pointReadableText( + type: elementTypeName(element.elementType), + label: label.isEmpty ? nil : label, + identifier: identifier.isEmpty ? nil : identifier, + value: valueText.isEmpty ? nil : valueText + ) } private func prefersExpandedTextRead(_ element: XCUIElement) -> Bool { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index 4cccb98361..2d082dbd88 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -91,6 +91,59 @@ extension RunnerTests { XCTAssertEqual(inspection.elements.last(where: { $0.identifier == "enabled-switch" })?.value, "1") } + func testPrivateAXPointTextResolutionIsNotLimitedByDescriptorCap() { + let smallerControls: [[String: Any]] = (0..<30).map { index in + [ + "type": NSNumber(value: XCUIElement.ElementType.button.rawValue), + "label": "Control \(index)", + "identifier": "control-\(index)", + "value": "", + "frame": ["x": 200, "y": 200, "width": 10, "height": 10], + "children": [], + ] + } + let field: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.textField.rawValue), + "label": "Full name", + "identifier": "full-name", + "value": "Alexandria Alexandra", + "frame": ["x": 100, "y": 180, "width": 200, "height": 40], + "children": [], + ] + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 400, "height": 800], + "children": smallerControls + [field], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 205, y: 205)) + + XCTAssertTrue(inspection.complete) + XCTAssertEqual(inspection.elements.count, 24) + XCTAssertFalse(inspection.elements.contains { $0.identifier == "full-name" }) + XCTAssertEqual(inspection.text, "Alexandria Alexandra") + } + + func testPointTextPolicyIsSharedAcrossXCTestAndPrivateAXDescriptors() { + let cases: [(String, String?, String?, String?, String?)] = [ + ("TextField", "Name", "name-field", "Alexandra", "Alexandra"), + ("SecureTextField", "Password", "password-field", "secret", "secret"), + ("Button", "Continue", "continue-button", "enabled", "Continue"), + ("Switch", nil, "enabled-switch", "1", "1"), + ] + + for (type, label, identifier, value, expected) in cases { + XCTAssertEqual( + pointReadableText(type: type, label: label, identifier: identifier, value: value), + expected, + "Expected shared text policy for \(type)" + ) + } + } + func testPrivateAXPointInspectionReturnsNoElementForHonestMiss() { let root: [String: Any] = [ "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), From 612f054660edc9fa5a7d3508602d4142d3388fe1 Mon Sep 17 00:00:00 2001 From: Christopher Sarkissian Date: Mon, 28 Sep 2026 19:09:59 -0700 Subject: [PATCH 21/21] fix(apple): preserve legacy get-text behavior --- .../RunnerTests+Interaction.swift | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index e84f91c9c1..fb9456fbc4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -318,19 +318,6 @@ extension RunnerTests { } func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { -#if os(iOS) && targetEnvironment(simulator) - // Prefer the contained Simulator AX read when it can prove a value. Besides - // avoiding the unbounded XCTest descendant queries below for populated - // text fields, this uses the same text-input-first policy as that fallback. - // An incomplete capture never answers the request; it falls through to the - // legacy XCTest path so a capped tree cannot turn a real value into a miss. - if let inspection = privateAXPointInspection(app: app, x: x, y: y), inspection.complete { - // A complete miss is authoritative too. Falling through would repeat a - // full XCTest descendant walk after the bounded AX capture already - // established that no readable element contains the point. - return inspection.text - } -#endif let point = CGPoint(x: x, y: y) let textInputCandidates = textInputCandidatesAt(app: app, point: point) let candidates = app.descendants(matching: .any).allElementsBoundByIndex