From 52d8c426c1c9c50a44d779509e5782326c283232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 13:07:52 +0200 Subject: [PATCH 1/4] feat(snapshot): interpret iOS node facts once at producer seams --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 18 ++ .../RunnerTests+Snapshot.swift | 20 +- .../RunnerTests+SnapshotAcquisition.swift | 18 +- .../RunnerTests+SystemModal.swift | 12 +- .../RunnerTests+SystemModalTests.swift | 20 ++ .../SnapshotModels.swift | 2 +- .../ios-react-native-value-semantics.json | 7 +- .../fixtures/native-snapshot-states.json | 201 ++++++++++++++++++ .../capture-kit/src/ios-react-native-value.ts | 96 --------- ...st.ts => ios-snapshot-acquisition.test.ts} | 35 ++- .../src/ios-snapshot-acquisition.ts | 124 ++++++++++- .../__tests__/runner-selector-query.test.ts | 34 ++- .../src/core/runner-selector-query.ts | 18 +- .../__tests__/snapshot-presentation.test.ts | 32 ++- .../src/runner/snapshot-presentation.ts | 18 +- .../src/snapshot-source/tree.test.ts | 20 ++ .../src/snapshot-source/tree.ts | 4 +- .../src/ios-snapshot-adapter.test.ts | 48 +++++ .../src/ios-snapshot-adapter.ts | 3 +- packages/provider-limrun/src/snapshot.test.ts | 17 +- packages/provider-limrun/src/snapshot.ts | 5 + .../src/webdriver-ios-snapshot.test.ts | 36 +++- .../src/webdriver-ios-snapshot.ts | 3 +- .../src/webdriver-source.test.ts | 69 +++++- .../src/webdriver-source.ts | 74 +++---- .../runtime/snapshot-unchanged.test.ts | 6 +- website/docs/docs/snapshots.md | 19 +- 27 files changed, 732 insertions(+), 227 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift create mode 100644 contracts/fixtures/native-snapshot-states.json delete mode 100644 packages/capture-kit/src/ios-react-native-value.ts rename packages/capture-kit/src/{ios-react-native-value.test.ts => ios-snapshot-acquisition.test.ts} (62%) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 93f622f9f0..6a9612eecc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -573,6 +573,24 @@ - (void)viewDidLoad { self.alertActivationBusyAnswer = busyAnswer; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-native-states"]) { + for (NSUInteger index = 0; index < 3; index++) { + UITextField *field = [[UITextField alloc] initWithFrame:CGRectMake(40, 120 + index * 60, 260, 44)]; + field.accessibilityIdentifier = @[@"native-field", @"native-secure", @"native-disabled"][index]; + field.placeholder = @"Native input"; + field.secureTextEntry = index == 1; + field.enabled = index != 2; + field.borderStyle = UITextBorderStyleRoundedRect; + [self.view addSubview:field]; + } + for (NSUInteger index = 0; index < 2; index++) { + UISwitch *toggle = [[UISwitch alloc] initWithFrame:CGRectMake(40, 340 + index * 60, 60, 40)]; + toggle.accessibilityIdentifier = index == 0 ? @"native-off" : @"native-on"; + toggle.on = index == 1; + [self.view addSubview:toggle]; + } + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) { BOOL digitCountValue = [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-count-value"]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 483b9ec9a8..0e0670aecb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -99,6 +99,8 @@ extension RunnerTests { XCUIElement.ElementType.activityIndicator.rawValue: "ActivityIndicator", XCUIElement.ElementType.progressIndicator.rawValue: "ProgressIndicator", XCUIElement.ElementType.checkBox.rawValue: "CheckBox", + XCUIElement.ElementType.radioButton.rawValue: "RadioButton", + XCUIElement.ElementType.toggle.rawValue: "Toggle", XCUIElement.ElementType.menuItem.rawValue: "MenuItem", XCUIElement.ElementType.webView.rawValue: "WebView", XCUIElement.ElementType.other.rawValue: "Other", @@ -497,23 +499,7 @@ extension RunnerTests { nodes[0] = interactiveRootNode(rect: rootRect) for candidate in candidates { nodes.append( - RawAXNode( - index: nodes.count, - type: candidate.type, - label: candidate.label, - identifier: candidate.identifier, - value: candidate.value, - placeholder: candidate.placeholder, - rect: candidate.rect, - enabled: candidate.enabled, - focused: candidate.focused, - selected: candidate.selected, - hittable: candidate.hittable, - depth: 1, - parentIndex: 0, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) + candidate.reindexed(index: nodes.count, depth: 1, parentIndex: 0) ) } return ( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 58330f6813..420b55e7ba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -357,23 +357,7 @@ extension RunnerTests { if uniqueCandidates.count < 2 { return [] } return uniqueCandidates.enumerated().map { offset, node in - RawAXNode( - index: startingIndex + offset, - type: node.type, - label: node.label, - identifier: node.identifier, - value: node.value, - placeholder: node.placeholder, - rect: node.rect, - enabled: node.enabled, - focused: node.focused, - selected: node.selected, - hittable: node.hittable, - depth: depth, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) + node.reindexed(index: startingIndex + offset, depth: depth, parentIndex: parentIndex) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift index c05d3fb2f9..bdc1eefbf7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift @@ -11,6 +11,14 @@ extension RunnerTests { guard case .resolved(let resolvedModal) = resolveBlockingSystemModal(deadline: deadline) else { return nil } + return snapshotForBlockingSystemModal(resolvedModal, deadline: deadline) + #endif + } + + func snapshotForBlockingSystemModal( + _ resolvedModal: ResolvedBlockingSystemModal, + deadline: Date = .distantFuture + ) -> DataPayload? { let modal = resolvedModal.root let actions = resolvedModal.actions @@ -62,7 +70,6 @@ extension RunnerTests { } return DataPayload(nodes: nodes, truncated: false) - #endif } func firstBlockingSystemModal( @@ -217,13 +224,14 @@ extension RunnerTests { ) -> PresentedNode { let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines) let identifier = (identifierOverride ?? element.identifier).trimmingCharacters(in: .whitespacesAndNewlines) + let value = element.value.map { String(describing: $0).trimmingCharacters(in: .whitespacesAndNewlines) } return SnapshotPresentation.singleElementRead( RawAXNode( index: index, type: type, label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier, - value: nil, + value: value.flatMap { $0.isEmpty ? nil : $0 }, rect: SnapshotRect(element.frame), enabled: element.isEnabled, focused: nil, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift new file mode 100644 index 0000000000..c20ae6e5a3 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift @@ -0,0 +1,20 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + @MainActor + func testBlockingModalSnapshotPreservesNativeSwitchValues() throws { + app.launchArguments = ["--agent-device-native-states"] + app.launch() + defer { app.terminate() } + XCTAssertTrue(app.switches["native-off"].waitForExistence(timeout: appExistenceTimeout)) + let resolved = ResolvedBlockingSystemModal(root: app, ownerApp: app, actions: actionableElements(in: app)) + let payload = try XCTUnwrap(snapshotForBlockingSystemModal(resolved)) + let off = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-off" }) + let on = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-on" }) + XCTAssertEqual(off.value, "0") + XCTAssertEqual(on.value, "1") + } +#endif +} diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift index bdb21cb713..44fbb2b1ce 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift @@ -91,7 +91,7 @@ public struct RawAXNode: Equatable { return updated } - func reindexed(index: Int, depth: Int, parentIndex: Int?) -> RawAXNode { + public func reindexed(index: Int, depth: Int, parentIndex: Int?) -> RawAXNode { var updated = self updated.index = index updated.depth = depth diff --git a/contracts/fixtures/ios-react-native-value-semantics.json b/contracts/fixtures/ios-react-native-value-semantics.json index 25a2f6c3de..81043eced0 100644 --- a/contracts/fixtures/ios-react-native-value-semantics.json +++ b/contracts/fixtures/ios-react-native-value-semantics.json @@ -301,7 +301,10 @@ "type": "TextField", "value": "checkbox, checked" }, - "expected": {} + "expected": { + "editable": true, + "password": false + } }, { "name": "literal editable generic node", @@ -334,7 +337,7 @@ "node": { "index": 0, "type": "Other", - "value": "Kontrollk\u00e4stchen, nicht markiert", + "value": "Kontrollkästchen, nicht markiert", "role": "RCTView" }, "expected": {} diff --git a/contracts/fixtures/native-snapshot-states.json b/contracts/fixtures/native-snapshot-states.json new file mode 100644 index 0000000000..bbe19a3d4d --- /dev/null +++ b/contracts/fixtures/native-snapshot-states.json @@ -0,0 +1,201 @@ +{ + "cases": [ + { + "name": "switch on", + "input": { + "type": "Switch", + "value": "1" + }, + "expected": { + "checked": true + } + }, + { + "name": "switch off", + "input": { + "type": "Switch", + "value": "0" + }, + "expected": { + "checked": false + } + }, + { + "name": "checkbox false", + "input": { + "type": "CheckBox", + "value": "false" + }, + "expected": { + "checked": false + } + }, + { + "name": "radio true", + "input": { + "type": "RadioButton", + "value": "true" + }, + "expected": { + "checked": true + } + }, + { + "name": "toggle off", + "input": { + "type": "Toggle", + "value": "0" + }, + "expected": { + "checked": false + } + }, + { + "name": "switch no value", + "input": { + "type": "Switch" + }, + "expected": {} + }, + { + "name": "mixed remains unknown", + "input": { + "type": "CheckBox", + "value": "mixed" + }, + "expected": {} + }, + { + "name": "RN encoding remains raw", + "input": { + "type": "Switch", + "value": "checked, disabled" + }, + "expected": {} + }, + { + "name": "unrelated numeric value", + "input": { + "type": "Slider", + "value": "1" + }, + "expected": {} + }, + { + "name": "unknown type", + "input": { + "value": "0" + }, + "expected": {} + }, + { + "name": "TextField", + "input": { + "type": "TextField" + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "SearchField", + "input": { + "type": "SearchField" + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "TextView", + "input": { + "type": "TextView" + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "SecureTextField", + "input": { + "type": "SecureTextField" + }, + "expected": { + "editable": true, + "password": true + } + }, + { + "name": "disabled native field capability", + "input": { + "type": "TextField", + "enabled": false + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "XCUI prefix", + "input": { + "type": "XCUIElementTypeSecureTextField" + }, + "expected": { + "editable": true, + "password": true + } + }, + { + "name": "explicit unchecked wins", + "input": { + "type": "Switch", + "value": "1", + "checked": false + }, + "expected": { + "checked": false + } + }, + { + "name": "explicit read only wins", + "input": { + "type": "TextField", + "editable": false, + "password": true + }, + "expected": { + "editable": false, + "password": true + } + }, + { + "name": "explicit non secure wins", + "input": { + "type": "SecureTextField", + "password": false + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "custom raw facts", + "input": { + "type": "Other", + "editable": false, + "password": false, + "checked": true + }, + "expected": { + "editable": false, + "password": false, + "checked": true + } + } + ] +} diff --git a/packages/capture-kit/src/ios-react-native-value.ts b/packages/capture-kit/src/ios-react-native-value.ts deleted file mode 100644 index a367d20b61..0000000000 --- a/packages/capture-kit/src/ios-react-native-value.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { normalizeType } from '@agent-device/contracts/snapshot'; -import type { IosValueSemantics, RawSnapshotNode } from '@agent-device/kernel/snapshot'; - -const RN_VIEW_CLASSES = new Set(['RCTView', 'RCTViewComponentView']); -const ENGLISH_ROLES: Readonly> = { - alert: 'alert', - checkbox: 'checkbox', - 'combo box': 'combobox', - menu: 'menu', - 'menu bar': 'menubar', - 'menu item': 'menuitem', - 'progress bar': 'progressbar', - 'radio button': 'radio', - 'radio group': 'radiogroup', - 'scroll bar': 'scrollbar', - 'spin button': 'spinbutton', - switch: 'switch', - tab: 'tab', - 'tab list': 'tablist', - timer: 'timer', - 'tool bar': 'toolbar', -}; - -type StateComponent = Pick; -const ENGLISH_STATES: Readonly> = { - checked: { checked: true }, - unchecked: { checked: false }, - mixed: { checked: 'mixed' }, - expanded: { expanded: true }, - collapsed: { expanded: false }, - busy: { busy: true }, -}; - -/** Interprets RN's value grammar without removing value evidence or overriding native facts. */ -export function interpretIosReactNativeValue(node: RawSnapshotNode): RawSnapshotNode { - if (node.iosValueSemantics || typeof node.value !== 'string') return node; - if (!isInterpretationCandidate(node)) return node; - const semantics = parseEnglishValue(node.value, node.role); - if (!semantics || !hasSupportedSignature(node, semantics)) return node; - return { - ...node, - iosValueSemantics: semantics, - ...(node.checked === undefined && typeof semantics.checked === 'boolean' - ? { checked: semantics.checked } - : {}), - }; -} - -function isInterpretationCandidate(node: RawSnapshotNode): boolean { - if (normalizeType(node.type ?? '') !== 'other' || node.editable || node.password) return false; - if (RN_VIEW_CLASSES.has(node.role ?? '')) return true; - return node.role === undefined || normalizeType(node.role) === 'other'; -} - -function hasSupportedSignature(node: RawSnapshotNode, semantics: IosValueSemantics): boolean { - const checkableRole = semantics.role === 'checkbox' || semantics.role === 'radio'; - if (semantics.role !== undefined && !checkableRole && node.role !== 'RCTView') return false; - if (RN_VIEW_CLASSES.has(node.role ?? '')) return true; - return checkableRole && semantics.checked !== undefined; -} - -function parseEnglishValue( - value: string, - nativeClass: string | undefined, -): IosValueSemantics | undefined { - const components = value.split(', '); - const first = components[0] ?? ''; - const role = Object.hasOwn(ENGLISH_ROLES, first) ? ENGLISH_ROLES[first] : undefined; - const semantics: IosValueSemantics = { - source: 'react-native-ios-value-heuristic', - encoding: 'en', - ...(role === undefined ? {} : { role }), - }; - let lastStateOrder = 0; - for (const component of components.slice(role === undefined ? 0 : 1)) { - if (!Object.hasOwn(ENGLISH_STATES, component)) break; - const state = ENGLISH_STATES[component]!; - if (Object.keys(state).some((key) => Object.hasOwn(semantics, key))) break; - const stateOrder = fabricStateOrder(state); - if (nativeClass === 'RCTViewComponentView' && stateOrder < lastStateOrder) break; - lastStateOrder = stateOrder; - Object.assign(semantics, state); - } - return semantics.role !== undefined || - semantics.checked !== undefined || - semantics.expanded !== undefined || - semantics.busy !== undefined - ? semantics - : undefined; -} - -function fabricStateOrder(state: StateComponent): number { - if (state.checked !== undefined) return 1; - if (state.expanded !== undefined) return 2; - return 3; -} diff --git a/packages/capture-kit/src/ios-react-native-value.test.ts b/packages/capture-kit/src/ios-snapshot-acquisition.test.ts similarity index 62% rename from packages/capture-kit/src/ios-react-native-value.test.ts rename to packages/capture-kit/src/ios-snapshot-acquisition.test.ts index 0470984959..379d25f419 100644 --- a/packages/capture-kit/src/ios-react-native-value.test.ts +++ b/packages/capture-kit/src/ios-snapshot-acquisition.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import { test } from 'vitest'; import fc from 'fast-check'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { interpretIosReactNativeValue } from './ios-react-native-value.ts'; +import { interpretIosNode } from './ios-snapshot-acquisition.ts'; const fixture = JSON.parse( readFileSync( @@ -22,7 +22,7 @@ const fixture = JSON.parse( for (const entry of fixture.cases) { test(`${entry.renderer}/${entry.locale}: ${entry.name}`, () => { const before = structuredClone(entry.node); - assert.deepEqual(interpretIosReactNativeValue(entry.node), { + assert.deepEqual(interpretIosNode(entry.node), { ...entry.node, ...entry.expected, }); @@ -34,11 +34,11 @@ test('preserves arbitrary native-control values and identities', () => { fc.assert( fc.property( fc.string(), - fc.constantFrom('TextField', 'Switch', 'Button', 'StaticText'), + fc.constantFrom('Slider', 'ProgressIndicator', 'Button', 'StaticText'), (value, type) => { const node = { index: 0, type, value }; const before = structuredClone(node); - assert.equal(interpretIosReactNativeValue(node), node); + assert.equal(interpretIosNode(node), node); assert.deepEqual(node, before); }, ), @@ -56,13 +56,36 @@ test('preserves raw checked facts and arbitrary suffixes without mutating or rei checked, }; const before = structuredClone(node); - const result = interpretIosReactNativeValue(node); + const result = interpretIosNode(node); assert.deepEqual(node, before); assert.equal(result.value, before.value); assert.equal(result.checked, before.checked); assert.equal(result.iosValueSemantics?.checked, 'mixed'); - assert.equal(interpretIosReactNativeValue(result), result); + assert.equal(interpretIosNode(result), result); }), { numRuns: 100 }, ); }); + +const nativeStateTable = JSON.parse( + readFileSync( + new URL('../../../contracts/fixtures/native-snapshot-states.json', import.meta.url), + 'utf8', + ), +) as { + cases: { + name: string; + input: Omit; + expected: Partial; + }[]; +}; +for (const row of nativeStateTable.cases) { + test(`native iOS state golden: ${row.name}`, () => { + const node = { index: 0, ...row.input }; + const before = structuredClone(node); + const result = interpretIosNode(node); + assert.deepEqual(result, { ...node, ...row.expected }); + assert.deepEqual(node, before); + assert.equal(interpretIosNode(result), result); + }); +} diff --git a/packages/capture-kit/src/ios-snapshot-acquisition.ts b/packages/capture-kit/src/ios-snapshot-acquisition.ts index f9474bf63e..10702e8214 100644 --- a/packages/capture-kit/src/ios-snapshot-acquisition.ts +++ b/packages/capture-kit/src/ios-snapshot-acquisition.ts @@ -11,6 +11,7 @@ import type { import { normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import type { + IosValueSemantics, RawSnapshotNode, Rect, SnapshotKeyboardBandFact, @@ -110,7 +111,7 @@ export function createIosSnapshotAcquisition( acquisition: { producer: input.producer, intent: 'full', - nodes: input.nodes, + nodes: input.nodes.map((node) => interpretIosNode(node)), viewport: input.viewport, ...(input.keyboard ? { keyboard: input.keyboard } : {}), lineage: input.lineage, @@ -164,3 +165,124 @@ function rootGeometryRank(status: IosSnapshotViewportRoot['rectStatus']): number function rectArea(rect: Rect | undefined): number { return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0; } + +/** Acquires native iOS type/value facts and RN value semantics without overriding producer facts. */ +export function interpretIosNode( + node: RawSnapshotNode, + options: Readonly<{ reactNativeValue?: boolean }> = {}, +): RawSnapshotNode { + const type = node.type?.replace(/^XCUIElementType/, ''); + const field = ['TextField', 'SecureTextField', 'SearchField', 'TextView'].includes(type ?? ''); + const checkable = ['Switch', 'CheckBox', 'RadioButton', 'Toggle'].includes(type ?? ''); + const facts: Pick = {}; + if (checkable && node.checked === undefined) { + const checked = nativeCheckedValue(node.value); + if (checked !== undefined) facts.checked = checked; + } + if (field) { + if (node.password === undefined) facts.password = type === 'SecureTextField'; + if (node.editable === undefined) facts.editable = true; + } + const interpreted = Object.keys(facts).length === 0 ? node : { ...node, ...facts }; + return options.reactNativeValue === false ? interpreted : interpretReactNativeValue(interpreted); +} + +function nativeCheckedValue(value: string | undefined): boolean | undefined { + if (value === '1' || value === 'true') return true; + if (value === '0' || value === 'false') return false; + return undefined; +} + +const RN_VIEW_CLASSES = new Set(['RCTView', 'RCTViewComponentView']); +const ENGLISH_ROLES: Readonly> = { + alert: 'alert', + checkbox: 'checkbox', + 'combo box': 'combobox', + menu: 'menu', + 'menu bar': 'menubar', + 'menu item': 'menuitem', + 'progress bar': 'progressbar', + 'radio button': 'radio', + 'radio group': 'radiogroup', + 'scroll bar': 'scrollbar', + 'spin button': 'spinbutton', + switch: 'switch', + tab: 'tab', + 'tab list': 'tablist', + timer: 'timer', + 'tool bar': 'toolbar', +}; + +type StateComponent = Pick; +const ENGLISH_STATES: Readonly> = { + checked: { checked: true }, + unchecked: { checked: false }, + mixed: { checked: 'mixed' }, + expanded: { expanded: true }, + collapsed: { expanded: false }, + busy: { busy: true }, +}; + +/** Interprets RN's value grammar without removing value evidence or overriding native facts. */ +function interpretReactNativeValue(node: RawSnapshotNode): RawSnapshotNode { + if (node.iosValueSemantics || typeof node.value !== 'string') return node; + if (!isInterpretationCandidate(node)) return node; + const semantics = parseEnglishValue(node.value, node.role); + if (!semantics || !hasSupportedSignature(node, semantics)) return node; + return { + ...node, + iosValueSemantics: semantics, + ...(node.checked === undefined && typeof semantics.checked === 'boolean' + ? { checked: semantics.checked } + : {}), + }; +} + +function isInterpretationCandidate(node: RawSnapshotNode): boolean { + if (normalizeType(node.type ?? '') !== 'other' || node.editable || node.password) return false; + if (RN_VIEW_CLASSES.has(node.role ?? '')) return true; + return node.role === undefined || normalizeType(node.role) === 'other'; +} + +function hasSupportedSignature(node: RawSnapshotNode, semantics: IosValueSemantics): boolean { + const checkableRole = semantics.role === 'checkbox' || semantics.role === 'radio'; + if (semantics.role !== undefined && !checkableRole && node.role !== 'RCTView') return false; + if (RN_VIEW_CLASSES.has(node.role ?? '')) return true; + return checkableRole && semantics.checked !== undefined; +} + +function parseEnglishValue( + value: string, + nativeClass: string | undefined, +): IosValueSemantics | undefined { + const components = value.split(', '); + const first = components[0] ?? ''; + const role = Object.hasOwn(ENGLISH_ROLES, first) ? ENGLISH_ROLES[first] : undefined; + const semantics: IosValueSemantics = { + source: 'react-native-ios-value-heuristic', + encoding: 'en', + ...(role === undefined ? {} : { role }), + }; + let lastStateOrder = 0; + for (const component of components.slice(role === undefined ? 0 : 1)) { + if (!Object.hasOwn(ENGLISH_STATES, component)) break; + const state = ENGLISH_STATES[component]!; + if (Object.keys(state).some((key) => Object.hasOwn(semantics, key))) break; + const stateOrder = fabricStateOrder(state); + if (nativeClass === 'RCTViewComponentView' && stateOrder < lastStateOrder) break; + lastStateOrder = stateOrder; + Object.assign(semantics, state); + } + return semantics.role !== undefined || + semantics.checked !== undefined || + semantics.expanded !== undefined || + semantics.busy !== undefined + ? semantics + : undefined; +} + +function fabricStateOrder(state: StateComponent): number { + if (state.checked !== undefined) return 1; + if (state.expanded !== undefined) return 2; + return 3; +} diff --git a/packages/platform-apple/src/core/__tests__/runner-selector-query.test.ts b/packages/platform-apple/src/core/__tests__/runner-selector-query.test.ts index 5d9c864bfe..a902909334 100644 --- a/packages/platform-apple/src/core/__tests__/runner-selector-query.test.ts +++ b/packages/platform-apple/src/core/__tests__/runner-selector-query.test.ts @@ -2,7 +2,7 @@ import { expect, test, vi } from 'vitest'; import { IOS_SIMULATOR } from '../../__tests__/device-fixtures.ts'; const { runAppleRunnerCommand } = vi.hoisted(() => ({ - runAppleRunnerCommand: vi.fn(async () => ({ found: true })), + runAppleRunnerCommand: vi.fn(async (): Promise> => ({ found: true })), })); vi.mock('../runner-client.ts', () => ({ runAppleRunnerCommand })); @@ -31,3 +31,35 @@ test('builds the one querySelector runner command for every caller', async () => options, ); }); + +test('querySelector interprets native and RN node facts at the runner result seam', async () => { + runAppleRunnerCommand.mockResolvedValueOnce({ + found: true, + nodes: [ + { index: 0, type: 'Switch', value: '1' }, + { index: 1, type: 'SecureTextField' }, + { index: 2, type: 'TextField', enabled: false }, + { index: 3, type: 'Other', value: 'checkbox, unchecked' }, + ], + }); + const result = await queryAppleRunnerSelector( + IOS_SIMULATOR, + { key: 'id', value: 'native-on' }, + 'com.example.app', + {}, + ); + expect(result).toMatchObject({ + found: true, + nodes: [ + { type: 'Switch', value: '1', checked: true }, + { type: 'SecureTextField', editable: true, password: true }, + { type: 'TextField', enabled: false, editable: true, password: false }, + { + type: 'Other', + value: 'checkbox, unchecked', + checked: false, + iosValueSemantics: { role: 'checkbox' }, + }, + ], + }); +}); diff --git a/packages/platform-apple/src/core/runner-selector-query.ts b/packages/platform-apple/src/core/runner-selector-query.ts index f33da420b7..5ff88560fe 100644 --- a/packages/platform-apple/src/core/runner-selector-query.ts +++ b/packages/platform-apple/src/core/runner-selector-query.ts @@ -1,6 +1,8 @@ +import { interpretIosNode } from '@agent-device/capture-kit/ios-snapshot-acquisition'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { ElementSelectorKey } from '@agent-device/contracts/interactor-types'; import type { AppleRunnerCommandOptions } from '../runner/index.ts'; -import type { DeviceInfo } from '@agent-device/kernel/device'; +import { isHandheldAppleDevice, type DeviceInfo } from '@agent-device/kernel/device'; import { runAppleRunnerCommand } from './runner-client.ts'; /** The single Apple runner `querySelector` command builder shared by bound and safety-probe reads. */ @@ -10,7 +12,7 @@ export async function queryAppleRunnerSelector( appBundleId: string | undefined, options: AppleRunnerCommandOptions, ): Promise> { - return await runAppleRunnerCommand( + const result = await runAppleRunnerCommand( device, { command: 'querySelector', @@ -20,4 +22,16 @@ export async function queryAppleRunnerSelector( }, options, ); + return { + ...result, + ...(Array.isArray(result.nodes) + ? { + nodes: (result.nodes as RawSnapshotNode[]).map((node) => + interpretIosNode(node, { + reactNativeValue: isHandheldAppleDevice(device), + }), + ), + } + : {}), + }; } diff --git a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts index 2d40311e16..bb34ec370e 100644 --- a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts +++ b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts @@ -1,4 +1,4 @@ -import { IOS_SIMULATOR } from '../../__tests__/device-fixtures.ts'; +import { IOS_SIMULATOR, MACOS_DEVICE } from '../../__tests__/device-fixtures.ts'; import { AppError } from '@agent-device/kernel/errors'; import assert from 'node:assert/strict'; import { test } from 'vitest'; @@ -426,3 +426,33 @@ test('runner presented and quality nodes share RN interpretation without replaci } assert.equal('checked' in node, false); }); + +for (const device of [IOS_SIMULATOR, MACOS_DEVICE]) { + test(`runner ${device.appleOs} reads native states on presented and quality payloads`, () => { + const nodes = [ + { index: 0, type: 'Switch', value: '0' }, + { index: 1, type: 'SecureTextField' }, + { index: 2, type: 'TextField', enabled: false }, + { index: 3, type: 'Switch', value: '1', checked: false }, + { index: 4, type: 'Other', value: 'checkbox, unchecked' }, + ]; + const result = readAppleSnapshotResult( + { nodes, qualityPayload: { nodes, truncated: false } }, + device, + ); + for (const interpreted of [result.nodes!, result.qualityPayload!.nodes]) { + assert.equal(interpreted[0]!.checked, false); + assert.deepEqual([interpreted[1]!.editable, interpreted[1]!.password], [true, true]); + assert.deepEqual( + [interpreted[2]!.enabled, interpreted[2]!.editable, interpreted[2]!.password], + [false, true, false], + ); + assert.equal(interpreted[3]!.checked, false); + assert.equal( + interpreted[4]!.iosValueSemantics?.role, + device.appleOs === 'ios' ? 'checkbox' : undefined, + ); + } + assert.equal('checked' in nodes[0]!, false); + }); +} diff --git a/packages/platform-apple/src/runner/snapshot-presentation.ts b/packages/platform-apple/src/runner/snapshot-presentation.ts index eac6560dea..f64a6381d1 100644 --- a/packages/platform-apple/src/runner/snapshot-presentation.ts +++ b/packages/platform-apple/src/runner/snapshot-presentation.ts @@ -1,5 +1,4 @@ import { isHandheldAppleDevice, type DeviceInfo } from '@agent-device/kernel/device'; -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; import type { IosRunnerQualityPayloadFacts, IosSnapshotInput, @@ -11,7 +10,10 @@ import { presentIosRunnerSnapshot, toIosSnapshotEngineErrorDetails, } from '@agent-device/capture-kit/ios-snapshot-engine'; -import { resolveIosViewportEvidenceFromRoots } from '@agent-device/capture-kit/ios-snapshot-acquisition'; +import { + interpretIosNode, + resolveIosViewportEvidenceFromRoots, +} from '@agent-device/capture-kit/ios-snapshot-acquisition'; import { renderSnapshotQualityWarnings } from '@agent-device/capture-kit/quality-warnings'; import { isSparseSnapshotQualityVerdict, @@ -70,15 +72,15 @@ export function readAppleSnapshotResult( ); const nodes = Array.isArray(result.nodes) ? (result.nodes as RawSnapshotNode[]) : undefined; const qualityPayload = readQualityPayload(result.qualityPayload, systemSurface); - const interpretValues = isHandheldAppleDevice(device); + const interpret = (node: RawSnapshotNode) => + interpretIosNode(node, { reactNativeValue: isHandheldAppleDevice(device) }); return { - nodes: interpretValues ? nodes?.map(interpretIosReactNativeValue) : nodes, + nodes: nodes?.map(interpret), truncated: typeof result.truncated === 'boolean' ? result.truncated : undefined, quality: readSnapshotQualityVerdict(result.snapshotQuality), - qualityPayload: - interpretValues && qualityPayload - ? { ...qualityPayload, nodes: qualityPayload.nodes.map(interpretIosReactNativeValue) } - : qualityPayload, + qualityPayload: qualityPayload + ? { ...qualityPayload, nodes: qualityPayload.nodes.map(interpret) } + : qualityPayload, runnerFatal: result.runnerFatal === true, ...(keyboard ? { keyboard } : {}), ...(systemSurface ? { systemSurface } : {}), diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 8413a49d2e..d8c29df4eb 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -630,3 +630,23 @@ test('bridge RN class values carry shared semantics while native class evidence expanded: true, }); }); + +test('the AX bridge publishes native switch and text entry facts without inventing unknown states', () => { + const tree = decodeSnapshotBridgeTree( + { + [application]: 'Application', + [children]: [ + { [automationType]: 40, XC_kAXXCAttributeValue: 0, [children]: [] }, + { [automationType]: 50, [traits]: '0', [children]: [] }, + { [automationType]: 49, [traits]: '256', [children]: [] }, + { [automationType]: 40, [children]: [] }, + ], + }, + { truncated: false }, + limits, + ).nodes; + assert.equal(tree[1]!.checked, false); + assert.deepEqual([tree[2]!.editable, tree[2]!.password], [true, true]); + assert.deepEqual([tree[3]!.enabled, tree[3]!.editable, tree[3]!.password], [false, true, false]); + assert.equal(tree[4]!.checked, undefined); +}); diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index 2c835cf548..65ff38aac0 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -1,4 +1,4 @@ -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import { isGeometricallyActionable, isPositiveFiniteRect, @@ -232,7 +232,7 @@ export function decodeSnapshotBridgeTree( } const index = nodes.length; const node = nodeFacts(value, index, parentIndex, depth); - nodes.push(interpretIosReactNativeValue(node)); + nodes.push(interpretIosNode(node)); maxTraversalDepth = Math.max(maxTraversalDepth, depth); if (isHostedRemoteLeaf(node, children.length, underRemoteHost)) { hostedRemoteLeaves.push(node.rect); diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts index d6a40bd337..a42176d710 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from 'node:fs'; +import type { IosTreeNode } from './snapshot.ts'; import { expect, test, vi } from 'vitest'; import { captureLimrunIosSnapshot } from './ios-snapshot-adapter.ts'; import { @@ -99,6 +101,52 @@ test('the Limrun acquired result keeps provider provenance and SDK screen dimens }); }); +const nativeStates = JSON.parse( + readFileSync( + new URL('../../../contracts/fixtures/native-snapshot-states.json', import.meta.url), + 'utf8', + ), +) as { + cases: { name: string; input: IosTreeNode; expected: Record }[]; +}; +for (const row of nativeStates.cases.filter( + ({ input }) => !['checked', 'password', 'editable'].some((key) => key in input), +)) { + test(`limrun acquires native states: ${row.name}`, async () => { + const result = await captureLimrunIosSnapshot(createLimrunSnapshotSession(row.input)); + const node = result.acquisition.nodes[0]!; + const states = Object.fromEntries( + ['checked', 'password', 'editable'].flatMap((key) => { + const value = node[key as 'checked' | 'password' | 'editable']; + return value === undefined ? [] : [[key, value]]; + }), + ); + expect(states).toEqual(row.expected); + }); +} + +test('Limrun secure TextField subrole survives acquisition without changing the native type', async () => { + const result = await captureLimrunIosSnapshot( + createLimrunSnapshotSession({ + type: 'TextField', + role: 'AXTextField', + subrole: 'AXSecureTextField', + traits: ['TextOperationsAvailable', 'Scrollable', 'TextEntry', 'SecureTextField'], + role_description: 'secure text field', + AXUniqueId: 'native-secure', + AXValue: 'Native input', + enabled: true, + }), + ); + expect(result.acquisition.nodes[0]).toMatchObject({ + type: 'TextField', + identifier: 'native-secure', + password: true, + editable: true, + roleDescription: 'secure text field', + }); +}); + test('Limrun carries docked tree evidence and never infers absence from missing nodes', async () => { const tree = limrunSnapshotTree(); expect( diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.ts b/packages/provider-limrun/src/ios-snapshot-adapter.ts index 4f517fa466..b83d0e3c51 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.ts @@ -1,4 +1,3 @@ -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; import { createIosSnapshotAcquisition, resolveIosViewportEvidenceFromRoots, @@ -13,7 +12,7 @@ export async function captureLimrunIosSnapshot( ): Promise { const tree = (await session.client.elementTree()) as IosTreeNode[]; const viewport = readLimrunViewport(tree, session.client.deviceInfo); - const nodes = flattenIosTree(tree).map(interpretIosReactNativeValue); + const nodes = flattenIosTree(tree); return createIosSnapshotAcquisition({ producer: 'limrun-ios-tree', ...acquireProviderKeyboard({ diff --git a/packages/provider-limrun/src/snapshot.test.ts b/packages/provider-limrun/src/snapshot.test.ts index 46d508a9bf..02e0cfd493 100644 --- a/packages/provider-limrun/src/snapshot.test.ts +++ b/packages/provider-limrun/src/snapshot.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'vitest'; -import { toIosSelector } from './snapshot.ts'; +import { flattenIosTree, toIosSelector } from './snapshot.ts'; test('agent-device selector keys become the accessibility keys the iOS instance matches', () => { expect(toIosSelector({ key: 'id', value: 'email-field' })).toEqual({ @@ -14,3 +14,18 @@ test('agent-device selector keys become the accessibility keys the iOS instance test('visible text targets the provider label field, which is where the tree exposes it', () => { expect(toIosSelector({ key: 'text', value: 'Sign In' })).toEqual({ AXLabel: 'Sign In' }); }); + +test('Limrun raw mapping forwards observed Header traits and SDK role descriptions', () => { + const [heading, unknown] = flattenIosTree([ + { type: 'Heading', traits: ['Scrollable', 'Header'], role_description: 'heading' }, + { type: 'TextField', traits: ['TextEntry'] }, + ]); + expect(heading).toMatchObject({ heading: true, roleDescription: 'heading' }); + expect(unknown!.heading).toBeUndefined(); + expect(unknown!.roleDescription).toBeUndefined(); + expect([unknown!.checked, unknown!.password, unknown!.editable]).toEqual([ + undefined, + undefined, + undefined, + ]); +}); diff --git a/packages/provider-limrun/src/snapshot.ts b/packages/provider-limrun/src/snapshot.ts index 0c87e6d05c..d2f68845bc 100644 --- a/packages/provider-limrun/src/snapshot.ts +++ b/packages/provider-limrun/src/snapshot.ts @@ -15,7 +15,9 @@ export type IosTreeNode = { rect?: { x?: number; y?: number; width?: number; height?: number }; enabled?: boolean; role?: string; + subrole?: string | null; selected?: boolean; + role_description?: string; hittable?: boolean; traits?: string[]; pid?: number; @@ -52,6 +54,9 @@ function mapIosNode( rect: readIosNodeRect(node), enabled: node.enabled, selected: node.selected, + password: node.subrole === 'AXSecureTextField' ? true : undefined, + heading: node.traits?.includes('Header') === true ? true : undefined, + roleDescription: node.role_description, hittable: node.hittable, depth: options.depth, parentIndex: options.parentIndex, diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index d39f6b5bdb..b35178ac50 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import { resolveKeyboardTapOcclusion } from '@agent-device/contracts/tap-keyboard-occlusion'; @@ -65,6 +66,39 @@ test('Appium iOS adapter carries missing viewport evidence without presenting or ]); }); +const iosNativeStateTable = JSON.parse( + readFileSync( + new URL('../../../contracts/fixtures/native-snapshot-states.json', import.meta.url), + 'utf8', + ), +) as { + cases: { + name: string; + input: Record; + expected: Record; + }[]; +}; +for (const row of iosNativeStateTable.cases) { + test(`Appium iOS native states: ${row.name}`, () => { + const type = row.input.type ?? 'Other'; + const attrs = Object.entries(row.input) + .filter(([key]) => key !== 'type') + .map(([key, value]) => `${key}="${value}"`) + .join(' '); + const node = acquireWebDriverIosSnapshot(`<${type} label="Node" ${attrs} />`).acquisition + .nodes[0]!; + assert.deepEqual( + Object.fromEntries( + ['checked', 'password', 'editable'].flatMap((key) => { + const value = node[key as 'checked' | 'password' | 'editable']; + return value === undefined ? [] : [[key, value]]; + }), + ), + row.expected, + ); + }); +} + test('Appium carries visibility-only and derived keyboard facts before projection', () => { assert.deepEqual(acquireWebDriverIosSnapshot(SOURCE, undefined, true).acquisition.keyboard, { kind: 'visibility-only', @@ -106,7 +140,7 @@ test.each([ '_UIRemoteKeyboardPlaceholderView', 'TUISystemInputAssistantView', 'TUISystemInputAssistantPageView', -])('app controls under %s remain unowned through projection and state assembly', (role) => { +])('app controls under %s remain unowned through projection and state assembly', async (role) => { const source = SOURCE.replace( '', ``, diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 102702f3f2..ecdf4fc9c5 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -1,4 +1,3 @@ -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; import { createIosSnapshotAcquisition, resolveIosViewportEvidenceFromRoots, @@ -23,7 +22,7 @@ export function acquireWebDriverIosSnapshot( producer: APPIUM_PRODUCER, viewport, ...acquireProviderKeyboard({ - nodes: sourceFacts.nodes.map(interpretIosReactNativeValue), + nodes: sourceFacts.nodes, viewport: viewport.kind === 'missing' ? null : viewport.rect, visibility, }), diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index 3174ef0499..04d11df0bf 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -52,7 +52,7 @@ test("Android WebDriver facts name a node by its text, carry a content descripti field!.hintShowing, field!.editable, ], - ['Ada Lovelace', 'Ada Lovelace', 'Name field', 'Your name', undefined, true], + ['Ada Lovelace', 'Ada Lovelace', 'Name field', 'Your name', undefined, undefined], ); assert.deepEqual( [label!.label, label!.value, 'editable' in label!], @@ -60,7 +60,7 @@ test("Android WebDriver facts name a node by its text, carry a content descripti ); }); -test('Android WebDriver facts read a hinted field as empty, a masked field as a password, and a disabled field as not editable', () => { +test('Android WebDriver facts read a hinted field as empty, a masked field as a password, and editability as unavailable when unreported', () => { const [hinted, secure, disabled, lookalike] = parseWebDriverSourceFacts( '' + '' + @@ -76,8 +76,8 @@ test('Android WebDriver facts read a hinted field as empty, a masked field as a [hinted!.value, hinted!.label, hinted!.placeholder, hinted!.hintShowing], ['', 'Your name', 'Your name', true], ); - assert.deepEqual([secure!.password, secure!.editable], [true, true]); - assert.deepEqual([disabled!.value, disabled!.editable], ['locked', false]); + assert.deepEqual([secure!.password, secure!.editable], [true, undefined]); + assert.deepEqual([disabled!.value, disabled!.editable], ['locked', undefined]); // Only the class name's last segment decides, as `isFillableType` reads it. assert.deepEqual( [lookalike!.label, lookalike!.value, 'editable' in lookalike!], @@ -104,9 +104,7 @@ test('Android WebDriver facts carry the checked state of checkable controls only ); }); -// The native iOS runner reports no checked, editable, or password facts, so the WebDriver -// projection stays at that parity: a switch keeps its state as its value. -test('iOS WebDriver facts keep a switch value as a value and add no field facts', () => { +test('iOS WebDriver source preserves native values for acquisition interpretation', () => { const [toggle, field, secure] = parseWebDriverSourceFacts( '' + '' + @@ -115,10 +113,10 @@ test('iOS WebDriver facts keep a switch value as a value and add no field facts' '', ).nodes; - assert.deepEqual([toggle!.value, 'checked' in toggle!], ['1', false]); + assert.deepEqual([toggle!.value, toggle!.checked], ['1', undefined]); assert.deepEqual( - [field!.value, 'editable' in field!, 'password' in secure!], - ['ada@example.com', false, false], + [field!.value, field!.editable, secure!.password], + ['ada@example.com', undefined, undefined], ); }); @@ -254,7 +252,7 @@ const ANDROID_CLASS_IDENTITY_CASES = [ widget: 'EditText', className: 'android.widget.EditText', kind: 'text-field', - editable: true, + editable: undefined, scrollContainer: false, }, { @@ -436,3 +434,52 @@ test('Android source preserves native package provenance on anonymous IME window ); assert.equal(nodes[0]?.identifier, undefined); }); + +test('WebDriver preserves explicit native states on custom controls and field overrides', () => { + for (const platform of ['ios', 'android'] as const) { + const [custom, field, unknown] = parseWebDriverSourceFacts( + '' + + '' + + `<${platform === 'ios' ? 'XCUIElementTypeSecureTextField' : 'android.widget.EditText'} editable="false" password="false" enabled="true" />` + + '' + + '', + platform, + ).nodes; + assert.deepEqual( + [ + custom!.checked, + custom!.password, + custom!.editable, + custom!.heading, + custom!.roleDescription, + ], + [false, false, false, false, 'Custom role'], + ); + assert.deepEqual([field!.editable, field!.password], [false, false]); + assert.deepEqual( + [unknown!.checked, unknown!.password, unknown!.editable, unknown!.heading], + [undefined, undefined, undefined, undefined], + ); + } +}); + +test('Android Appium role descriptions read the exact native extras key and preserve explicit attributes', () => { + const [native, explicit, empty, unrelated] = parseWebDriverSourceFacts( + '' + + '' + + '' + + '' + + '' + + '', + 'android', + ).nodes; + assert.deepEqual( + [ + native!.roleDescription, + explicit!.roleDescription, + empty!.roleDescription, + unrelated!.roleDescription, + ], + ['Tab List', 'Link', undefined, undefined], + ); +}); diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 683d580e58..6da5680ebf 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -1,4 +1,4 @@ -import { androidFieldValue, isFillableType, normalizeType } from '@agent-device/contracts/snapshot'; +import { androidFieldValue } from '@agent-device/contracts/snapshot'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { parseBounds } from '@agent-device/kernel/bounds'; import { AppError } from '@agent-device/kernel/errors'; @@ -166,7 +166,6 @@ function sourceNodeFromAttributes( rect: RawSnapshotNode['rect'], platform: WebDriverSourcePlatform, ): RawSnapshotNode { - const field = textFieldFacts(type, attrs, platform); return { index, type, @@ -179,8 +178,13 @@ function sourceNodeFromAttributes( ...(platform === 'android' && attrs.package ? { bundleId: attrs.package } : {}), rect, ...sourceStateFacts(attrs, rect, platform), - ...(field === undefined ? {} : field.facts), - ...checkedFact(type, attrs, platform), + ...nativeStateFacts(type, attrs, platform), + ...optionalFact('heading', parseWebDriverBoolean(attrs.heading)), + ...optionalFact( + 'roleDescription', + firstWebDriverAttribute(attrs, ['roleDescription', 'role-description']) ?? + (platform === 'android' ? androidRoleDescription(attrs.extras) : undefined), + ), depth, parentIndex, }; @@ -209,40 +213,6 @@ function labelFacts( }; } -type TextFieldFacts = Readonly<{ - facts: Pick; -}>; - -/** - * The field facts a text entry control carries. UiAutomator2 reports a field's content as `text` - * (the same attribute a label carries on every other node), its hint as `hint`, and whether it - * masks input as `password`. XCUITest reports content as `value`, which every node already - * carries, and the native iOS runner reports no field facts, so none are derived there. - */ -function textFieldFacts( - type: string, - attrs: Record, - platform: WebDriverSourcePlatform, -): TextFieldFacts | undefined { - return platform === 'android' ? androidTextFieldFacts(type, attrs) : undefined; -} - -function androidTextFieldFacts( - type: string, - attrs: Record, -): TextFieldFacts | undefined { - const password = parseWebDriverBoolean(attrs.password); - if (!(normalizeType(type) && isFillableType(type, 'android')) && password !== true) { - return undefined; - } - return { - facts: { - editable: parseWebDriverBoolean(attrs.enabled, true) === true, - ...optionalFact('password', password), - }, - }; -} - function androidValueFacts( type: string, attrs: Record, @@ -288,21 +258,27 @@ function optionalFact( /** Android classes that are checkable when the page source names no `checkable` attribute. */ const ANDROID_CHECKABLE_CLASS = /(CheckBox|RadioButton|Switch|ToggleButton|CheckedTextView)$/; -/** - * The checked state of a checkable Android control. UiAutomator2 reports `checkable` and - * `checked` on every node, `false` on the many that cannot be checked, so only a checkable node - * carries the fact. XCUITest reports a switch's state as its `value`, which stays a value, as the - * native iOS runner reports it. - */ -function checkedFact( +function nativeStateFacts( type: string, attrs: Record, platform: WebDriverSourcePlatform, -): Pick { - if (platform !== 'android') return {}; - const checkable = parseWebDriverBoolean(attrs.checkable) ?? ANDROID_CHECKABLE_CLASS.test(type); +): Pick { + const raw = { + ...optionalFact('password', parseWebDriverBoolean(attrs.password)), + ...optionalFact('editable', parseWebDriverBoolean(attrs.editable)), + }; const checked = parseWebDriverBoolean(attrs.checked); - return checkable && checked !== undefined ? { checked } : {}; + if (platform === 'ios') { + return { ...raw, ...optionalFact('checked', checked) }; + } + const checkable = parseWebDriverBoolean(attrs.checkable) ?? ANDROID_CHECKABLE_CLASS.test(type); + return { ...raw, ...(checkable && checked !== undefined ? { checked } : {}) }; +} + +function androidRoleDescription(extras: string | undefined): string | undefined { + const prefix = 'AccessibilityNodeInfo.roleDescription='; + const entry = extras?.split(';').find((value) => value.startsWith(prefix)); + return nonEmptyWebDriverAttribute(entry?.slice(prefix.length)); } function sourceStateFacts( diff --git a/src/commands/capture/runtime/snapshot-unchanged.test.ts b/src/commands/capture/runtime/snapshot-unchanged.test.ts index b678d721ee..3edf9933be 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'vitest'; -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import { attachRefs } from '@agent-device/kernel/snapshot'; import fc from 'fast-check'; import { @@ -342,10 +342,10 @@ test.each([{ editable: true }, { password: true }])( checked: false, }; const previous = snapshot('Preferences', { - nodes: attachRefs([interpretIosReactNativeValue(node)]), + nodes: attachRefs([interpretIosNode(node)]), }); const current = snapshot('Preferences', { - nodes: attachRefs([interpretIosReactNativeValue({ ...node, ...change })]), + nodes: attachRefs([interpretIosNode({ ...node, ...change })]), }); expect(previous.nodes[0]?.iosValueSemantics?.checked).toBe(true); expect(current.nodes[0]?.iosValueSemantics).toBeUndefined(); diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 67c1103f21..60d5590471 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -347,6 +347,10 @@ Android snapshot nodes and `get attrs` (including the digest response) carry the them. Explicit `false` and `0` are kept; an absent field means the fact was unavailable, not false. `hintShowing` and `placeholder` need Android API 26 or later, `heading` API 28 or later. +On iOS, the Apple runner, Simulator accessibility bridge, Appium, and Limrun publish native +`checked`, `password`, and `editable` facts when available. macOS runner snapshots publish the +same native facts. + - `selected` is the accessibility selected state an app sets on a control — the active bottom-tab or segmented-control item, or the chosen row of a list. Android reports it explicitly as `true` or `false`; an older helper APK omits the field, which means the answer is unavailable rather than @@ -356,9 +360,20 @@ them. Explicit `false` and `0` are kept; an absent field means the fact was unav view an app marked checkable. Android reports it as `true` or `false` on those nodes only; a node that cannot be checked, or an older helper APK, omits the field. Snapshot text marks the node `[checked]` or `[unchecked]`, so a toggle that reads as plain text is one Android did not report as - checkable. + checkable. On iOS and the macOS runner, native switches, checkboxes, radio buttons, and toggles + carry `checked` when their native value reports a boolean state; an unavailable or unrecognized + state stays absent. +- `editable` is the native text-entry capability when available. `is editable` returns false for a + disabled node, even when `editable` is true. Otherwise it uses the published fact, including + explicit false, and falls back to the platform type only when the fact is absent. Android + Appium leaves `editable` absent when the source does not report it. +- `password` identifies secure text entry. On iOS and the macOS runner, native secure fields report + true and ordinary native text-entry fields report false. Android carries the reported native + password flag. - `heading` is the accessibility heading flag an app sets on a node, the way React Native's - `accessibilityRole="header"` does on a plain `View`; it is present only as `true`. + `accessibilityRole="header"` does on a plain `View`. The Android helper publishes true for a + heading; Appium preserves an explicit true or false when reported. Limrun reports true when + the node carries the native heading trait. - `roleDescription` is the localized role description an app sets beside the native class, verbatim (React Native writes `Tab`, `Tab List`, `Radio Group`, `Link`, `Menu`), when the class alone would not say what the control is. The `type` stays the class; a consumer maps the description to a role. From b1d30355fb22e7321db71e74d84815acf4662e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 13:07:52 +0200 Subject: [PATCH 2/4] chore(gates): retire iOS decoder exports and classify acquisition imports --- packages/capture-kit/package.json | 4 ---- scripts/layering/package-boundaries.test.ts | 1 - .../snapshot-assembly-presentation-policy.test.ts | 10 ++++++++++ .../snapshot-assembly-presentation-policy.ts | 13 ++++++------- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index d4f9d14a04..8b24bef561 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -317,10 +317,6 @@ "./durable-capture/session-binding": { "types": "./src/durable-capture/session-binding.ts", "default": "./src/durable-capture/session-binding.ts" - }, - "./ios-react-native-value": { - "types": "./src/ios-react-native-value.ts", - "default": "./src/ios-react-native-value.ts" } }, "devDependencies": { diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index c9e6100ad5..7eaca8f078 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -485,7 +485,6 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/durable-capture-runtime-recovery', '@agent-device/capture-kit/durable-capture/session-binding', '@agent-device/capture-kit/durable-json', - '@agent-device/capture-kit/ios-react-native-value', '@agent-device/capture-kit/ios-snapshot-acquisition', '@agent-device/capture-kit/ios-snapshot-engine', '@agent-device/capture-kit/ios-snapshot-planning', diff --git a/scripts/layering/snapshot-assembly-presentation-policy.test.ts b/scripts/layering/snapshot-assembly-presentation-policy.test.ts index 8e0e808810..6b2497315a 100644 --- a/scripts/layering/snapshot-assembly-presentation-policy.test.ts +++ b/scripts/layering/snapshot-assembly-presentation-policy.test.ts @@ -148,3 +148,13 @@ test('R74 fails closed when the assembly moves out from under it', () => { [...SNAPSHOT_ASSEMBLY_FILES], ); }); + +test('R74 accepts producer imports of raw iOS node acquisition facts', () => { + const result = violations( + appended( + producerAdapter, + `\nimport { interpretIosNode } from '@agent-device/capture-kit/ios-snapshot-acquisition';\nvoid interpretIosNode;\n`, + ), + ); + assert.deepEqual(result, []); +}); diff --git a/scripts/layering/snapshot-assembly-presentation-policy.ts b/scripts/layering/snapshot-assembly-presentation-policy.ts index 679f1bbbf9..0ec161d629 100644 --- a/scripts/layering/snapshot-assembly-presentation-policy.ts +++ b/scripts/layering/snapshot-assembly-presentation-policy.ts @@ -36,13 +36,11 @@ export const SNAPSHOT_PRODUCER_ADAPTER_ROOTS: readonly string[] = [ /** * iOS presentation. The engine owns projection, geometry, scope, depth, actionability narrowing - * and semantic compaction; the acquisition module owns the producer capability table that decides - * them. Reading either one from the assembly or from a producer adapter is how the deleted - * backend-name policy grows back. + * and semantic compaction. Producer adapters can acquire shared raw-node facts but cannot present + * them; the assembly cannot read either presentation or producer acquisition policy. */ const IOS_PRESENTATION_TARGETS: readonly string[] = [ 'packages/capture-kit/src/ios-snapshot-engine/', - 'packages/capture-kit/src/ios-snapshot-acquisition.ts', ]; /** @@ -84,8 +82,9 @@ function missingOwnerViolations(sources: ReadonlyMap): LayeringV function importViolations(edges: readonly ResolvedImportEdge[]): LayeringViolation[] { const violations: LayeringViolation[] = []; for (const edge of edges) { - if (!isIosPresentationTarget(edge.target)) continue; - if (SNAPSHOT_ASSEMBLY_FILES.includes(edge.file)) { + const presentation = isIosPresentationTarget(edge.target); + const acquisition = edge.target === 'packages/capture-kit/src/ios-snapshot-acquisition.ts'; + if (SNAPSHOT_ASSEMBLY_FILES.includes(edge.file) && (presentation || acquisition)) { violations.push({ rule: SNAPSHOT_ASSEMBLY_PRESENTATION_RULE, file: edge.file, @@ -94,7 +93,7 @@ function importViolations(edges: readonly ResolvedImportEdge[]): LayeringViolati }); continue; } - if (isProducerAdapter(edge.file)) { + if (isProducerAdapter(edge.file) && presentation) { violations.push({ rule: SNAPSHOT_ASSEMBLY_PRESENTATION_RULE, file: edge.file, From 29ed6a90b08be21ac37b71496cdd1a748119891f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 13:43:32 +0200 Subject: [PATCH 3/4] refactor(snapshot): isolate iOS node interpretation --- ...est.ts => ios-node-interpretation.test.ts} | 2 +- .../src/ios-node-interpretation.ts | 123 +++++++++++++++++ .../src/ios-snapshot-acquisition.ts | 124 +----------------- .../src/core/runner-selector-query.ts | 2 +- .../src/runner/snapshot-presentation.ts | 6 +- .../src/snapshot-source/tree.ts | 2 +- .../src/ios-snapshot-adapter.ts | 3 +- .../src/webdriver-ios-snapshot.ts | 3 +- .../runtime/snapshot-unchanged.test.ts | 2 +- 9 files changed, 134 insertions(+), 133 deletions(-) rename packages/capture-kit/src/{ios-snapshot-acquisition.test.ts => ios-node-interpretation.test.ts} (97%) create mode 100644 packages/capture-kit/src/ios-node-interpretation.ts diff --git a/packages/capture-kit/src/ios-snapshot-acquisition.test.ts b/packages/capture-kit/src/ios-node-interpretation.test.ts similarity index 97% rename from packages/capture-kit/src/ios-snapshot-acquisition.test.ts rename to packages/capture-kit/src/ios-node-interpretation.test.ts index 379d25f419..1746cef89b 100644 --- a/packages/capture-kit/src/ios-snapshot-acquisition.test.ts +++ b/packages/capture-kit/src/ios-node-interpretation.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import { test } from 'vitest'; import fc from 'fast-check'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { interpretIosNode } from './ios-snapshot-acquisition.ts'; +import { interpretIosNode } from './ios-node-interpretation.ts'; const fixture = JSON.parse( readFileSync( diff --git a/packages/capture-kit/src/ios-node-interpretation.ts b/packages/capture-kit/src/ios-node-interpretation.ts new file mode 100644 index 0000000000..4099513606 --- /dev/null +++ b/packages/capture-kit/src/ios-node-interpretation.ts @@ -0,0 +1,123 @@ +import { normalizeType } from '@agent-device/contracts/snapshot'; +import type { IosValueSemantics, RawSnapshotNode } from '@agent-device/kernel/snapshot'; + +/** Acquires native iOS type/value facts and RN value semantics without overriding producer facts. */ +export function interpretIosNode( + node: RawSnapshotNode, + options: Readonly<{ reactNativeValue?: boolean }> = {}, +): RawSnapshotNode { + const type = node.type?.replace(/^XCUIElementType/, ''); + const field = ['TextField', 'SecureTextField', 'SearchField', 'TextView'].includes(type ?? ''); + const checkable = ['Switch', 'CheckBox', 'RadioButton', 'Toggle'].includes(type ?? ''); + const facts: Pick = {}; + if (checkable && node.checked === undefined) { + const checked = nativeCheckedValue(node.value); + if (checked !== undefined) facts.checked = checked; + } + if (field) { + if (node.password === undefined) facts.password = type === 'SecureTextField'; + if (node.editable === undefined) facts.editable = true; + } + const interpreted = Object.keys(facts).length === 0 ? node : { ...node, ...facts }; + return options.reactNativeValue === false ? interpreted : interpretReactNativeValue(interpreted); +} + +function nativeCheckedValue(value: string | undefined): boolean | undefined { + if (value === '1' || value === 'true') return true; + if (value === '0' || value === 'false') return false; + return undefined; +} + +const RN_VIEW_CLASSES = new Set(['RCTView', 'RCTViewComponentView']); +const ENGLISH_ROLES: Readonly> = { + alert: 'alert', + checkbox: 'checkbox', + 'combo box': 'combobox', + menu: 'menu', + 'menu bar': 'menubar', + 'menu item': 'menuitem', + 'progress bar': 'progressbar', + 'radio button': 'radio', + 'radio group': 'radiogroup', + 'scroll bar': 'scrollbar', + 'spin button': 'spinbutton', + switch: 'switch', + tab: 'tab', + 'tab list': 'tablist', + timer: 'timer', + 'tool bar': 'toolbar', +}; + +type StateComponent = Pick; +const ENGLISH_STATES: Readonly> = { + checked: { checked: true }, + unchecked: { checked: false }, + mixed: { checked: 'mixed' }, + expanded: { expanded: true }, + collapsed: { expanded: false }, + busy: { busy: true }, +}; + +/** Interprets RN's value grammar without removing value evidence or overriding native facts. */ +function interpretReactNativeValue(node: RawSnapshotNode): RawSnapshotNode { + if (node.iosValueSemantics || typeof node.value !== 'string') return node; + if (!isInterpretationCandidate(node)) return node; + const semantics = parseEnglishValue(node.value, node.role); + if (!semantics || !hasSupportedSignature(node, semantics)) return node; + return { + ...node, + iosValueSemantics: semantics, + ...(node.checked === undefined && typeof semantics.checked === 'boolean' + ? { checked: semantics.checked } + : {}), + }; +} + +function isInterpretationCandidate(node: RawSnapshotNode): boolean { + if (normalizeType(node.type ?? '') !== 'other' || node.editable || node.password) return false; + if (RN_VIEW_CLASSES.has(node.role ?? '')) return true; + return node.role === undefined || normalizeType(node.role) === 'other'; +} + +function hasSupportedSignature(node: RawSnapshotNode, semantics: IosValueSemantics): boolean { + const checkableRole = semantics.role === 'checkbox' || semantics.role === 'radio'; + if (semantics.role !== undefined && !checkableRole && node.role !== 'RCTView') return false; + if (RN_VIEW_CLASSES.has(node.role ?? '')) return true; + return checkableRole && semantics.checked !== undefined; +} + +function parseEnglishValue( + value: string, + nativeClass: string | undefined, +): IosValueSemantics | undefined { + const components = value.split(', '); + const first = components[0] ?? ''; + const role = Object.hasOwn(ENGLISH_ROLES, first) ? ENGLISH_ROLES[first] : undefined; + const semantics: IosValueSemantics = { + source: 'react-native-ios-value-heuristic', + encoding: 'en', + ...(role === undefined ? {} : { role }), + }; + let lastStateOrder = 0; + for (const component of components.slice(role === undefined ? 0 : 1)) { + if (!Object.hasOwn(ENGLISH_STATES, component)) break; + const state = ENGLISH_STATES[component]!; + if (Object.keys(state).some((key) => Object.hasOwn(semantics, key))) break; + const stateOrder = fabricStateOrder(state); + if (nativeClass === 'RCTViewComponentView' && stateOrder < lastStateOrder) break; + lastStateOrder = stateOrder; + Object.assign(semantics, state); + } + return semantics.role !== undefined || + semantics.checked !== undefined || + semantics.expanded !== undefined || + semantics.busy !== undefined + ? semantics + : undefined; +} + +function fabricStateOrder(state: StateComponent): number { + if (state.checked !== undefined) return 1; + if (state.expanded !== undefined) return 2; + return 3; +} diff --git a/packages/capture-kit/src/ios-snapshot-acquisition.ts b/packages/capture-kit/src/ios-snapshot-acquisition.ts index 10702e8214..f9474bf63e 100644 --- a/packages/capture-kit/src/ios-snapshot-acquisition.ts +++ b/packages/capture-kit/src/ios-snapshot-acquisition.ts @@ -11,7 +11,6 @@ import type { import { normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import type { - IosValueSemantics, RawSnapshotNode, Rect, SnapshotKeyboardBandFact, @@ -111,7 +110,7 @@ export function createIosSnapshotAcquisition( acquisition: { producer: input.producer, intent: 'full', - nodes: input.nodes.map((node) => interpretIosNode(node)), + nodes: input.nodes, viewport: input.viewport, ...(input.keyboard ? { keyboard: input.keyboard } : {}), lineage: input.lineage, @@ -165,124 +164,3 @@ function rootGeometryRank(status: IosSnapshotViewportRoot['rectStatus']): number function rectArea(rect: Rect | undefined): number { return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0; } - -/** Acquires native iOS type/value facts and RN value semantics without overriding producer facts. */ -export function interpretIosNode( - node: RawSnapshotNode, - options: Readonly<{ reactNativeValue?: boolean }> = {}, -): RawSnapshotNode { - const type = node.type?.replace(/^XCUIElementType/, ''); - const field = ['TextField', 'SecureTextField', 'SearchField', 'TextView'].includes(type ?? ''); - const checkable = ['Switch', 'CheckBox', 'RadioButton', 'Toggle'].includes(type ?? ''); - const facts: Pick = {}; - if (checkable && node.checked === undefined) { - const checked = nativeCheckedValue(node.value); - if (checked !== undefined) facts.checked = checked; - } - if (field) { - if (node.password === undefined) facts.password = type === 'SecureTextField'; - if (node.editable === undefined) facts.editable = true; - } - const interpreted = Object.keys(facts).length === 0 ? node : { ...node, ...facts }; - return options.reactNativeValue === false ? interpreted : interpretReactNativeValue(interpreted); -} - -function nativeCheckedValue(value: string | undefined): boolean | undefined { - if (value === '1' || value === 'true') return true; - if (value === '0' || value === 'false') return false; - return undefined; -} - -const RN_VIEW_CLASSES = new Set(['RCTView', 'RCTViewComponentView']); -const ENGLISH_ROLES: Readonly> = { - alert: 'alert', - checkbox: 'checkbox', - 'combo box': 'combobox', - menu: 'menu', - 'menu bar': 'menubar', - 'menu item': 'menuitem', - 'progress bar': 'progressbar', - 'radio button': 'radio', - 'radio group': 'radiogroup', - 'scroll bar': 'scrollbar', - 'spin button': 'spinbutton', - switch: 'switch', - tab: 'tab', - 'tab list': 'tablist', - timer: 'timer', - 'tool bar': 'toolbar', -}; - -type StateComponent = Pick; -const ENGLISH_STATES: Readonly> = { - checked: { checked: true }, - unchecked: { checked: false }, - mixed: { checked: 'mixed' }, - expanded: { expanded: true }, - collapsed: { expanded: false }, - busy: { busy: true }, -}; - -/** Interprets RN's value grammar without removing value evidence or overriding native facts. */ -function interpretReactNativeValue(node: RawSnapshotNode): RawSnapshotNode { - if (node.iosValueSemantics || typeof node.value !== 'string') return node; - if (!isInterpretationCandidate(node)) return node; - const semantics = parseEnglishValue(node.value, node.role); - if (!semantics || !hasSupportedSignature(node, semantics)) return node; - return { - ...node, - iosValueSemantics: semantics, - ...(node.checked === undefined && typeof semantics.checked === 'boolean' - ? { checked: semantics.checked } - : {}), - }; -} - -function isInterpretationCandidate(node: RawSnapshotNode): boolean { - if (normalizeType(node.type ?? '') !== 'other' || node.editable || node.password) return false; - if (RN_VIEW_CLASSES.has(node.role ?? '')) return true; - return node.role === undefined || normalizeType(node.role) === 'other'; -} - -function hasSupportedSignature(node: RawSnapshotNode, semantics: IosValueSemantics): boolean { - const checkableRole = semantics.role === 'checkbox' || semantics.role === 'radio'; - if (semantics.role !== undefined && !checkableRole && node.role !== 'RCTView') return false; - if (RN_VIEW_CLASSES.has(node.role ?? '')) return true; - return checkableRole && semantics.checked !== undefined; -} - -function parseEnglishValue( - value: string, - nativeClass: string | undefined, -): IosValueSemantics | undefined { - const components = value.split(', '); - const first = components[0] ?? ''; - const role = Object.hasOwn(ENGLISH_ROLES, first) ? ENGLISH_ROLES[first] : undefined; - const semantics: IosValueSemantics = { - source: 'react-native-ios-value-heuristic', - encoding: 'en', - ...(role === undefined ? {} : { role }), - }; - let lastStateOrder = 0; - for (const component of components.slice(role === undefined ? 0 : 1)) { - if (!Object.hasOwn(ENGLISH_STATES, component)) break; - const state = ENGLISH_STATES[component]!; - if (Object.keys(state).some((key) => Object.hasOwn(semantics, key))) break; - const stateOrder = fabricStateOrder(state); - if (nativeClass === 'RCTViewComponentView' && stateOrder < lastStateOrder) break; - lastStateOrder = stateOrder; - Object.assign(semantics, state); - } - return semantics.role !== undefined || - semantics.checked !== undefined || - semantics.expanded !== undefined || - semantics.busy !== undefined - ? semantics - : undefined; -} - -function fabricStateOrder(state: StateComponent): number { - if (state.checked !== undefined) return 1; - if (state.expanded !== undefined) return 2; - return 3; -} diff --git a/packages/platform-apple/src/core/runner-selector-query.ts b/packages/platform-apple/src/core/runner-selector-query.ts index 5ff88560fe..0b3d590c8d 100644 --- a/packages/platform-apple/src/core/runner-selector-query.ts +++ b/packages/platform-apple/src/core/runner-selector-query.ts @@ -1,4 +1,4 @@ -import { interpretIosNode } from '@agent-device/capture-kit/ios-snapshot-acquisition'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { ElementSelectorKey } from '@agent-device/contracts/interactor-types'; import type { AppleRunnerCommandOptions } from '../runner/index.ts'; diff --git a/packages/platform-apple/src/runner/snapshot-presentation.ts b/packages/platform-apple/src/runner/snapshot-presentation.ts index f64a6381d1..60c54496bc 100644 --- a/packages/platform-apple/src/runner/snapshot-presentation.ts +++ b/packages/platform-apple/src/runner/snapshot-presentation.ts @@ -10,10 +10,8 @@ import { presentIosRunnerSnapshot, toIosSnapshotEngineErrorDetails, } from '@agent-device/capture-kit/ios-snapshot-engine'; -import { - interpretIosNode, - resolveIosViewportEvidenceFromRoots, -} from '@agent-device/capture-kit/ios-snapshot-acquisition'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; +import { resolveIosViewportEvidenceFromRoots } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import { renderSnapshotQualityWarnings } from '@agent-device/capture-kit/quality-warnings'; import { isSparseSnapshotQualityVerdict, diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index 65ff38aac0..be87d3bf9e 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -1,4 +1,4 @@ -import { interpretIosNode } from '@agent-device/capture-kit/ios-snapshot-acquisition'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { isGeometricallyActionable, isPositiveFiniteRect, diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.ts b/packages/provider-limrun/src/ios-snapshot-adapter.ts index b83d0e3c51..201782a94b 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.ts @@ -1,3 +1,4 @@ +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { createIosSnapshotAcquisition, resolveIosViewportEvidenceFromRoots, @@ -12,7 +13,7 @@ export async function captureLimrunIosSnapshot( ): Promise { const tree = (await session.client.elementTree()) as IosTreeNode[]; const viewport = readLimrunViewport(tree, session.client.deviceInfo); - const nodes = flattenIosTree(tree); + const nodes = flattenIosTree(tree).map((node) => interpretIosNode(node)); return createIosSnapshotAcquisition({ producer: 'limrun-ios-tree', ...acquireProviderKeyboard({ diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index ecdf4fc9c5..ce88471665 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -1,3 +1,4 @@ +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { createIosSnapshotAcquisition, resolveIosViewportEvidenceFromRoots, @@ -22,7 +23,7 @@ export function acquireWebDriverIosSnapshot( producer: APPIUM_PRODUCER, viewport, ...acquireProviderKeyboard({ - nodes: sourceFacts.nodes, + nodes: sourceFacts.nodes.map((node) => interpretIosNode(node)), viewport: viewport.kind === 'missing' ? null : viewport.rect, visibility, }), diff --git a/src/commands/capture/runtime/snapshot-unchanged.test.ts b/src/commands/capture/runtime/snapshot-unchanged.test.ts index 3edf9933be..1575b6b73b 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'vitest'; -import { interpretIosNode } from '@agent-device/capture-kit/ios-snapshot-acquisition'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { attachRefs } from '@agent-device/kernel/snapshot'; import fc from 'fast-check'; import { From 64573fd90119870854aee208572ebd61712ca584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 13:43:32 +0200 Subject: [PATCH 4/4] chore(gates): restore protected iOS acquisition boundary --- packages/capture-kit/package.json | 4 ++++ scripts/layering/package-boundaries.test.ts | 1 + .../snapshot-assembly-presentation-policy.test.ts | 10 ---------- .../snapshot-assembly-presentation-policy.ts | 13 +++++++------ 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 8b24bef561..6529518511 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -110,6 +110,10 @@ "types": "./src/screen-recording-transport.ts", "default": "./src/screen-recording-transport.ts" }, + "./ios-node-interpretation": { + "types": "./src/ios-node-interpretation.ts", + "default": "./src/ios-node-interpretation.ts" + }, "./ios-snapshot-acquisition": { "types": "./src/ios-snapshot-acquisition.ts", "default": "./src/ios-snapshot-acquisition.ts" diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 7eaca8f078..b3628e0b34 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -485,6 +485,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/durable-capture-runtime-recovery', '@agent-device/capture-kit/durable-capture/session-binding', '@agent-device/capture-kit/durable-json', + '@agent-device/capture-kit/ios-node-interpretation', '@agent-device/capture-kit/ios-snapshot-acquisition', '@agent-device/capture-kit/ios-snapshot-engine', '@agent-device/capture-kit/ios-snapshot-planning', diff --git a/scripts/layering/snapshot-assembly-presentation-policy.test.ts b/scripts/layering/snapshot-assembly-presentation-policy.test.ts index 6b2497315a..8e0e808810 100644 --- a/scripts/layering/snapshot-assembly-presentation-policy.test.ts +++ b/scripts/layering/snapshot-assembly-presentation-policy.test.ts @@ -148,13 +148,3 @@ test('R74 fails closed when the assembly moves out from under it', () => { [...SNAPSHOT_ASSEMBLY_FILES], ); }); - -test('R74 accepts producer imports of raw iOS node acquisition facts', () => { - const result = violations( - appended( - producerAdapter, - `\nimport { interpretIosNode } from '@agent-device/capture-kit/ios-snapshot-acquisition';\nvoid interpretIosNode;\n`, - ), - ); - assert.deepEqual(result, []); -}); diff --git a/scripts/layering/snapshot-assembly-presentation-policy.ts b/scripts/layering/snapshot-assembly-presentation-policy.ts index 0ec161d629..679f1bbbf9 100644 --- a/scripts/layering/snapshot-assembly-presentation-policy.ts +++ b/scripts/layering/snapshot-assembly-presentation-policy.ts @@ -36,11 +36,13 @@ export const SNAPSHOT_PRODUCER_ADAPTER_ROOTS: readonly string[] = [ /** * iOS presentation. The engine owns projection, geometry, scope, depth, actionability narrowing - * and semantic compaction. Producer adapters can acquire shared raw-node facts but cannot present - * them; the assembly cannot read either presentation or producer acquisition policy. + * and semantic compaction; the acquisition module owns the producer capability table that decides + * them. Reading either one from the assembly or from a producer adapter is how the deleted + * backend-name policy grows back. */ const IOS_PRESENTATION_TARGETS: readonly string[] = [ 'packages/capture-kit/src/ios-snapshot-engine/', + 'packages/capture-kit/src/ios-snapshot-acquisition.ts', ]; /** @@ -82,9 +84,8 @@ function missingOwnerViolations(sources: ReadonlyMap): LayeringV function importViolations(edges: readonly ResolvedImportEdge[]): LayeringViolation[] { const violations: LayeringViolation[] = []; for (const edge of edges) { - const presentation = isIosPresentationTarget(edge.target); - const acquisition = edge.target === 'packages/capture-kit/src/ios-snapshot-acquisition.ts'; - if (SNAPSHOT_ASSEMBLY_FILES.includes(edge.file) && (presentation || acquisition)) { + if (!isIosPresentationTarget(edge.target)) continue; + if (SNAPSHOT_ASSEMBLY_FILES.includes(edge.file)) { violations.push({ rule: SNAPSHOT_ASSEMBLY_PRESENTATION_RULE, file: edge.file, @@ -93,7 +94,7 @@ function importViolations(edges: readonly ResolvedImportEdge[]): LayeringViolati }); continue; } - if (isProducerAdapter(edge.file) && presentation) { + if (isProducerAdapter(edge.file)) { violations.push({ rule: SNAPSHOT_ASSEMBLY_PRESENTATION_RULE, file: edge.file,