diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 93f622f9f0..6a9612eecc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -573,6 +573,24 @@ - (void)viewDidLoad { self.alertActivationBusyAnswer = busyAnswer; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-native-states"]) { + for (NSUInteger index = 0; index < 3; index++) { + UITextField *field = [[UITextField alloc] initWithFrame:CGRectMake(40, 120 + index * 60, 260, 44)]; + field.accessibilityIdentifier = @[@"native-field", @"native-secure", @"native-disabled"][index]; + field.placeholder = @"Native input"; + field.secureTextEntry = index == 1; + field.enabled = index != 2; + field.borderStyle = UITextBorderStyleRoundedRect; + [self.view addSubview:field]; + } + for (NSUInteger index = 0; index < 2; index++) { + UISwitch *toggle = [[UISwitch alloc] initWithFrame:CGRectMake(40, 340 + index * 60, 60, 40)]; + toggle.accessibilityIdentifier = index == 0 ? @"native-off" : @"native-on"; + toggle.on = index == 1; + [self.view addSubview:toggle]; + } + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) { BOOL digitCountValue = [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-count-value"]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 483b9ec9a8..0e0670aecb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -99,6 +99,8 @@ extension RunnerTests { XCUIElement.ElementType.activityIndicator.rawValue: "ActivityIndicator", XCUIElement.ElementType.progressIndicator.rawValue: "ProgressIndicator", XCUIElement.ElementType.checkBox.rawValue: "CheckBox", + XCUIElement.ElementType.radioButton.rawValue: "RadioButton", + XCUIElement.ElementType.toggle.rawValue: "Toggle", XCUIElement.ElementType.menuItem.rawValue: "MenuItem", XCUIElement.ElementType.webView.rawValue: "WebView", XCUIElement.ElementType.other.rawValue: "Other", @@ -497,23 +499,7 @@ extension RunnerTests { nodes[0] = interactiveRootNode(rect: rootRect) for candidate in candidates { nodes.append( - RawAXNode( - index: nodes.count, - type: candidate.type, - label: candidate.label, - identifier: candidate.identifier, - value: candidate.value, - placeholder: candidate.placeholder, - rect: candidate.rect, - enabled: candidate.enabled, - focused: candidate.focused, - selected: candidate.selected, - hittable: candidate.hittable, - depth: 1, - parentIndex: 0, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) + candidate.reindexed(index: nodes.count, depth: 1, parentIndex: 0) ) } return ( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 58330f6813..420b55e7ba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -357,23 +357,7 @@ extension RunnerTests { if uniqueCandidates.count < 2 { return [] } return uniqueCandidates.enumerated().map { offset, node in - RawAXNode( - index: startingIndex + offset, - type: node.type, - label: node.label, - identifier: node.identifier, - value: node.value, - placeholder: node.placeholder, - rect: node.rect, - enabled: node.enabled, - focused: node.focused, - selected: node.selected, - hittable: node.hittable, - depth: depth, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) + node.reindexed(index: startingIndex + offset, depth: depth, parentIndex: parentIndex) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift index c05d3fb2f9..bdc1eefbf7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift @@ -11,6 +11,14 @@ extension RunnerTests { guard case .resolved(let resolvedModal) = resolveBlockingSystemModal(deadline: deadline) else { return nil } + return snapshotForBlockingSystemModal(resolvedModal, deadline: deadline) + #endif + } + + func snapshotForBlockingSystemModal( + _ resolvedModal: ResolvedBlockingSystemModal, + deadline: Date = .distantFuture + ) -> DataPayload? { let modal = resolvedModal.root let actions = resolvedModal.actions @@ -62,7 +70,6 @@ extension RunnerTests { } return DataPayload(nodes: nodes, truncated: false) - #endif } func firstBlockingSystemModal( @@ -217,13 +224,14 @@ extension RunnerTests { ) -> PresentedNode { let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines) let identifier = (identifierOverride ?? element.identifier).trimmingCharacters(in: .whitespacesAndNewlines) + let value = element.value.map { String(describing: $0).trimmingCharacters(in: .whitespacesAndNewlines) } return SnapshotPresentation.singleElementRead( RawAXNode( index: index, type: type, label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier, - value: nil, + value: value.flatMap { $0.isEmpty ? nil : $0 }, rect: SnapshotRect(element.frame), enabled: element.isEnabled, focused: nil, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift new file mode 100644 index 0000000000..c20ae6e5a3 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift @@ -0,0 +1,20 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + @MainActor + func testBlockingModalSnapshotPreservesNativeSwitchValues() throws { + app.launchArguments = ["--agent-device-native-states"] + app.launch() + defer { app.terminate() } + XCTAssertTrue(app.switches["native-off"].waitForExistence(timeout: appExistenceTimeout)) + let resolved = ResolvedBlockingSystemModal(root: app, ownerApp: app, actions: actionableElements(in: app)) + let payload = try XCTUnwrap(snapshotForBlockingSystemModal(resolved)) + let off = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-off" }) + let on = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-on" }) + XCTAssertEqual(off.value, "0") + XCTAssertEqual(on.value, "1") + } +#endif +} diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift index bdb21cb713..44fbb2b1ce 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift @@ -91,7 +91,7 @@ public struct RawAXNode: Equatable { return updated } - func reindexed(index: Int, depth: Int, parentIndex: Int?) -> RawAXNode { + public func reindexed(index: Int, depth: Int, parentIndex: Int?) -> RawAXNode { var updated = self updated.index = index updated.depth = depth diff --git a/contracts/fixtures/ios-react-native-value-semantics.json b/contracts/fixtures/ios-react-native-value-semantics.json index 25a2f6c3de..81043eced0 100644 --- a/contracts/fixtures/ios-react-native-value-semantics.json +++ b/contracts/fixtures/ios-react-native-value-semantics.json @@ -301,7 +301,10 @@ "type": "TextField", "value": "checkbox, checked" }, - "expected": {} + "expected": { + "editable": true, + "password": false + } }, { "name": "literal editable generic node", @@ -334,7 +337,7 @@ "node": { "index": 0, "type": "Other", - "value": "Kontrollk\u00e4stchen, nicht markiert", + "value": "Kontrollkästchen, nicht markiert", "role": "RCTView" }, "expected": {} diff --git a/contracts/fixtures/native-snapshot-states.json b/contracts/fixtures/native-snapshot-states.json new file mode 100644 index 0000000000..bbe19a3d4d --- /dev/null +++ b/contracts/fixtures/native-snapshot-states.json @@ -0,0 +1,201 @@ +{ + "cases": [ + { + "name": "switch on", + "input": { + "type": "Switch", + "value": "1" + }, + "expected": { + "checked": true + } + }, + { + "name": "switch off", + "input": { + "type": "Switch", + "value": "0" + }, + "expected": { + "checked": false + } + }, + { + "name": "checkbox false", + "input": { + "type": "CheckBox", + "value": "false" + }, + "expected": { + "checked": false + } + }, + { + "name": "radio true", + "input": { + "type": "RadioButton", + "value": "true" + }, + "expected": { + "checked": true + } + }, + { + "name": "toggle off", + "input": { + "type": "Toggle", + "value": "0" + }, + "expected": { + "checked": false + } + }, + { + "name": "switch no value", + "input": { + "type": "Switch" + }, + "expected": {} + }, + { + "name": "mixed remains unknown", + "input": { + "type": "CheckBox", + "value": "mixed" + }, + "expected": {} + }, + { + "name": "RN encoding remains raw", + "input": { + "type": "Switch", + "value": "checked, disabled" + }, + "expected": {} + }, + { + "name": "unrelated numeric value", + "input": { + "type": "Slider", + "value": "1" + }, + "expected": {} + }, + { + "name": "unknown type", + "input": { + "value": "0" + }, + "expected": {} + }, + { + "name": "TextField", + "input": { + "type": "TextField" + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "SearchField", + "input": { + "type": "SearchField" + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "TextView", + "input": { + "type": "TextView" + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "SecureTextField", + "input": { + "type": "SecureTextField" + }, + "expected": { + "editable": true, + "password": true + } + }, + { + "name": "disabled native field capability", + "input": { + "type": "TextField", + "enabled": false + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "XCUI prefix", + "input": { + "type": "XCUIElementTypeSecureTextField" + }, + "expected": { + "editable": true, + "password": true + } + }, + { + "name": "explicit unchecked wins", + "input": { + "type": "Switch", + "value": "1", + "checked": false + }, + "expected": { + "checked": false + } + }, + { + "name": "explicit read only wins", + "input": { + "type": "TextField", + "editable": false, + "password": true + }, + "expected": { + "editable": false, + "password": true + } + }, + { + "name": "explicit non secure wins", + "input": { + "type": "SecureTextField", + "password": false + }, + "expected": { + "editable": true, + "password": false + } + }, + { + "name": "custom raw facts", + "input": { + "type": "Other", + "editable": false, + "password": false, + "checked": true + }, + "expected": { + "editable": false, + "password": false, + "checked": true + } + } + ] +} diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index d4f9d14a04..6529518511 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -110,6 +110,10 @@ "types": "./src/screen-recording-transport.ts", "default": "./src/screen-recording-transport.ts" }, + "./ios-node-interpretation": { + "types": "./src/ios-node-interpretation.ts", + "default": "./src/ios-node-interpretation.ts" + }, "./ios-snapshot-acquisition": { "types": "./src/ios-snapshot-acquisition.ts", "default": "./src/ios-snapshot-acquisition.ts" @@ -317,10 +321,6 @@ "./durable-capture/session-binding": { "types": "./src/durable-capture/session-binding.ts", "default": "./src/durable-capture/session-binding.ts" - }, - "./ios-react-native-value": { - "types": "./src/ios-react-native-value.ts", - "default": "./src/ios-react-native-value.ts" } }, "devDependencies": { diff --git a/packages/capture-kit/src/ios-react-native-value.test.ts b/packages/capture-kit/src/ios-node-interpretation.test.ts similarity index 62% rename from packages/capture-kit/src/ios-react-native-value.test.ts rename to packages/capture-kit/src/ios-node-interpretation.test.ts index 0470984959..1746cef89b 100644 --- a/packages/capture-kit/src/ios-react-native-value.test.ts +++ b/packages/capture-kit/src/ios-node-interpretation.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import { test } from 'vitest'; import fc from 'fast-check'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { interpretIosReactNativeValue } from './ios-react-native-value.ts'; +import { interpretIosNode } from './ios-node-interpretation.ts'; const fixture = JSON.parse( readFileSync( @@ -22,7 +22,7 @@ const fixture = JSON.parse( for (const entry of fixture.cases) { test(`${entry.renderer}/${entry.locale}: ${entry.name}`, () => { const before = structuredClone(entry.node); - assert.deepEqual(interpretIosReactNativeValue(entry.node), { + assert.deepEqual(interpretIosNode(entry.node), { ...entry.node, ...entry.expected, }); @@ -34,11 +34,11 @@ test('preserves arbitrary native-control values and identities', () => { fc.assert( fc.property( fc.string(), - fc.constantFrom('TextField', 'Switch', 'Button', 'StaticText'), + fc.constantFrom('Slider', 'ProgressIndicator', 'Button', 'StaticText'), (value, type) => { const node = { index: 0, type, value }; const before = structuredClone(node); - assert.equal(interpretIosReactNativeValue(node), node); + assert.equal(interpretIosNode(node), node); assert.deepEqual(node, before); }, ), @@ -56,13 +56,36 @@ test('preserves raw checked facts and arbitrary suffixes without mutating or rei checked, }; const before = structuredClone(node); - const result = interpretIosReactNativeValue(node); + const result = interpretIosNode(node); assert.deepEqual(node, before); assert.equal(result.value, before.value); assert.equal(result.checked, before.checked); assert.equal(result.iosValueSemantics?.checked, 'mixed'); - assert.equal(interpretIosReactNativeValue(result), result); + assert.equal(interpretIosNode(result), result); }), { numRuns: 100 }, ); }); + +const nativeStateTable = JSON.parse( + readFileSync( + new URL('../../../contracts/fixtures/native-snapshot-states.json', import.meta.url), + 'utf8', + ), +) as { + cases: { + name: string; + input: Omit; + expected: Partial; + }[]; +}; +for (const row of nativeStateTable.cases) { + test(`native iOS state golden: ${row.name}`, () => { + const node = { index: 0, ...row.input }; + const before = structuredClone(node); + const result = interpretIosNode(node); + assert.deepEqual(result, { ...node, ...row.expected }); + assert.deepEqual(node, before); + assert.equal(interpretIosNode(result), result); + }); +} diff --git a/packages/capture-kit/src/ios-react-native-value.ts b/packages/capture-kit/src/ios-node-interpretation.ts similarity index 72% rename from packages/capture-kit/src/ios-react-native-value.ts rename to packages/capture-kit/src/ios-node-interpretation.ts index a367d20b61..4099513606 100644 --- a/packages/capture-kit/src/ios-react-native-value.ts +++ b/packages/capture-kit/src/ios-node-interpretation.ts @@ -1,6 +1,33 @@ import { normalizeType } from '@agent-device/contracts/snapshot'; import type { IosValueSemantics, RawSnapshotNode } from '@agent-device/kernel/snapshot'; +/** Acquires native iOS type/value facts and RN value semantics without overriding producer facts. */ +export function interpretIosNode( + node: RawSnapshotNode, + options: Readonly<{ reactNativeValue?: boolean }> = {}, +): RawSnapshotNode { + const type = node.type?.replace(/^XCUIElementType/, ''); + const field = ['TextField', 'SecureTextField', 'SearchField', 'TextView'].includes(type ?? ''); + const checkable = ['Switch', 'CheckBox', 'RadioButton', 'Toggle'].includes(type ?? ''); + const facts: Pick = {}; + if (checkable && node.checked === undefined) { + const checked = nativeCheckedValue(node.value); + if (checked !== undefined) facts.checked = checked; + } + if (field) { + if (node.password === undefined) facts.password = type === 'SecureTextField'; + if (node.editable === undefined) facts.editable = true; + } + const interpreted = Object.keys(facts).length === 0 ? node : { ...node, ...facts }; + return options.reactNativeValue === false ? interpreted : interpretReactNativeValue(interpreted); +} + +function nativeCheckedValue(value: string | undefined): boolean | undefined { + if (value === '1' || value === 'true') return true; + if (value === '0' || value === 'false') return false; + return undefined; +} + const RN_VIEW_CLASSES = new Set(['RCTView', 'RCTViewComponentView']); const ENGLISH_ROLES: Readonly> = { alert: 'alert', @@ -32,7 +59,7 @@ const ENGLISH_STATES: Readonly> = { }; /** Interprets RN's value grammar without removing value evidence or overriding native facts. */ -export function interpretIosReactNativeValue(node: RawSnapshotNode): RawSnapshotNode { +function interpretReactNativeValue(node: RawSnapshotNode): RawSnapshotNode { if (node.iosValueSemantics || typeof node.value !== 'string') return node; if (!isInterpretationCandidate(node)) return node; const semantics = parseEnglishValue(node.value, node.role); diff --git a/packages/platform-apple/src/core/__tests__/runner-selector-query.test.ts b/packages/platform-apple/src/core/__tests__/runner-selector-query.test.ts index 5d9c864bfe..a902909334 100644 --- a/packages/platform-apple/src/core/__tests__/runner-selector-query.test.ts +++ b/packages/platform-apple/src/core/__tests__/runner-selector-query.test.ts @@ -2,7 +2,7 @@ import { expect, test, vi } from 'vitest'; import { IOS_SIMULATOR } from '../../__tests__/device-fixtures.ts'; const { runAppleRunnerCommand } = vi.hoisted(() => ({ - runAppleRunnerCommand: vi.fn(async () => ({ found: true })), + runAppleRunnerCommand: vi.fn(async (): Promise> => ({ found: true })), })); vi.mock('../runner-client.ts', () => ({ runAppleRunnerCommand })); @@ -31,3 +31,35 @@ test('builds the one querySelector runner command for every caller', async () => options, ); }); + +test('querySelector interprets native and RN node facts at the runner result seam', async () => { + runAppleRunnerCommand.mockResolvedValueOnce({ + found: true, + nodes: [ + { index: 0, type: 'Switch', value: '1' }, + { index: 1, type: 'SecureTextField' }, + { index: 2, type: 'TextField', enabled: false }, + { index: 3, type: 'Other', value: 'checkbox, unchecked' }, + ], + }); + const result = await queryAppleRunnerSelector( + IOS_SIMULATOR, + { key: 'id', value: 'native-on' }, + 'com.example.app', + {}, + ); + expect(result).toMatchObject({ + found: true, + nodes: [ + { type: 'Switch', value: '1', checked: true }, + { type: 'SecureTextField', editable: true, password: true }, + { type: 'TextField', enabled: false, editable: true, password: false }, + { + type: 'Other', + value: 'checkbox, unchecked', + checked: false, + iosValueSemantics: { role: 'checkbox' }, + }, + ], + }); +}); diff --git a/packages/platform-apple/src/core/runner-selector-query.ts b/packages/platform-apple/src/core/runner-selector-query.ts index f33da420b7..0b3d590c8d 100644 --- a/packages/platform-apple/src/core/runner-selector-query.ts +++ b/packages/platform-apple/src/core/runner-selector-query.ts @@ -1,6 +1,8 @@ +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { ElementSelectorKey } from '@agent-device/contracts/interactor-types'; import type { AppleRunnerCommandOptions } from '../runner/index.ts'; -import type { DeviceInfo } from '@agent-device/kernel/device'; +import { isHandheldAppleDevice, type DeviceInfo } from '@agent-device/kernel/device'; import { runAppleRunnerCommand } from './runner-client.ts'; /** The single Apple runner `querySelector` command builder shared by bound and safety-probe reads. */ @@ -10,7 +12,7 @@ export async function queryAppleRunnerSelector( appBundleId: string | undefined, options: AppleRunnerCommandOptions, ): Promise> { - return await runAppleRunnerCommand( + const result = await runAppleRunnerCommand( device, { command: 'querySelector', @@ -20,4 +22,16 @@ export async function queryAppleRunnerSelector( }, options, ); + return { + ...result, + ...(Array.isArray(result.nodes) + ? { + nodes: (result.nodes as RawSnapshotNode[]).map((node) => + interpretIosNode(node, { + reactNativeValue: isHandheldAppleDevice(device), + }), + ), + } + : {}), + }; } diff --git a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts index 2d40311e16..bb34ec370e 100644 --- a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts +++ b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts @@ -1,4 +1,4 @@ -import { IOS_SIMULATOR } from '../../__tests__/device-fixtures.ts'; +import { IOS_SIMULATOR, MACOS_DEVICE } from '../../__tests__/device-fixtures.ts'; import { AppError } from '@agent-device/kernel/errors'; import assert from 'node:assert/strict'; import { test } from 'vitest'; @@ -426,3 +426,33 @@ test('runner presented and quality nodes share RN interpretation without replaci } assert.equal('checked' in node, false); }); + +for (const device of [IOS_SIMULATOR, MACOS_DEVICE]) { + test(`runner ${device.appleOs} reads native states on presented and quality payloads`, () => { + const nodes = [ + { index: 0, type: 'Switch', value: '0' }, + { index: 1, type: 'SecureTextField' }, + { index: 2, type: 'TextField', enabled: false }, + { index: 3, type: 'Switch', value: '1', checked: false }, + { index: 4, type: 'Other', value: 'checkbox, unchecked' }, + ]; + const result = readAppleSnapshotResult( + { nodes, qualityPayload: { nodes, truncated: false } }, + device, + ); + for (const interpreted of [result.nodes!, result.qualityPayload!.nodes]) { + assert.equal(interpreted[0]!.checked, false); + assert.deepEqual([interpreted[1]!.editable, interpreted[1]!.password], [true, true]); + assert.deepEqual( + [interpreted[2]!.enabled, interpreted[2]!.editable, interpreted[2]!.password], + [false, true, false], + ); + assert.equal(interpreted[3]!.checked, false); + assert.equal( + interpreted[4]!.iosValueSemantics?.role, + device.appleOs === 'ios' ? 'checkbox' : undefined, + ); + } + assert.equal('checked' in nodes[0]!, false); + }); +} diff --git a/packages/platform-apple/src/runner/snapshot-presentation.ts b/packages/platform-apple/src/runner/snapshot-presentation.ts index eac6560dea..60c54496bc 100644 --- a/packages/platform-apple/src/runner/snapshot-presentation.ts +++ b/packages/platform-apple/src/runner/snapshot-presentation.ts @@ -1,5 +1,4 @@ import { isHandheldAppleDevice, type DeviceInfo } from '@agent-device/kernel/device'; -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; import type { IosRunnerQualityPayloadFacts, IosSnapshotInput, @@ -11,6 +10,7 @@ import { presentIosRunnerSnapshot, toIosSnapshotEngineErrorDetails, } from '@agent-device/capture-kit/ios-snapshot-engine'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { resolveIosViewportEvidenceFromRoots } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import { renderSnapshotQualityWarnings } from '@agent-device/capture-kit/quality-warnings'; import { @@ -70,15 +70,15 @@ export function readAppleSnapshotResult( ); const nodes = Array.isArray(result.nodes) ? (result.nodes as RawSnapshotNode[]) : undefined; const qualityPayload = readQualityPayload(result.qualityPayload, systemSurface); - const interpretValues = isHandheldAppleDevice(device); + const interpret = (node: RawSnapshotNode) => + interpretIosNode(node, { reactNativeValue: isHandheldAppleDevice(device) }); return { - nodes: interpretValues ? nodes?.map(interpretIosReactNativeValue) : nodes, + nodes: nodes?.map(interpret), truncated: typeof result.truncated === 'boolean' ? result.truncated : undefined, quality: readSnapshotQualityVerdict(result.snapshotQuality), - qualityPayload: - interpretValues && qualityPayload - ? { ...qualityPayload, nodes: qualityPayload.nodes.map(interpretIosReactNativeValue) } - : qualityPayload, + qualityPayload: qualityPayload + ? { ...qualityPayload, nodes: qualityPayload.nodes.map(interpret) } + : qualityPayload, runnerFatal: result.runnerFatal === true, ...(keyboard ? { keyboard } : {}), ...(systemSurface ? { systemSurface } : {}), diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 8413a49d2e..d8c29df4eb 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -630,3 +630,23 @@ test('bridge RN class values carry shared semantics while native class evidence expanded: true, }); }); + +test('the AX bridge publishes native switch and text entry facts without inventing unknown states', () => { + const tree = decodeSnapshotBridgeTree( + { + [application]: 'Application', + [children]: [ + { [automationType]: 40, XC_kAXXCAttributeValue: 0, [children]: [] }, + { [automationType]: 50, [traits]: '0', [children]: [] }, + { [automationType]: 49, [traits]: '256', [children]: [] }, + { [automationType]: 40, [children]: [] }, + ], + }, + { truncated: false }, + limits, + ).nodes; + assert.equal(tree[1]!.checked, false); + assert.deepEqual([tree[2]!.editable, tree[2]!.password], [true, true]); + assert.deepEqual([tree[3]!.enabled, tree[3]!.editable, tree[3]!.password], [false, true, false]); + assert.equal(tree[4]!.checked, undefined); +}); diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index 2c835cf548..be87d3bf9e 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -1,4 +1,4 @@ -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { isGeometricallyActionable, isPositiveFiniteRect, @@ -232,7 +232,7 @@ export function decodeSnapshotBridgeTree( } const index = nodes.length; const node = nodeFacts(value, index, parentIndex, depth); - nodes.push(interpretIosReactNativeValue(node)); + nodes.push(interpretIosNode(node)); maxTraversalDepth = Math.max(maxTraversalDepth, depth); if (isHostedRemoteLeaf(node, children.length, underRemoteHost)) { hostedRemoteLeaves.push(node.rect); diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts index d6a40bd337..a42176d710 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from 'node:fs'; +import type { IosTreeNode } from './snapshot.ts'; import { expect, test, vi } from 'vitest'; import { captureLimrunIosSnapshot } from './ios-snapshot-adapter.ts'; import { @@ -99,6 +101,52 @@ test('the Limrun acquired result keeps provider provenance and SDK screen dimens }); }); +const nativeStates = JSON.parse( + readFileSync( + new URL('../../../contracts/fixtures/native-snapshot-states.json', import.meta.url), + 'utf8', + ), +) as { + cases: { name: string; input: IosTreeNode; expected: Record }[]; +}; +for (const row of nativeStates.cases.filter( + ({ input }) => !['checked', 'password', 'editable'].some((key) => key in input), +)) { + test(`limrun acquires native states: ${row.name}`, async () => { + const result = await captureLimrunIosSnapshot(createLimrunSnapshotSession(row.input)); + const node = result.acquisition.nodes[0]!; + const states = Object.fromEntries( + ['checked', 'password', 'editable'].flatMap((key) => { + const value = node[key as 'checked' | 'password' | 'editable']; + return value === undefined ? [] : [[key, value]]; + }), + ); + expect(states).toEqual(row.expected); + }); +} + +test('Limrun secure TextField subrole survives acquisition without changing the native type', async () => { + const result = await captureLimrunIosSnapshot( + createLimrunSnapshotSession({ + type: 'TextField', + role: 'AXTextField', + subrole: 'AXSecureTextField', + traits: ['TextOperationsAvailable', 'Scrollable', 'TextEntry', 'SecureTextField'], + role_description: 'secure text field', + AXUniqueId: 'native-secure', + AXValue: 'Native input', + enabled: true, + }), + ); + expect(result.acquisition.nodes[0]).toMatchObject({ + type: 'TextField', + identifier: 'native-secure', + password: true, + editable: true, + roleDescription: 'secure text field', + }); +}); + test('Limrun carries docked tree evidence and never infers absence from missing nodes', async () => { const tree = limrunSnapshotTree(); expect( diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.ts b/packages/provider-limrun/src/ios-snapshot-adapter.ts index 4f517fa466..201782a94b 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.ts @@ -1,4 +1,4 @@ -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { createIosSnapshotAcquisition, resolveIosViewportEvidenceFromRoots, @@ -13,7 +13,7 @@ export async function captureLimrunIosSnapshot( ): Promise { const tree = (await session.client.elementTree()) as IosTreeNode[]; const viewport = readLimrunViewport(tree, session.client.deviceInfo); - const nodes = flattenIosTree(tree).map(interpretIosReactNativeValue); + const nodes = flattenIosTree(tree).map((node) => interpretIosNode(node)); return createIosSnapshotAcquisition({ producer: 'limrun-ios-tree', ...acquireProviderKeyboard({ diff --git a/packages/provider-limrun/src/snapshot.test.ts b/packages/provider-limrun/src/snapshot.test.ts index 46d508a9bf..02e0cfd493 100644 --- a/packages/provider-limrun/src/snapshot.test.ts +++ b/packages/provider-limrun/src/snapshot.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'vitest'; -import { toIosSelector } from './snapshot.ts'; +import { flattenIosTree, toIosSelector } from './snapshot.ts'; test('agent-device selector keys become the accessibility keys the iOS instance matches', () => { expect(toIosSelector({ key: 'id', value: 'email-field' })).toEqual({ @@ -14,3 +14,18 @@ test('agent-device selector keys become the accessibility keys the iOS instance test('visible text targets the provider label field, which is where the tree exposes it', () => { expect(toIosSelector({ key: 'text', value: 'Sign In' })).toEqual({ AXLabel: 'Sign In' }); }); + +test('Limrun raw mapping forwards observed Header traits and SDK role descriptions', () => { + const [heading, unknown] = flattenIosTree([ + { type: 'Heading', traits: ['Scrollable', 'Header'], role_description: 'heading' }, + { type: 'TextField', traits: ['TextEntry'] }, + ]); + expect(heading).toMatchObject({ heading: true, roleDescription: 'heading' }); + expect(unknown!.heading).toBeUndefined(); + expect(unknown!.roleDescription).toBeUndefined(); + expect([unknown!.checked, unknown!.password, unknown!.editable]).toEqual([ + undefined, + undefined, + undefined, + ]); +}); diff --git a/packages/provider-limrun/src/snapshot.ts b/packages/provider-limrun/src/snapshot.ts index 0c87e6d05c..d2f68845bc 100644 --- a/packages/provider-limrun/src/snapshot.ts +++ b/packages/provider-limrun/src/snapshot.ts @@ -15,7 +15,9 @@ export type IosTreeNode = { rect?: { x?: number; y?: number; width?: number; height?: number }; enabled?: boolean; role?: string; + subrole?: string | null; selected?: boolean; + role_description?: string; hittable?: boolean; traits?: string[]; pid?: number; @@ -52,6 +54,9 @@ function mapIosNode( rect: readIosNodeRect(node), enabled: node.enabled, selected: node.selected, + password: node.subrole === 'AXSecureTextField' ? true : undefined, + heading: node.traits?.includes('Header') === true ? true : undefined, + roleDescription: node.role_description, hittable: node.hittable, depth: options.depth, parentIndex: options.parentIndex, diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index d39f6b5bdb..b35178ac50 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import { resolveKeyboardTapOcclusion } from '@agent-device/contracts/tap-keyboard-occlusion'; @@ -65,6 +66,39 @@ test('Appium iOS adapter carries missing viewport evidence without presenting or ]); }); +const iosNativeStateTable = JSON.parse( + readFileSync( + new URL('../../../contracts/fixtures/native-snapshot-states.json', import.meta.url), + 'utf8', + ), +) as { + cases: { + name: string; + input: Record; + expected: Record; + }[]; +}; +for (const row of iosNativeStateTable.cases) { + test(`Appium iOS native states: ${row.name}`, () => { + const type = row.input.type ?? 'Other'; + const attrs = Object.entries(row.input) + .filter(([key]) => key !== 'type') + .map(([key, value]) => `${key}="${value}"`) + .join(' '); + const node = acquireWebDriverIosSnapshot(`<${type} label="Node" ${attrs} />`).acquisition + .nodes[0]!; + assert.deepEqual( + Object.fromEntries( + ['checked', 'password', 'editable'].flatMap((key) => { + const value = node[key as 'checked' | 'password' | 'editable']; + return value === undefined ? [] : [[key, value]]; + }), + ), + row.expected, + ); + }); +} + test('Appium carries visibility-only and derived keyboard facts before projection', () => { assert.deepEqual(acquireWebDriverIosSnapshot(SOURCE, undefined, true).acquisition.keyboard, { kind: 'visibility-only', @@ -106,7 +140,7 @@ test.each([ '_UIRemoteKeyboardPlaceholderView', 'TUISystemInputAssistantView', 'TUISystemInputAssistantPageView', -])('app controls under %s remain unowned through projection and state assembly', (role) => { +])('app controls under %s remain unowned through projection and state assembly', async (role) => { const source = SOURCE.replace( '', ``, diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 102702f3f2..ce88471665 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -1,4 +1,4 @@ -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { createIosSnapshotAcquisition, resolveIosViewportEvidenceFromRoots, @@ -23,7 +23,7 @@ export function acquireWebDriverIosSnapshot( producer: APPIUM_PRODUCER, viewport, ...acquireProviderKeyboard({ - nodes: sourceFacts.nodes.map(interpretIosReactNativeValue), + nodes: sourceFacts.nodes.map((node) => interpretIosNode(node)), viewport: viewport.kind === 'missing' ? null : viewport.rect, visibility, }), diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index 3174ef0499..04d11df0bf 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -52,7 +52,7 @@ test("Android WebDriver facts name a node by its text, carry a content descripti field!.hintShowing, field!.editable, ], - ['Ada Lovelace', 'Ada Lovelace', 'Name field', 'Your name', undefined, true], + ['Ada Lovelace', 'Ada Lovelace', 'Name field', 'Your name', undefined, undefined], ); assert.deepEqual( [label!.label, label!.value, 'editable' in label!], @@ -60,7 +60,7 @@ test("Android WebDriver facts name a node by its text, carry a content descripti ); }); -test('Android WebDriver facts read a hinted field as empty, a masked field as a password, and a disabled field as not editable', () => { +test('Android WebDriver facts read a hinted field as empty, a masked field as a password, and editability as unavailable when unreported', () => { const [hinted, secure, disabled, lookalike] = parseWebDriverSourceFacts( '' + '' + @@ -76,8 +76,8 @@ test('Android WebDriver facts read a hinted field as empty, a masked field as a [hinted!.value, hinted!.label, hinted!.placeholder, hinted!.hintShowing], ['', 'Your name', 'Your name', true], ); - assert.deepEqual([secure!.password, secure!.editable], [true, true]); - assert.deepEqual([disabled!.value, disabled!.editable], ['locked', false]); + assert.deepEqual([secure!.password, secure!.editable], [true, undefined]); + assert.deepEqual([disabled!.value, disabled!.editable], ['locked', undefined]); // Only the class name's last segment decides, as `isFillableType` reads it. assert.deepEqual( [lookalike!.label, lookalike!.value, 'editable' in lookalike!], @@ -104,9 +104,7 @@ test('Android WebDriver facts carry the checked state of checkable controls only ); }); -// The native iOS runner reports no checked, editable, or password facts, so the WebDriver -// projection stays at that parity: a switch keeps its state as its value. -test('iOS WebDriver facts keep a switch value as a value and add no field facts', () => { +test('iOS WebDriver source preserves native values for acquisition interpretation', () => { const [toggle, field, secure] = parseWebDriverSourceFacts( '' + '' + @@ -115,10 +113,10 @@ test('iOS WebDriver facts keep a switch value as a value and add no field facts' '', ).nodes; - assert.deepEqual([toggle!.value, 'checked' in toggle!], ['1', false]); + assert.deepEqual([toggle!.value, toggle!.checked], ['1', undefined]); assert.deepEqual( - [field!.value, 'editable' in field!, 'password' in secure!], - ['ada@example.com', false, false], + [field!.value, field!.editable, secure!.password], + ['ada@example.com', undefined, undefined], ); }); @@ -254,7 +252,7 @@ const ANDROID_CLASS_IDENTITY_CASES = [ widget: 'EditText', className: 'android.widget.EditText', kind: 'text-field', - editable: true, + editable: undefined, scrollContainer: false, }, { @@ -436,3 +434,52 @@ test('Android source preserves native package provenance on anonymous IME window ); assert.equal(nodes[0]?.identifier, undefined); }); + +test('WebDriver preserves explicit native states on custom controls and field overrides', () => { + for (const platform of ['ios', 'android'] as const) { + const [custom, field, unknown] = parseWebDriverSourceFacts( + '' + + '' + + `<${platform === 'ios' ? 'XCUIElementTypeSecureTextField' : 'android.widget.EditText'} editable="false" password="false" enabled="true" />` + + '' + + '', + platform, + ).nodes; + assert.deepEqual( + [ + custom!.checked, + custom!.password, + custom!.editable, + custom!.heading, + custom!.roleDescription, + ], + [false, false, false, false, 'Custom role'], + ); + assert.deepEqual([field!.editable, field!.password], [false, false]); + assert.deepEqual( + [unknown!.checked, unknown!.password, unknown!.editable, unknown!.heading], + [undefined, undefined, undefined, undefined], + ); + } +}); + +test('Android Appium role descriptions read the exact native extras key and preserve explicit attributes', () => { + const [native, explicit, empty, unrelated] = parseWebDriverSourceFacts( + '' + + '' + + '' + + '' + + '' + + '', + 'android', + ).nodes; + assert.deepEqual( + [ + native!.roleDescription, + explicit!.roleDescription, + empty!.roleDescription, + unrelated!.roleDescription, + ], + ['Tab List', 'Link', undefined, undefined], + ); +}); diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 683d580e58..6da5680ebf 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -1,4 +1,4 @@ -import { androidFieldValue, isFillableType, normalizeType } from '@agent-device/contracts/snapshot'; +import { androidFieldValue } from '@agent-device/contracts/snapshot'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { parseBounds } from '@agent-device/kernel/bounds'; import { AppError } from '@agent-device/kernel/errors'; @@ -166,7 +166,6 @@ function sourceNodeFromAttributes( rect: RawSnapshotNode['rect'], platform: WebDriverSourcePlatform, ): RawSnapshotNode { - const field = textFieldFacts(type, attrs, platform); return { index, type, @@ -179,8 +178,13 @@ function sourceNodeFromAttributes( ...(platform === 'android' && attrs.package ? { bundleId: attrs.package } : {}), rect, ...sourceStateFacts(attrs, rect, platform), - ...(field === undefined ? {} : field.facts), - ...checkedFact(type, attrs, platform), + ...nativeStateFacts(type, attrs, platform), + ...optionalFact('heading', parseWebDriverBoolean(attrs.heading)), + ...optionalFact( + 'roleDescription', + firstWebDriverAttribute(attrs, ['roleDescription', 'role-description']) ?? + (platform === 'android' ? androidRoleDescription(attrs.extras) : undefined), + ), depth, parentIndex, }; @@ -209,40 +213,6 @@ function labelFacts( }; } -type TextFieldFacts = Readonly<{ - facts: Pick; -}>; - -/** - * The field facts a text entry control carries. UiAutomator2 reports a field's content as `text` - * (the same attribute a label carries on every other node), its hint as `hint`, and whether it - * masks input as `password`. XCUITest reports content as `value`, which every node already - * carries, and the native iOS runner reports no field facts, so none are derived there. - */ -function textFieldFacts( - type: string, - attrs: Record, - platform: WebDriverSourcePlatform, -): TextFieldFacts | undefined { - return platform === 'android' ? androidTextFieldFacts(type, attrs) : undefined; -} - -function androidTextFieldFacts( - type: string, - attrs: Record, -): TextFieldFacts | undefined { - const password = parseWebDriverBoolean(attrs.password); - if (!(normalizeType(type) && isFillableType(type, 'android')) && password !== true) { - return undefined; - } - return { - facts: { - editable: parseWebDriverBoolean(attrs.enabled, true) === true, - ...optionalFact('password', password), - }, - }; -} - function androidValueFacts( type: string, attrs: Record, @@ -288,21 +258,27 @@ function optionalFact( /** Android classes that are checkable when the page source names no `checkable` attribute. */ const ANDROID_CHECKABLE_CLASS = /(CheckBox|RadioButton|Switch|ToggleButton|CheckedTextView)$/; -/** - * The checked state of a checkable Android control. UiAutomator2 reports `checkable` and - * `checked` on every node, `false` on the many that cannot be checked, so only a checkable node - * carries the fact. XCUITest reports a switch's state as its `value`, which stays a value, as the - * native iOS runner reports it. - */ -function checkedFact( +function nativeStateFacts( type: string, attrs: Record, platform: WebDriverSourcePlatform, -): Pick { - if (platform !== 'android') return {}; - const checkable = parseWebDriverBoolean(attrs.checkable) ?? ANDROID_CHECKABLE_CLASS.test(type); +): Pick { + const raw = { + ...optionalFact('password', parseWebDriverBoolean(attrs.password)), + ...optionalFact('editable', parseWebDriverBoolean(attrs.editable)), + }; const checked = parseWebDriverBoolean(attrs.checked); - return checkable && checked !== undefined ? { checked } : {}; + if (platform === 'ios') { + return { ...raw, ...optionalFact('checked', checked) }; + } + const checkable = parseWebDriverBoolean(attrs.checkable) ?? ANDROID_CHECKABLE_CLASS.test(type); + return { ...raw, ...(checkable && checked !== undefined ? { checked } : {}) }; +} + +function androidRoleDescription(extras: string | undefined): string | undefined { + const prefix = 'AccessibilityNodeInfo.roleDescription='; + const entry = extras?.split(';').find((value) => value.startsWith(prefix)); + return nonEmptyWebDriverAttribute(entry?.slice(prefix.length)); } function sourceStateFacts( diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index c9e6100ad5..b3628e0b34 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -485,7 +485,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/durable-capture-runtime-recovery', '@agent-device/capture-kit/durable-capture/session-binding', '@agent-device/capture-kit/durable-json', - '@agent-device/capture-kit/ios-react-native-value', + '@agent-device/capture-kit/ios-node-interpretation', '@agent-device/capture-kit/ios-snapshot-acquisition', '@agent-device/capture-kit/ios-snapshot-engine', '@agent-device/capture-kit/ios-snapshot-planning', diff --git a/src/commands/capture/runtime/snapshot-unchanged.test.ts b/src/commands/capture/runtime/snapshot-unchanged.test.ts index b678d721ee..1575b6b73b 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'vitest'; -import { interpretIosReactNativeValue } from '@agent-device/capture-kit/ios-react-native-value'; +import { interpretIosNode } from '@agent-device/capture-kit/ios-node-interpretation'; import { attachRefs } from '@agent-device/kernel/snapshot'; import fc from 'fast-check'; import { @@ -342,10 +342,10 @@ test.each([{ editable: true }, { password: true }])( checked: false, }; const previous = snapshot('Preferences', { - nodes: attachRefs([interpretIosReactNativeValue(node)]), + nodes: attachRefs([interpretIosNode(node)]), }); const current = snapshot('Preferences', { - nodes: attachRefs([interpretIosReactNativeValue({ ...node, ...change })]), + nodes: attachRefs([interpretIosNode({ ...node, ...change })]), }); expect(previous.nodes[0]?.iosValueSemantics?.checked).toBe(true); expect(current.nodes[0]?.iosValueSemantics).toBeUndefined(); diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 67c1103f21..60d5590471 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -347,6 +347,10 @@ Android snapshot nodes and `get attrs` (including the digest response) carry the them. Explicit `false` and `0` are kept; an absent field means the fact was unavailable, not false. `hintShowing` and `placeholder` need Android API 26 or later, `heading` API 28 or later. +On iOS, the Apple runner, Simulator accessibility bridge, Appium, and Limrun publish native +`checked`, `password`, and `editable` facts when available. macOS runner snapshots publish the +same native facts. + - `selected` is the accessibility selected state an app sets on a control — the active bottom-tab or segmented-control item, or the chosen row of a list. Android reports it explicitly as `true` or `false`; an older helper APK omits the field, which means the answer is unavailable rather than @@ -356,9 +360,20 @@ them. Explicit `false` and `0` are kept; an absent field means the fact was unav view an app marked checkable. Android reports it as `true` or `false` on those nodes only; a node that cannot be checked, or an older helper APK, omits the field. Snapshot text marks the node `[checked]` or `[unchecked]`, so a toggle that reads as plain text is one Android did not report as - checkable. + checkable. On iOS and the macOS runner, native switches, checkboxes, radio buttons, and toggles + carry `checked` when their native value reports a boolean state; an unavailable or unrecognized + state stays absent. +- `editable` is the native text-entry capability when available. `is editable` returns false for a + disabled node, even when `editable` is true. Otherwise it uses the published fact, including + explicit false, and falls back to the platform type only when the fact is absent. Android + Appium leaves `editable` absent when the source does not report it. +- `password` identifies secure text entry. On iOS and the macOS runner, native secure fields report + true and ordinary native text-entry fields report false. Android carries the reported native + password flag. - `heading` is the accessibility heading flag an app sets on a node, the way React Native's - `accessibilityRole="header"` does on a plain `View`; it is present only as `true`. + `accessibilityRole="header"` does on a plain `View`. The Android helper publishes true for a + heading; Appium preserves an explicit true or false when reported. Limrun reports true when + the node carries the native heading trait. - `roleDescription` is the localized role description an app sets beside the native class, verbatim (React Native writes `Tab`, `Tab List`, `Radio Group`, `Link`, `Menu`), when the class alone would not say what the control is. The `type` stays the class; a consumer maps the description to a role.