feat(snapshot): interpret iOS node facts once at producer seams

apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m

@@ -573,6 +573,24 @@ - (void)viewDidLoad {
     self.alertActivationBusyAnswer = busyAnswer;
   }
 
+  if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-native-states"]) {
+    for (NSUInteger index = 0; index < 3; index++) {
+      UITextField *field = [[UITextField alloc] initWithFrame:CGRectMake(40, 120 + index * 60, 260, 44)];
+      field.accessibilityIdentifier = @[@"native-field", @"native-secure", @"native-disabled"][index];
+      field.placeholder = @"Native input";
+      field.secureTextEntry = index == 1;
+      field.enabled = index != 2;
+      field.borderStyle = UITextBorderStyleRoundedRect;
+      [self.view addSubview:field];
+    }
+    for (NSUInteger index = 0; index < 2; index++) {
+      UISwitch *toggle = [[UISwitch alloc] initWithFrame:CGRectMake(40, 340 + index * 60, 60, 40)];
+      toggle.accessibilityIdentifier = index == 0 ? @"native-off" : @"native-on";
+      toggle.on = index == 1;
+      [self.view addSubview:toggle];
+    }
+  }
+
   if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) {
     BOOL digitCountValue =
         [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-count-value"];

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -99,6 +99,8 @@ extension RunnerTests {
     XCUIElement.ElementType.activityIndicator.rawValue: "ActivityIndicator",
     XCUIElement.ElementType.progressIndicator.rawValue: "ProgressIndicator",
     XCUIElement.ElementType.checkBox.rawValue: "CheckBox",
+    XCUIElement.ElementType.radioButton.rawValue: "RadioButton",
+    XCUIElement.ElementType.toggle.rawValue: "Toggle",
     XCUIElement.ElementType.menuItem.rawValue: "MenuItem",
     XCUIElement.ElementType.webView.rawValue: "WebView",
     XCUIElement.ElementType.other.rawValue: "Other",
@@ -497,23 +499,7 @@ extension RunnerTests {
     nodes[0] = interactiveRootNode(rect: rootRect)
     for candidate in candidates {
       nodes.append(
-        RawAXNode(
-          index: nodes.count,
-          type: candidate.type,
-          label: candidate.label,
-          identifier: candidate.identifier,
-          value: candidate.value,
-          placeholder: candidate.placeholder,
-          rect: candidate.rect,
-          enabled: candidate.enabled,
-          focused: candidate.focused,
-          selected: candidate.selected,
-          hittable: candidate.hittable,
-          depth: 1,
-          parentIndex: 0,
-          hiddenContentAbove: nil,
-          hiddenContentBelow: nil
-        )
+        candidate.reindexed(index: nodes.count, depth: 1, parentIndex: 0)
       )
     }
     return (

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift

@@ -357,23 +357,7 @@ extension RunnerTests {
     if uniqueCandidates.count < 2 { return [] }
 
     return uniqueCandidates.enumerated().map { offset, node in
-      RawAXNode(
-        index: startingIndex + offset,
-        type: node.type,
-        label: node.label,
-        identifier: node.identifier,
-        value: node.value,
-        placeholder: node.placeholder,
-        rect: node.rect,
-        enabled: node.enabled,
-        focused: node.focused,
-        selected: node.selected,
-        hittable: node.hittable,
-        depth: depth,
-        parentIndex: parentIndex,
-        hiddenContentAbove: nil,
-        hiddenContentBelow: nil
-      )
+      node.reindexed(index: startingIndex + offset, depth: depth, parentIndex: parentIndex)
     }
   }
 

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift

@@ -11,6 +11,14 @@ extension RunnerTests {
     guard case .resolved(let resolvedModal) = resolveBlockingSystemModal(deadline: deadline) else {
       return nil
     }
+    return snapshotForBlockingSystemModal(resolvedModal, deadline: deadline)
+    #endif
+  }
+
+  func snapshotForBlockingSystemModal(
+    _ resolvedModal: ResolvedBlockingSystemModal,
+    deadline: Date = .distantFuture
+  ) -> DataPayload? {
     let modal = resolvedModal.root
     let actions = resolvedModal.actions
 
@@ -62,7 +70,6 @@ extension RunnerTests {
     }
 
     return DataPayload(nodes: nodes, truncated: false)
-    #endif
   }
 
   func firstBlockingSystemModal(
@@ -217,13 +224,14 @@ extension RunnerTests {
   ) -> PresentedNode {
     let label = (labelOverride ?? element.label).trimmingCharacters(in: .whitespacesAndNewlines)
     let identifier = (identifierOverride ?? element.identifier).trimmingCharacters(in: .whitespacesAndNewlines)
+    let value = element.value.map { String(describing: $0).trimmingCharacters(in: .whitespacesAndNewlines) }
     return SnapshotPresentation.singleElementRead(
       RawAXNode(
         index: index,
         type: type,
         label: label.isEmpty ? nil : label,
         identifier: identifier.isEmpty ? nil : identifier,
-        value: nil,
+        value: value.flatMap { $0.isEmpty ? nil : $0 },
         rect: SnapshotRect(element.frame),
         enabled: element.isEnabled,
         focused: nil,

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift

@@ -0,0 +1,20 @@
+import XCTest
+import AgentDeviceSnapshotPresentation
+
+extension RunnerTests {
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
+  @MainActor
+  func testBlockingModalSnapshotPreservesNativeSwitchValues() throws {
+    app.launchArguments = ["--agent-device-native-states"]
+    app.launch()
+    defer { app.terminate() }
+    XCTAssertTrue(app.switches["native-off"].waitForExistence(timeout: appExistenceTimeout))
+    let resolved = ResolvedBlockingSystemModal(root: app, ownerApp: app, actions: actionableElements(in: app))
+    let payload = try XCTUnwrap(snapshotForBlockingSystemModal(resolved))
+    let off = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-off" })
+    let on = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-on" })
+    XCTAssertEqual(off.value, "0")
+    XCTAssertEqual(on.value, "1")
+  }
+#endif
+}

apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift

@@ -91,7 +91,7 @@ public struct RawAXNode: Equatable {
     return updated
   }
 
-  func reindexed(index: Int, depth: Int, parentIndex: Int?) -> RawAXNode {
+  public func reindexed(index: Int, depth: Int, parentIndex: Int?) -> RawAXNode {
     var updated = self
     updated.index = index
     updated.depth = depth

contracts/fixtures/ios-react-native-value-semantics.json

@@ -301,7 +301,10 @@
         "type": "TextField",
         "value": "checkbox, checked"
       },
-      "expected": {}
+      "expected": {
+        "editable": true,
+        "password": false
+      }
     },
     {
       "name": "literal editable generic node",
@@ -334,7 +337,7 @@
       "node": {
         "index": 0,
         "type": "Other",
-        "value": "Kontrollk\u00e4stchen, nicht markiert",
+        "value": "Kontrollkästchen, nicht markiert",
         "role": "RCTView"
       },
       "expected": {}

contracts/fixtures/native-snapshot-states.json

@@ -0,0 +1,201 @@
+{
+  "cases": [
+    {
+      "name": "switch on",
+      "input": {
+        "type": "Switch",
+        "value": "1"
+      },
+      "expected": {
+        "checked": true
+      }
+    },
+    {
+      "name": "switch off",
+      "input": {
+        "type": "Switch",
+        "value": "0"
+      },
+      "expected": {
+        "checked": false
+      }
+    },
+    {
+      "name": "checkbox false",
+      "input": {
+        "type": "CheckBox",
+        "value": "false"
+      },
+      "expected": {
+        "checked": false
+      }
+    },
+    {
+      "name": "radio true",
+      "input": {
+        "type": "RadioButton",
+        "value": "true"
+      },
+      "expected": {
+        "checked": true
+      }
+    },
+    {
+      "name": "toggle off",
+      "input": {
+        "type": "Toggle",
+        "value": "0"
+      },
+      "expected": {
+        "checked": false
+      }
+    },
+    {
+      "name": "switch no value",
+      "input": {
+        "type": "Switch"
+      },
+      "expected": {}
+    },
+    {
+      "name": "mixed remains unknown",
+      "input": {
+        "type": "CheckBox",
+        "value": "mixed"
+      },
+      "expected": {}
+    },
+    {
+      "name": "RN encoding remains raw",
+      "input": {
+        "type": "Switch",
+        "value": "checked, disabled"
+      },
+      "expected": {}
+    },
+    {
+      "name": "unrelated numeric value",
+      "input": {
+        "type": "Slider",
+        "value": "1"
+      },
+      "expected": {}
+    },
+    {
+      "name": "unknown type",
+      "input": {
+        "value": "0"
+      },
+      "expected": {}
+    },
+    {
+      "name": "TextField",
+      "input": {
+        "type": "TextField"
+      },
+      "expected": {
+        "editable": true,
+        "password": false
+      }
+    },
+    {
+      "name": "SearchField",
+      "input": {
+        "type": "SearchField"
+      },
+      "expected": {
+        "editable": true,
+        "password": false
+      }
+    },
+    {
+      "name": "TextView",
+      "input": {
+        "type": "TextView"
+      },
+      "expected": {
+        "editable": true,
+        "password": false
+      }
+    },
+    {
+      "name": "SecureTextField",
+      "input": {
+        "type": "SecureTextField"
+      },
+      "expected": {
+        "editable": true,
+        "password": true
+      }
+    },
+    {
+      "name": "disabled native field capability",
+      "input": {
+        "type": "TextField",
+        "enabled": false
+      },
+      "expected": {
+        "editable": true,
+        "password": false
+      }
+    },
+    {
+      "name": "XCUI prefix",
+      "input": {
+        "type": "XCUIElementTypeSecureTextField"
+      },
+      "expected": {
+        "editable": true,
+        "password": true
+      }
+    },
+    {
+      "name": "explicit unchecked wins",
+      "input": {
+        "type": "Switch",
+        "value": "1",
+        "checked": false
+      },
+      "expected": {
+        "checked": false
+      }
+    },
+    {
+      "name": "explicit read only wins",
+      "input": {
+        "type": "TextField",
+        "editable": false,
+        "password": true
+      },
+      "expected": {
+        "editable": false,
+        "password": true
+      }
+    },
+    {
+      "name": "explicit non secure wins",
+      "input": {
+        "type": "SecureTextField",
+        "password": false
+      },
+      "expected": {
+        "editable": true,
+        "password": false
+      }
+    },
+    {
+      "name": "custom raw facts",
+      "input": {
+        "type": "Other",
+        "editable": false,
+        "password": false,
+        "checked": true
+      },
+      "expected": {
+        "editable": false,
+        "password": false,
+        "checked": true
+      }
+    }
+  ]
+}

packages/capture-kit/package.json

@@ -110,6 +110,10 @@
       "types": "./src/screen-recording-transport.ts",
       "default": "./src/screen-recording-transport.ts"
     },
+    "./ios-node-interpretation": {
+      "types": "./src/ios-node-interpretation.ts",
+      "default": "./src/ios-node-interpretation.ts"
+    },
     "./ios-snapshot-acquisition": {
       "types": "./src/ios-snapshot-acquisition.ts",
       "default": "./src/ios-snapshot-acquisition.ts"
@@ -317,10 +321,6 @@
     "./durable-capture/session-binding": {
       "types": "./src/durable-capture/session-binding.ts",
       "default": "./src/durable-capture/session-binding.ts"
-    },
-    "./ios-react-native-value": {
-      "types": "./src/ios-react-native-value.ts",
-      "default": "./src/ios-react-native-value.ts"
     }
   },
   "devDependencies": {