From 6ce4bbb0b030198180babd4abdd934b5e328b0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 12:27:57 +0200 Subject: [PATCH 1/5] fix(settle): keep iOS keyboard chrome classified through interactive capture The regular snapshot projection dropped the Keyboard container as ineligible, re-parenting ~40 Key nodes to the app root where the settle chrome classifier could no longer anchor them, and the interactive projection flattens QuickType/dock siblings beside app content entirely. Every keyboard-raising fill paid the 1.5s broad-transition confirmation window on a keyboard pop counted as a screen replacement. - Add Keyboard to both projection eligibility sets (TS + Swift, parity) - Classify keyboard chrome by exact UIKit owned-class roles and the inputView placeholder marker (which owns no subtree) - Compare the broad-transition check against a chrome-free candidate signal so a keyboard pop no longer disarms the quiet window --- .../SnapshotPresentationProjection.swift | 4 + .../ios-snapshot-engine-conformance.json | 165 ++++++++++++++++++ .../src/ios-snapshot-engine/projection.ts | 4 + .../src/snapshot-chrome.fixtures.ts | 108 ++++++++++++ .../capture-kit/src/snapshot-chrome.test.ts | 127 +++++++++++++- packages/capture-kit/src/snapshot-chrome.ts | 108 +++++++++++- .../runtime/settle-keyboard-chrome.test.ts | 55 ++++++ .../runtime/stable-capture-signal.ts | 8 +- .../runtime/stable-capture.test.ts | 32 ++++ .../interaction/runtime/stable-capture.ts | 47 ++++- 10 files changed, 642 insertions(+), 16 deletions(-) create mode 100644 packages/capture-kit/src/snapshot-chrome.fixtures.ts create mode 100644 src/commands/interaction/runtime/settle-keyboard-chrome.test.ts diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift index 0c84c2f352..4fccadf38b 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift @@ -8,6 +8,10 @@ extension SnapshotPresentation { "Cell", "CheckBox", "CollectionView", + // The keyboard plane. The shared chrome classifier (`snapshot-chrome`) anchors keyboard + // detection on this container; dropping it re-parents every `Key` onto the app root, where no + // structural rule can tell a key from app content (#3469). + "Keyboard", "Link", "MenuItem", "Picker", diff --git a/contracts/fixtures/ios-snapshot-engine-conformance.json b/contracts/fixtures/ios-snapshot-engine-conformance.json index 983931ef98..5704f9a28b 100644 --- a/contracts/fixtures/ios-snapshot-engine-conformance.json +++ b/contracts/fixtures/ios-snapshot-engine-conformance.json @@ -1332,6 +1332,171 @@ } ] } + }, + { + "name": "keyboard plane and its key subtree survive interactive projection", + "swift": true, + "projection": "regular", + "interactiveOnly": true, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Button", + "label": "Continue", + "rect": { "x": 10, "y": 20, "width": 80, "height": 30 }, + "enabled": true, + "hittable": true, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Other", + "identifier": "inputView", + "rect": { "x": 0, "y": 120, "width": 320, "height": 120 }, + "enabled": true, + "hittable": true, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 3, + "type": "Keyboard", + "rect": { "x": 0, "y": 150, "width": 320, "height": 90 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 2 + }, + { + "index": 4, + "type": "Key", + "label": "q", + "rect": { "x": 4, "y": 157, "width": 29, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 3, + "parentIndex": 3 + }, + { + "index": 5, + "type": "Key", + "label": "w", + "rect": { "x": 37, "y": 157, "width": 29, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 3, + "parentIndex": 3 + }, + { + "index": 6, + "type": "Button", + "label": "Next keyboard", + "rect": { "x": 4, "y": 205, "width": 40, "height": 30 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 2 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "Button", + "label": "Continue", + "rect": { "x": 10, "y": 20, "width": 80, "height": 30 }, + "depth": 1, + "parentIndex": 0, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 2, + "type": "Other", + "label": null, + "rect": { "x": 0, "y": 120, "width": 320, "height": 120 }, + "depth": 1, + "parentIndex": 0, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 3, + "type": "Keyboard", + "label": null, + "rect": { "x": 0, "y": 150, "width": 320, "height": 90 }, + "depth": 2, + "parentIndex": 2, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 4, + "type": "Key", + "label": "q", + "rect": { "x": 4, "y": 157, "width": 29, "height": 40 }, + "depth": 3, + "parentIndex": 3, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 5, + "type": "Key", + "label": "w", + "rect": { "x": 37, "y": 157, "width": 29, "height": 40 }, + "depth": 3, + "parentIndex": 3, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 6, + "type": "Button", + "label": "Next keyboard", + "rect": { "x": 4, "y": 205, "width": 40, "height": 30 }, + "depth": 2, + "parentIndex": 2, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } } ] } diff --git a/packages/capture-kit/src/ios-snapshot-engine/projection.ts b/packages/capture-kit/src/ios-snapshot-engine/projection.ts index 863a081afd..04c07e7753 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/projection.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/projection.ts @@ -10,6 +10,10 @@ const REGULAR_ELIGIBLE_TYPES = new Set([ 'cell', 'checkbox', 'collectionview', + // The keyboard plane. The shared chrome classifier (`snapshot-chrome`) anchors keyboard + // detection on this container; dropping it re-parents every `Key` onto the app root, where no + // structural rule can tell a key from app content (#3469). + 'keyboard', 'link', 'menuitem', 'picker', diff --git a/packages/capture-kit/src/snapshot-chrome.fixtures.ts b/packages/capture-kit/src/snapshot-chrome.fixtures.ts new file mode 100644 index 0000000000..51c88dce7b --- /dev/null +++ b/packages/capture-kit/src/snapshot-chrome.fixtures.ts @@ -0,0 +1,108 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; + +/** + * The projected `snapshot -i` tree of a REAL fill-with-keyboard-up capture: + * iPhone 17 simulator (iOS 27), `com.callstack.agentdevicelab`, Catalog tab, + * `fill id=catalog-search kit` (October 2026), passed through the current iOS + * snapshot engine (regular projection, `interactiveOnly: true`) from the + * runner's raw capture. `rect` values are rounded to points; every + * classification input (type, role, parent chain, semantic text, hittability, + * enabled) is verbatim. + * + * The keyboard surfaces arrive flattened beside app content: app nodes 0-30, + * the system `inputView` host frame (32), the QuickType assistant bar (33-39), + * the `[Keyboard]` plane (40) with its layout wrapper (41) and keys (42-79), + * and the dock's "Next keyboard" / "Dictate" buttons (80-82). This is exactly + * the shape that made `--settle` misbehave (#3469): before the keyboard plane + * survived regular projection and `snapshot-chrome` learned the keyboard-owned + * UIKit classes, `withoutSettleChrome` stripped 0 of these 83 nodes, so every + * key spent a settle-diff line and the broad-transition check read the ~40 + * new `Key` nodes as a screen replacement. + */ +export function iosInteractiveKeyboardCaptureNodes(): RawSnapshotNode[] { + return [ + {"index":0,"depth":0,"type":"Application","role":"UIApplication","label":"Agent Device Tester","rect":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"hittable":false}, + {"index":1,"parentIndex":0,"depth":1,"type":"ScrollView","role":"RCTEnhancedScrollView","rect":{"x":18,"y":62,"width":366,"height":729},"enabled":true,"hittable":true}, + {"index":2,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"Catalog","rect":{"x":18,"y":62,"width":284,"height":36},"enabled":true,"hittable":true}, + {"index":3,"parentIndex":1,"depth":2,"type":"Other","role":"RCTViewComponentView","identifier":"catalog-title","rect":{"x":18,"y":62,"width":366,"height":109},"enabled":true,"hittable":true}, + {"index":4,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"1 results","rect":{"x":324,"y":68,"width":50,"height":14},"enabled":true,"hittable":true}, + {"index":5,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"Search, filter, scroll, favorite, and drill into detail without extra dependencies.","rect":{"x":18,"y":104,"width":284,"height":46},"enabled":true,"hittable":true}, + {"index":6,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"Catalog scroll: top","identifier":"catalog-scroll-state","rect":{"x":18,"y":195,"width":366,"height":17},"enabled":true,"hittable":true}, + {"index":7,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"Search","rect":{"x":34,"y":228,"width":333,"height":25},"enabled":true,"hittable":true}, + {"index":8,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"Search updates after a short debounce.","rect":{"x":34,"y":260,"width":333,"height":20},"enabled":true,"hittable":true}, + {"index":9,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"Find a product","rect":{"x":34,"y":294,"width":333,"height":17},"enabled":true,"hittable":true}, + {"index":10,"parentIndex":1,"depth":2,"type":"TextField","role":"RCTUITextField","label":"Search products","identifier":"catalog-search","value":"kit","rect":{"x":35,"y":319,"width":333,"height":51},"enabled":true,"hittable":true}, + {"index":11,"parentIndex":1,"depth":2,"type":"Button","role":"RCTViewComponentView","label":"All","identifier":"category-all","rect":{"x":34,"y":385,"width":46,"height":38},"enabled":true,"hittable":true}, + {"index":12,"parentIndex":1,"depth":2,"type":"Button","role":"RCTViewComponentView","label":"Starter Kits","identifier":"category-starter-kits","rect":{"x":88,"y":385,"width":106,"height":38},"enabled":true,"hittable":true}, + {"index":13,"parentIndex":1,"depth":2,"type":"Button","role":"RCTViewComponentView","label":"Produce","identifier":"category-produce","rect":{"x":202,"y":385,"width":85,"height":38},"enabled":true,"hittable":true}, + {"index":14,"parentIndex":1,"depth":2,"type":"Button","role":"RCTViewComponentView","label":"Bakery","identifier":"category-bakery","rect":{"x":34,"y":431,"width":75,"height":38},"enabled":true,"hittable":true}, + {"index":15,"parentIndex":1,"depth":2,"type":"Button","role":"RCTViewComponentView","label":"Pantry","identifier":"category-pantry","rect":{"x":118,"y":431,"width":73,"height":38},"enabled":true,"hittable":true}, + {"index":16,"parentIndex":1,"depth":2,"type":"Other","role":"RCTViewComponentView","identifier":"product-card-citrus-kit","rect":{"x":18,"y":497,"width":366,"height":294},"enabled":true,"hittable":true}, + {"index":17,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"Citrus Starter Kit","rect":{"x":34,"y":513,"width":333,"height":25},"enabled":true,"hittable":true}, + {"index":18,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"A bright bundle for quick search, detail, and quantity flows.","rect":{"x":34,"y":544,"width":333,"height":40},"enabled":true,"hittable":true}, + {"index":19,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"Popular","rect":{"x":45,"y":605,"width":45,"height":14},"enabled":true,"hittable":true}, + {"index":20,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"$12","rect":{"x":336,"y":601,"width":32,"height":22},"enabled":true,"hittable":true}, + {"index":21,"parentIndex":1,"depth":2,"type":"Button","role":"RCTViewComponentView","label":"Save Citrus Starter Kit","identifier":"favorite-citrus-kit","rect":{"x":34,"y":640,"width":56,"height":32},"enabled":true,"hittable":true}, + {"index":22,"parentIndex":1,"depth":2,"type":"StaticText","role":"RCTParagraphComponentView","label":"In cart: 0","rect":{"x":311,"y":648,"width":57,"height":16},"enabled":true,"hittable":true}, + {"index":23,"parentIndex":1,"depth":2,"type":"Button","role":"RCTViewComponentView","label":"View details","identifier":"details-citrus-kit","rect":{"x":34,"y":686,"width":333,"height":45},"enabled":true,"hittable":true}, + {"index":24,"parentIndex":1,"depth":2,"type":"Button","role":"RCTViewComponentView","label":"Add to cart","identifier":"add-citrus-kit","rect":{"x":34,"y":741,"width":333,"height":45},"enabled":true,"hittable":true}, + {"index":25,"parentIndex":0,"depth":1,"type":"TabBar","role":"UITabBar","label":"Tab Bar","rect":{"x":0,"y":791,"width":402,"height":83},"enabled":true,"hittable":true}, + {"index":26,"parentIndex":25,"depth":2,"type":"Button","role":"_UITabButton","label":"Home","rect":{"x":25,"y":795,"width":85,"height":54},"enabled":true,"hittable":true}, + {"index":27,"parentIndex":25,"depth":2,"type":"Button","role":"_UITabButton","label":"Catalog","rect":{"x":90,"y":795,"width":86,"height":54},"enabled":true,"hittable":true}, + {"index":28,"parentIndex":25,"depth":2,"type":"Button","role":"_UITabButton","label":"Form","rect":{"x":156,"y":795,"width":85,"height":54},"enabled":true,"hittable":true}, + {"index":29,"parentIndex":25,"depth":2,"type":"Button","role":"_UITabButton","label":"Audio","rect":{"x":221,"y":795,"width":85,"height":54},"enabled":true,"hittable":true}, + {"index":30,"parentIndex":25,"depth":2,"type":"Button","role":"_UITabButton","label":"Settings","rect":{"x":287,"y":795,"width":90,"height":54},"enabled":true,"hittable":true}, + {"index":31,"parentIndex":0,"depth":1,"type":"Toolbar","role":"_UIFloatingBarContainerView","label":"Toolbar","identifier":"Toolbar","rect":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"hittable":true}, + {"index":32,"parentIndex":0,"depth":1,"type":"Other","role":"_UIRemoteKeyboardPlaceholderView","identifier":"inputView","rect":{"x":0,"y":546,"width":402,"height":328},"enabled":true,"hittable":true}, + {"index":33,"parentIndex":0,"depth":1,"type":"Other","role":"TUISystemInputAssistantView","identifier":"SystemInputAssistantView","rect":{"x":0,"y":546,"width":402,"height":44},"enabled":true,"hittable":true}, + {"index":34,"parentIndex":33,"depth":2,"type":"Other","role":"TUISystemInputAssistantPageView","identifier":"CenterPageView","rect":{"x":0,"y":546,"width":402,"height":44},"enabled":true,"hittable":true}, + {"index":35,"parentIndex":34,"depth":3,"type":"ScrollView","role":"UIScrollView","rect":{"x":0,"y":546,"width":402,"height":44},"enabled":true,"hittable":true}, + {"index":36,"parentIndex":34,"depth":3,"type":"Other","role":"TUIPredictionView","label":"Typing Predictions","rect":{"x":0,"y":546,"width":402,"height":44},"enabled":true,"hittable":true}, + {"index":37,"parentIndex":36,"depth":4,"type":"Button","role":"TUIPredictionViewCell","label":"“k”","rect":{"x":0,"y":546,"width":134,"height":44},"enabled":true,"hittable":true}, + {"index":38,"parentIndex":36,"depth":4,"type":"Button","role":"TUIPredictionViewCell","label":"keep","rect":{"x":134,"y":546,"width":134,"height":44},"enabled":true,"hittable":true}, + {"index":39,"parentIndex":36,"depth":4,"type":"Button","role":"TUIPredictionViewCell","label":"kind","rect":{"x":268,"y":546,"width":134,"height":44},"enabled":true,"hittable":true}, + {"index":40,"parentIndex":0,"depth":1,"type":"Keyboard","role":"UIKeyboardAutomatic","rect":{"x":0,"y":590,"width":402,"height":226},"enabled":true,"hittable":true}, + {"index":41,"parentIndex":40,"depth":2,"type":"Other","role":"UIKeyboardLayoutStar","identifier":"UIKeyboardLayoutStar Preview","rect":{"x":0,"y":590,"width":402,"height":226},"enabled":true,"hittable":true}, + {"index":42,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"Padding-Left","rect":{"x":0,"y":590,"width":0,"height":0},"enabled":true,"hittable":false}, + {"index":43,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"Padding-Right","rect":{"x":0,"y":590,"width":0,"height":0},"enabled":true,"hittable":false}, + {"index":44,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"Padding-Left","rect":{"x":0,"y":590,"width":0,"height":0},"enabled":true,"hittable":false}, + {"index":45,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"Padding-Right","rect":{"x":0,"y":590,"width":0,"height":0},"enabled":true,"hittable":false}, + {"index":46,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"Padding-Left","rect":{"x":0,"y":590,"width":0,"height":0},"enabled":true,"hittable":false}, + {"index":47,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"Padding-Right","rect":{"x":0,"y":590,"width":0,"height":0},"enabled":true,"hittable":false}, + {"index":48,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"q","rect":{"x":5,"y":597,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":49,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"w","rect":{"x":44,"y":597,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":50,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"e","rect":{"x":84,"y":597,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":51,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"r","rect":{"x":123,"y":597,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":52,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"t","rect":{"x":163,"y":597,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":53,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"y","rect":{"x":202,"y":597,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":54,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"u","rect":{"x":242,"y":597,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":55,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"i","rect":{"x":281,"y":597,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":56,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"o","rect":{"x":321,"y":597,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":57,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"p","rect":{"x":360,"y":597,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":58,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"a","rect":{"x":24,"y":651,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":59,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"s","rect":{"x":64,"y":651,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":60,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"d","rect":{"x":103,"y":651,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":61,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"f","rect":{"x":143,"y":651,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":62,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"g","rect":{"x":182,"y":651,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":63,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"h","rect":{"x":222,"y":651,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":64,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"j","rect":{"x":261,"y":651,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":65,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"k","rect":{"x":301,"y":651,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":66,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"l","rect":{"x":340,"y":651,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":67,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"shift","identifier":"shift","rect":{"x":5,"y":705,"width":51,"height":54},"enabled":true,"hittable":true}, + {"index":68,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"z","rect":{"x":64,"y":705,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":69,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"x","rect":{"x":103,"y":705,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":70,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"c","rect":{"x":143,"y":705,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":71,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"v","rect":{"x":182,"y":705,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":72,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"b","rect":{"x":222,"y":705,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":73,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"n","rect":{"x":261,"y":705,"width":39,"height":54},"enabled":true,"hittable":true}, + {"index":74,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"m","rect":{"x":301,"y":705,"width":40,"height":54},"enabled":true,"hittable":true}, + {"index":75,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"delete","identifier":"delete","rect":{"x":348,"y":705,"width":52,"height":54},"enabled":true,"hittable":true}, + {"index":76,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"numbers","identifier":"more","rect":{"x":5,"y":759,"width":49,"height":54},"enabled":true,"hittable":true}, + {"index":77,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"Emoji","identifier":"emoji","rect":{"x":54,"y":759,"width":49,"height":54},"enabled":true,"hittable":true}, + {"index":78,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"space","identifier":"space","rect":{"x":103,"y":759,"width":197,"height":54},"enabled":true,"hittable":true}, + {"index":79,"parentIndex":41,"depth":3,"type":"Key","role":"UIAccessibilityElementKBKey","label":"return","identifier":"Return","rect":{"x":301,"y":759,"width":99,"height":54},"enabled":true,"hittable":true}, + {"index":80,"parentIndex":0,"depth":1,"type":"Button","role":"UIKeyboardDockItemButton","label":"Next keyboard","rect":{"x":8,"y":806,"width":68,"height":68},"enabled":true,"hittable":true}, + {"index":81,"parentIndex":0,"depth":1,"type":"Button","role":"UIKeyboardDockItemButton","label":"Dictate","identifier":"dictation","rect":{"x":325,"y":805,"width":68,"height":69},"enabled":true,"hittable":true}, + {"index":82,"parentIndex":0,"depth":1,"type":"Button","role":"UIKeyboardDockItemButton","label":"Dictate","identifier":"dictation","rect":{"x":325,"y":805,"width":68,"height":69},"enabled":true,"hittable":true}, + ]; +} diff --git a/packages/capture-kit/src/snapshot-chrome.test.ts b/packages/capture-kit/src/snapshot-chrome.test.ts index 55bd3e54e0..5e4b3b6481 100644 --- a/packages/capture-kit/src/snapshot-chrome.test.ts +++ b/packages/capture-kit/src/snapshot-chrome.test.ts @@ -1,7 +1,8 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { attachRefs, type RawSnapshotNode, type SnapshotNode } from '@agent-device/kernel/snapshot'; -import { collectSettleChromeRefs } from './snapshot-chrome.ts'; +import { collectSettleChromeRefs, withoutSettleChrome } from './snapshot-chrome.ts'; +import { iosInteractiveKeyboardCaptureNodes } from './snapshot-chrome.fixtures.ts'; function refFor(nodes: SnapshotNode[], label: string): string { const node = nodes.find((candidate) => candidate.label === label); @@ -9,6 +10,12 @@ function refFor(nodes: SnapshotNode[], label: string): string { return node.ref; } +function refForIdentifier(nodes: SnapshotNode[], identifier: string): string { + const node = nodes.find((candidate) => candidate.identifier === identifier); + assert.ok(node?.ref, `expected a node identified "${identifier}" with a ref`); + return node.ref; +} + // iOS keyboard window (iPhone 17 Pro shape): the [Keyboard] container holds the // keys, and the "Next keyboard"/"Dictate" assistant buttons render inside the // keyboard's own frame at the bottom of the screen. An inputAccessoryView @@ -200,3 +207,121 @@ test('Android classifier filters IME keys but keeps app dialog and unmarked Syst assert.equal(chromeRefs.has(refFor(nodes, 'Allow')), false); assert.equal(chromeRefs.has(refFor(nodes, 'Volume')), false); }); + +// #3469: the interactive projection flattens the keyboard's assistant bar and +// dock buttons beside app content and keeps the [Keyboard] plane as a bare +// top-level line, so the window walk cannot reach them. The keyboard-owned +// UIKit classes in `role` carry the classification through the flattening. +test('the real interactive keyboard capture collapses to one [Keyboard] diff line', () => { + const nodes = attachRefs(iosInteractiveKeyboardCaptureNodes()); + const survivors = withoutSettleChrome(nodes, undefined); + const chromeRefs = collectSettleChromeRefs(nodes, undefined); + + // Every key, the QuickType bar (host, page, predictions), and the dock + // buttons are chrome; the keyboard plane itself stays visible as the + // "keyboard appeared" signal. + const keyboard = nodes.find((node) => node.type === 'Keyboard'); + assert.ok(keyboard?.ref); + assert.equal(chromeRefs.has(keyboard.ref), true); + for (const node of nodes) { + const isKeyboardSurface = + node.type === 'Key' || + node.role === 'UIKeyboardLayoutStar' || + node.role === 'TUISystemInputAssistantView' || + node.role === 'TUIPredictionViewCell' || + node.role === 'UIKeyboardDockItemButton'; + if (isKeyboardSurface) assert.equal(chromeRefs.has(node.ref), true, `node ${node.index}`); + } + + // The stripped set collapses the ~50 keyboard-surface nodes; what survives is + // app content plus the lone keyboard plane line. + assert.equal(survivors.filter((node) => node.type === 'Key').length, 0); + assert.deepEqual( + survivors.filter((node) => node.index >= 31).map((node) => node.type), + ['Toolbar', 'Keyboard'], + ); + // App content never rides along: the search field the fill targeted survives. + assert.equal(chromeRefs.has(refFor(nodes, 'Search products')), false); +}); + +// The system inputView host frame is chrome itself, but an app's custom +// UITextField.inputView content renders under it and must survive. +test('the inputView host frame is stripped without swallowing app input-view content', () => { + const nodes = attachRefs([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Other', + identifier: 'inputView', + role: '_UIRemoteKeyboardPlaceholderView', + rect: { x: 0, y: 590, width: 402, height: 284 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'Custom keypad key', + identifier: 'app-keypad-7', + rect: { x: 10, y: 600, width: 80, height: 80 }, + hittable: true, + }, + ]); + const survivors = withoutSettleChrome(nodes, undefined); + + assert.equal( + survivors.some((node) => node.index === 1), + false, + 'the placeholder host frame itself is chrome', + ); + assert.equal( + collectSettleChromeRefs(nodes, undefined).has(refFor(nodes, 'Custom keypad key')), + false, + 'content the app hosts inside its own inputView is not keyboard chrome', + ); +}); + +// An app that hosts editable text inside a keyboard-owned subtree (its own +// composer riding the input-assistant host) keeps the field it is typing into. +test('editable text hosted inside a keyboard-owned subtree survives classification', () => { + const nodes = attachRefs([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Other', + role: 'TUISystemInputAssistantView', + identifier: 'SystemInputAssistantView', + rect: { x: 0, y: 546, width: 402, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'TextField', + label: 'Compose', + rect: { x: 8, y: 550, width: 300, height: 36 }, + hittable: true, + }, + ]); + const chromeRefs = collectSettleChromeRefs(nodes, undefined); + + assert.equal(chromeRefs.has(refForIdentifier(nodes, 'SystemInputAssistantView')), true); + assert.equal(chromeRefs.has(refFor(nodes, 'Compose')), false); +}); diff --git a/packages/capture-kit/src/snapshot-chrome.ts b/packages/capture-kit/src/snapshot-chrome.ts index c747a0c9c7..96f55a63ff 100644 --- a/packages/capture-kit/src/snapshot-chrome.ts +++ b/packages/capture-kit/src/snapshot-chrome.ts @@ -74,12 +74,67 @@ type SettleChrome = { const EMPTY_SETTLE_CHROME: SettleChrome = { strippedIndexes: new Set(), refs: new Set() }; +/** + * UIKit classes the keyboard owns whose nodes the regular projection can flatten beside app + * content — the key plane and a key when no `[Keyboard]` container survives, the system + * input-assistant (QuickType) bar, and the dock's "Next keyboard" / "Dictate" buttons. Once the + * projection has dropped the keyboard's own window, neither the container-subtree nor the + * window walk can reach them, but each node still carries its producing class in `role`. + * + * Exact class names only, so a near miss fails closed rather than widening what counts as + * chrome (the same discipline `geometry-policy.ts` applies to private UIKit roles). Classes are + * locale-independent, unlike every label these nodes carry, and private to the keyboard + * surfaces — an app cannot host one in its own view tree. + */ +const IOS_KEYBOARD_OWNED_ROLES: ReadonlySet = new Set([ + 'UIKeyboardLayoutStar', + 'UIAccessibilityElementKBKey', + 'UIKeyboardDockItemButton', + 'TUISystemInputAssistantView', + 'TUISystemInputAssistantPageView', + 'TUIPredictionView', + 'TUIPredictionViewCell', +]); + +/** + * The system `inputView` host frame: chrome itself, but it OWNS NO SUBTREE — an app's custom + * `UITextField.inputView` content (a drawn keypad, a composer) renders under this placeholder and + * stays a control the agent must see. Stripping only the marker keeps the keyboard's appearance + * at one visible line without swallowing app content; if the class is ever renamed, the inert + * marker line returns as noise rather than app content vanishing — noise fails loudly. + */ +const IOS_KEYBOARD_CHROME_MARKER_ROLES: ReadonlySet = new Set([ + '_UIRemoteKeyboardPlaceholderView', +]); + +function hasRole(node: SnapshotNode, roles: ReadonlySet): boolean { + return node.role !== undefined && roles.has(node.role); +} + +function isKeyboardOwnedRoleNode(node: SnapshotNode): boolean { + return hasRole(node, IOS_KEYBOARD_OWNED_ROLES); +} + function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { + const byIndex = new Map(nodes.map((node) => [node.index, node])); const containerIndexes = new Set( nodes.filter((node) => normalizeType(node.type ?? '') === 'keyboard').map((node) => node.index), ); - if (containerIndexes.size === 0) return EMPTY_SETTLE_CHROME; - const byIndex = new Map(nodes.map((node) => [node.index, node])); + const ownedSubtreeRootIndexes = new Set( + nodes.filter(isKeyboardOwnedRoleNode).map((node) => node.index), + ); + const markerIndexes = new Set( + nodes + .filter((node) => hasRole(node, IOS_KEYBOARD_CHROME_MARKER_ROLES)) + .map((node) => node.index), + ); + if ( + containerIndexes.size === 0 && + ownedSubtreeRootIndexes.size === 0 && + markerIndexes.size === 0 + ) { + return EMPTY_SETTLE_CHROME; + } const containerSubtreeIndexes = collectSubtreeIndexes(nodes, byIndex, containerIndexes); const chromeIndexes = new Set(containerSubtreeIndexes); const windowIndexes = resolveKeyboardWindowIndexes(nodes, byIndex, containerSubtreeIndexes); @@ -97,6 +152,21 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { )) { chromeIndexes.delete(index); } + // Keyboard-owned classes the projection flattened out of every reachable subtree are chrome + // by their own declaration, so the app-accessory guard above never exempts them. An editable + // text node inside one of them is the same inputAccessoryView guard in disguise: an app that + // hosts a text field in its own input view keeps it, as it does under the window pass. + for (const index of collectOwnedKeyboardChromeIndexes( + nodes, + byIndex, + ownedSubtreeRootIndexes, + containerSubtreeIndexes, + )) { + chromeIndexes.add(index); + } + // The system inputView host frame is itself chrome, but owning no subtree: an app's custom + // input view content under it survives classification. + for (const index of markerIndexes) chromeIndexes.add(index); const refs = new Set( nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref), ); @@ -108,6 +178,40 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { return { strippedIndexes, refs }; } +/** + * The subtree under each keyboard-owned-class root, minus editable text the app hosts inside it + * outside the keyboard container. A keyboard's own `Key` nodes never trip that guard — the walk + * claims them with their own subtree roots even when a stripped field re-parented them — and an + * app that hosts its whole composer inside its input view still loses every key, exactly as the + * container-descendant walk always did. + */ +function collectOwnedKeyboardChromeIndexes( + nodes: SnapshotNode[], + byIndex: Map, + ownedSubtreeRootIndexes: ReadonlySet, + containerSubtreeIndexes: ReadonlySet, +): Set { + const chromeIndexes = collectSubtreeIndexes(nodes, byIndex, ownedSubtreeRootIndexes); + for (const node of nodes) { + if ( + chromeIndexes.has(node.index) && + EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) && + !containerSubtreeIndexes.has(node.index) + ) { + chromeIndexes.delete(node.index); + for (const descendant of nodes) { + if ( + chromeIndexes.has(descendant.index) && + hasAncestorIn(descendant, byIndex, new Set([node.index])) + ) { + chromeIndexes.delete(descendant.index); + } + } + } + } + return chromeIndexes; +} + /** * Indexes of app-owned accessory content in a keyboard window: non-keyboard * nodes whose center sits ABOVE the keyboard container's top edge (an diff --git a/src/commands/interaction/runtime/settle-keyboard-chrome.test.ts b/src/commands/interaction/runtime/settle-keyboard-chrome.test.ts new file mode 100644 index 0000000000..f3005e293e --- /dev/null +++ b/src/commands/interaction/runtime/settle-keyboard-chrome.test.ts @@ -0,0 +1,55 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '@agent-device/capture-kit/snapshot-state-fixtures'; +import { iosInteractiveKeyboardCaptureNodes } from '@agent-device/capture-kit/snapshot-chrome-fixtures'; +import { selector } from './selector-read-utils.ts'; +import { createSettleDevice } from './__tests__/settle-device-fixtures.ts'; + +// #3469: `--settle` on iOS captured with `interactiveOnly: true`, whose +// projection flattened the keyboard beside app content, so +// `withoutSettleChrome` classified nothing: every key spent a settled-diff +// line. The fixture is the REAL projected `snapshot -i` tree of the Catalog +// search fill (iPhone 17 sim, iOS 27); the pre-fill baseline is its +// app-content prefix (nodes 0-30). + +function appOnlyNodes() { + return iosInteractiveKeyboardCaptureNodes().filter((node) => node.index <= 30); +} + +test('the settled diff after a fill shows the keyboard as one line, not one per key', async () => { + const before = makeSnapshotState(appOnlyNodes(), { + backend: 'xctest', + snapshotQuality: { state: 'healthy', backend: 'tree' }, + }); + const keyboardUp = makeSnapshotState(iosInteractiveKeyboardCaptureNodes(), { + backend: 'xctest', + snapshotQuality: { state: 'healthy', backend: 'tree' }, + }); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + // Capture 1 = pre-action baseline. Captures 2+ = settle loop sees the keyboard. + return { snapshot: captures === 1 ? before : keyboardUp }; + }, + }); + + const result = await device.interactions.fill(selector('id=catalog-search'), 'kit', { + session: 'default', + settle: {}, + }); + + const diff = result.settle?.diff; + assert.ok(diff); + const texts = diff.lines.map((line) => line.text).join('\n'); + // One line for the keyboard plane; no keys, QuickType candidates, or dock + // buttons spend budget. + assert.match(texts, /\[keyboard\]/); + assert.ok(!diff.lines.some((line) => /\[key\]/.test(line.text)), texts); + assert.ok(!/Next keyboard|Dictate|Typing Predictions|keep|kind/.test(texts), texts); + // App content is unchanged; the keyboard line and the floating toolbar the + // keyboard-window capture also reports are the only additions. + assert.equal(diff.summary.additions, 2); + assert.equal(diff.summary.removals, 0); +}); diff --git a/src/commands/interaction/runtime/stable-capture-signal.ts b/src/commands/interaction/runtime/stable-capture-signal.ts index 5193ecabf7..07b6299691 100644 --- a/src/commands/interaction/runtime/stable-capture-signal.ts +++ b/src/commands/interaction/runtime/stable-capture-signal.ts @@ -57,12 +57,12 @@ export function stableCaptureSignalsEqual( export function stableCaptureSignalsHaveBroadReplacement( left: StableCaptureSignal | undefined, - right: StableCaptureSignal, + right: StableCaptureSignal | undefined, ): boolean { // A backend transition must reset the ordinary quiet window, but it must not conceal a broad - // semantic replacement. Both signals already carry the same visible XCTest projection, and the - // transition baseline is only armed for XCTest sessions. - if (!left || left.nodes.length === 0 || right.nodes.length === 0) return false; + // semantic replacement. Both signals carry the same chrome-free visible projection the settle + // diff collapses, and the transition baseline is only armed for XCTest sessions. + if (!left || !right || left.nodes.length === 0 || right.nodes.length === 0) return false; const remainingIdentities = new Map(); for (const node of left.nodes) { remainingIdentities.set(node.identity, (remainingIdentities.get(node.identity) ?? 0) + 1); diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index 5a48eb150c..a714cc5491 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -2,6 +2,8 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import { makeSnapshotState } from '@agent-device/capture-kit/snapshot-state-fixtures'; +import { iosInteractiveKeyboardCaptureNodes } from '@agent-device/capture-kit/snapshot-chrome-fixtures'; import { createLocalArtifactAdapter } from '../../../io.ts'; import { createAgentDevice } from '../../../runtime.ts'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; @@ -239,6 +241,36 @@ test('settle keeps the default quiet window for a backend-less overlapping local assert.ok(outcome.waitedMs < 800, `local mutation settled after ${outcome.waitedMs}ms`); }); +// Why keyboard chrome is stripped from BOTH sides rather than the candidate +// only: with the keyboard up on both sides, its ~50 identical chrome nodes +// dominate the denominator, so no shared-identity ratio reaches the broad bar +// — on main too (keys counted on both sides, already shared). Stripping the +// same projection the settle diff collapses therefore only ever removes the +// false broad verdict (keyboard arrival counted as replacement); it cannot +// disarm a broad check that previously fired. The check that replaces the +// keyboard-up pair is the one the fill actually performs: baseline without +// the keyboard, capture with it, overlapping content — default window. +test('settle keeps the default quiet window when only keyboard chrome arrives', async () => { + const keyboardUp = makeSnapshotState(iosInteractiveKeyboardCaptureNodes(), { + backend: 'xctest', + snapshotQuality: { state: 'healthy', backend: 'tree' }, + }); + const { runtime } = staticSnapshotRuntime(keyboardUp); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + quietMs: 500, + timeoutMs: 5_000, + broadTransitionBaselineNodes: keyboardUp.nodes.filter((node) => node.index <= 30), + }, + ); + + assert.equal(outcome.settled, true); + assert.ok(outcome.waitedMs < 800, `keyboard arrival settled after ${outcome.waitedMs}ms`); +}); + function staticSnapshotRuntime(snapshot: SnapshotState) { let elapsedMs = 0; const runtime = createAgentDevice({ diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index 91ee51abbd..85ad0c6cda 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -19,6 +19,7 @@ import { type StableCaptureSignal, } from './stable-capture-signal.ts'; import { preferredSnapshotBackendForVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; +import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome'; /** * The quiet-window stable-capture loop shared by `wait stable` and the @@ -114,11 +115,13 @@ export async function runStableCaptureLoop( params.broadTransitionBaselineNodes, capture.snapshot, runtime.backend.platform, + session?.appBundleId, ); const signal = stableCaptureSignal(capture.snapshot); - requiredQuietMs = stableCaptureTransitionQuietMs({ + requiredQuietMs = stableCaptureNextQuietWindowMs({ baseline: transitionBaseline, - signal, + snapshot: capture.snapshot, + appBundleId: session?.appBundleId, requestedQuietMs: quietMs, currentQuietMs: requiredQuietMs, }); @@ -187,6 +190,7 @@ function stableCaptureTransitionBaseline( baselineNodes: SnapshotNode[] | undefined, snapshot: Parameters[0] | undefined, platform: AgentDeviceRuntime['backend']['platform'], + appBundleId: string | undefined, ): StableCaptureSignal | undefined { // SnapshotState.backend is optional at the runtime boundary. The bound iOS backend remains // authoritative when a presented capture omits that provenance; a declared non-XCTest backend @@ -205,7 +209,25 @@ function stableCaptureTransitionBaseline( // a coherent transitional tree win the default 500ms quiet window. const hasTinyPresentedBaseline = baselineNodes.length <= TINY_STABLE_TREE_NODE_COUNT; if (!hasCompleteViewportProjection && !hasTinyPresentedBaseline) return undefined; - return stableCaptureSignal({ ...snapshot, nodes: baselineNodes }); + return stableCaptureReplacementSignal({ ...snapshot, nodes: baselineNodes }, appBundleId); +} + +/** + * The comparison signal for the broad-replacement verdict: the capture minus the chrome the + * settle diff also collapses (`withoutSettleChrome`). Keyboard surface arriving or leaving — + * ~40 keys, the candidate bar, the dock buttons on a fill — is not a screen replacement, and + * counting it as one made every keyboard-raising action pay the 1.5 s confirmation window + * (#3469). The survivor rule lives in the shared classifier, so this signal and the diff agree + * on what keyboard chrome is; here it only decides the quiet window, never the visible diff. + */ +function stableCaptureReplacementSignal( + snapshot: Parameters[0], + appBundleId: string | undefined, +): StableCaptureSignal { + return stableCaptureSignal({ + ...snapshot, + nodes: withoutSettleChrome(snapshot.nodes, appBundleId), + }); } function isBoundIosXCTestCapture( @@ -216,16 +238,23 @@ function isBoundIosXCTestCapture( return backend === undefined && platform === 'ios'; } -function stableCaptureTransitionQuietMs(params: { +/** + * The quiet window the candidate capture demands: the broad-replacement verdict runs on the + * chrome-free candidate signal (see `stableCaptureReplacementSignal`), so a keyboard pop never + * reaches the confirmation branch at all. + */ +function stableCaptureNextQuietWindowMs(params: { baseline: StableCaptureSignal | undefined; - signal: StableCaptureSignal; + snapshot: Parameters[0]; + appBundleId: string | undefined; requestedQuietMs: number; currentQuietMs: number; }): number { - if ( - params.requestedQuietMs < DEFAULT_STABLE_QUIET_MS || - !stableCaptureSignalsHaveBroadReplacement(params.baseline, params.signal) - ) { + if (params.requestedQuietMs < DEFAULT_STABLE_QUIET_MS || !params.baseline) { + return params.currentQuietMs; + } + const replacementSignal = stableCaptureReplacementSignal(params.snapshot, params.appBundleId); + if (!stableCaptureSignalsHaveBroadReplacement(params.baseline, replacementSignal)) { return params.currentQuietMs; } return Math.max(params.requestedQuietMs, BROAD_TRANSITION_CONFIRMATION_QUIET_MS); From e2ad8997ed7f52e8bc51be8634911568acfdd592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 12:27:57 +0200 Subject: [PATCH 2/5] chore(gates): publish capture-kit chrome fixtures subpath and pin its format style The shared keyboard capture fixture joins the capture-kit exports map, so the layering exports manifest gains its subpath, and oxfmt ignores the fixture file the way it already ignores runner-requests.json: one captured node per reviewable diff line. --- .oxfmtrc.json | 4 +++- packages/capture-kit/package.json | 4 ++++ scripts/layering/package-boundaries.test.ts | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 00e66b8708..975e0efe0c 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -12,6 +12,8 @@ "scripts/maestro-conformance/corpus/**", "fallow-baselines/**", // One entry per line, so each pinned runner request changes as one reviewable diff line. - "contracts/fixtures/runner-requests.json" + "contracts/fixtures/runner-requests.json", + // One captured node per line, so the real keyboard capture stays a compact reviewable table. + "packages/capture-kit/src/snapshot-chrome.fixtures.ts" ] } diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 05e8112ceb..8b24bef561 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -230,6 +230,10 @@ "types": "./src/snapshot-chrome.ts", "default": "./src/snapshot-chrome.ts" }, + "./snapshot-chrome-fixtures": { + "types": "./src/snapshot-chrome.fixtures.ts", + "default": "./src/snapshot-chrome.fixtures.ts" + }, "./snapshot-desktop-projection": { "types": "./src/snapshot-desktop-projection.ts", "default": "./src/snapshot-desktop-projection.ts" diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index ec654b60f3..b732afa650 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -492,6 +492,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/screenshot-overlay', '@agent-device/capture-kit/scroll-edge-state', '@agent-device/capture-kit/snapshot-chrome', + '@agent-device/capture-kit/snapshot-chrome-fixtures', '@agent-device/capture-kit/snapshot-desktop-projection', '@agent-device/capture-kit/snapshot-desktop-surface', '@agent-device/capture-kit/snapshot-diff', From ab1f758ccd9e4e09320aab08968911b45d9e1811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 13:00:06 +0200 Subject: [PATCH 3/5] fix(settle): let the app-accessory exemption outrank the owned-role chrome walk An app's `UITextField.inputAssistantItem` bar buttons render under the system assistant view, above the keys; the window pass exempted them and the owned-role subtree walk re-claimed them, hiding real app controls from the settle diff. An exempted node now re-joins chrome only when it declares a keyboard-owned role itself, so a system prediction cell still collapses while an app button under the same host survives. Also from review: hoist the chrome-free replacement projection behind a baseline/candidate-change guard so the 300ms poll of an unchanged screen pays no second tree pass, and count node 31 in the fixture breakdown. --- .../src/snapshot-chrome.fixtures.ts | 1 + .../capture-kit/src/snapshot-chrome.test.ts | 79 ++++++++++++++++ packages/capture-kit/src/snapshot-chrome.ts | 89 +++++++++++++------ .../interaction/runtime/stable-capture.ts | 37 ++++++-- 4 files changed, 173 insertions(+), 33 deletions(-) diff --git a/packages/capture-kit/src/snapshot-chrome.fixtures.ts b/packages/capture-kit/src/snapshot-chrome.fixtures.ts index 51c88dce7b..913586019f 100644 --- a/packages/capture-kit/src/snapshot-chrome.fixtures.ts +++ b/packages/capture-kit/src/snapshot-chrome.fixtures.ts @@ -10,6 +10,7 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; * enabled) is verbatim. * * The keyboard surfaces arrive flattened beside app content: app nodes 0-30, + * the floating home-indicator toolbar the window capture also reports (31), * the system `inputView` host frame (32), the QuickType assistant bar (33-39), * the `[Keyboard]` plane (40) with its layout wrapper (41) and keys (42-79), * and the dock's "Next keyboard" / "Dictate" buttons (80-82). This is exactly diff --git a/packages/capture-kit/src/snapshot-chrome.test.ts b/packages/capture-kit/src/snapshot-chrome.test.ts index 5e4b3b6481..6f559ef2bc 100644 --- a/packages/capture-kit/src/snapshot-chrome.test.ts +++ b/packages/capture-kit/src/snapshot-chrome.test.ts @@ -325,3 +325,82 @@ test('editable text hosted inside a keyboard-owned subtree survives classificati assert.equal(chromeRefs.has(refForIdentifier(nodes, 'SystemInputAssistantView')), true); assert.equal(chromeRefs.has(refFor(nodes, 'Compose')), false); }); + +// The window pass exempts app content above the keys, and the keyboard-owned subtree walk must +// not outrank that exemption: an app's UITextField.inputAssistantItem bar button renders UNDER +// the system assistant view and declares no keyboard-owned role, so it stays an app control — +// while a system prediction cell beside it, which owns its chrome by role wherever it sits, +// still collapses. +test('app inputAssistantItem buttons under the assistant view keep the accessory exemption', () => { + const nodes = attachRefs([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Window', + label: 'Keyboard Window', + rect: { x: 0, y: 300, width: 402, height: 574 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Other', + role: 'TUISystemInputAssistantView', + label: 'Assistant Bar', + rect: { x: 0, y: 300, width: 402, height: 44 }, + }, + { + index: 3, + depth: 3, + parentIndex: 2, + type: 'Button', + label: 'Paste', + rect: { x: 8, y: 306, width: 60, height: 32 }, + hittable: true, + }, + { + index: 4, + depth: 3, + parentIndex: 2, + type: 'Button', + role: 'TUIPredictionViewCell', + label: 'keep', + rect: { x: 200, y: 306, width: 60, height: 32 }, + hittable: true, + }, + { + index: 5, + depth: 2, + parentIndex: 1, + type: 'Keyboard', + label: 'Keyboard', + rect: { x: 0, y: 360, width: 402, height: 514 }, + }, + { + index: 6, + depth: 3, + parentIndex: 5, + type: 'Key', + label: 'q', + rect: { x: 0, y: 380, width: 39, height: 54 }, + }, + ]); + const chromeRefs = collectSettleChromeRefs(nodes, undefined); + + assert.equal( + chromeRefs.has(refFor(nodes, 'Paste')), + false, + 'the app-provided assistant-item button is exempt, not re-claimed through the assistant subtree', + ); + assert.equal(chromeRefs.has(refFor(nodes, 'keep')), true); + assert.equal(chromeRefs.has(refFor(nodes, 'Assistant Bar')), true); + assert.equal(chromeRefs.has(refFor(nodes, 'q')), true); +}); diff --git a/packages/capture-kit/src/snapshot-chrome.ts b/packages/capture-kit/src/snapshot-chrome.ts index 96f55a63ff..047c596fcd 100644 --- a/packages/capture-kit/src/snapshot-chrome.ts +++ b/packages/capture-kit/src/snapshot-chrome.ts @@ -143,24 +143,31 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { } // Preserve app-owned accessory controls hosted above the keyboard: they were // swept in by the whole-window pass but are not keyboard chrome. - for (const index of collectKeyboardAccessoryIndexes( + const accessoryExemptIndexes = collectKeyboardAccessoryIndexes( nodes, byIndex, windowIndexes, containerIndexes, containerSubtreeIndexes, - )) { + ); + for (const index of accessoryExemptIndexes) { chromeIndexes.delete(index); } // Keyboard-owned classes the projection flattened out of every reachable subtree are chrome - // by their own declaration, so the app-accessory guard above never exempts them. An editable - // text node inside one of them is the same inputAccessoryView guard in disguise: an app that - // hosts a text field in its own input view keeps it, as it does under the window pass. + // by their own declaration — but the walk may NOT outrank the app-accessory exemption above: + // an app's `UITextField.inputAssistantItem` bar buttons render UNDER the system assistant + // view, above the keys, and the window pass just exempted them; re-claiming them through the + // assistant's subtree would hide real app controls from the settle diff. An exempted node + // only re-joins chrome when IT declares a keyboard-owned role (a prediction cell is + // system-owned wherever it sits); an app node under an owned host stays exempt. In the + // flattened interactive shape there is no window anchor, the exemption is empty, and the + // walk claims the key plane exactly as the container-descendant walk always did. for (const index of collectOwnedKeyboardChromeIndexes( nodes, byIndex, ownedSubtreeRootIndexes, containerSubtreeIndexes, + accessoryExemptIndexes, )) { chromeIndexes.add(index); } @@ -179,39 +186,69 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { } /** - * The subtree under each keyboard-owned-class root, minus editable text the app hosts inside it - * outside the keyboard container. A keyboard's own `Key` nodes never trip that guard — the walk - * claims them with their own subtree roots even when a stripped field re-parented them — and an - * app that hosts its whole composer inside its input view still loses every key, exactly as the - * container-descendant walk always did. + * The subtree under each keyboard-owned-class root, minus two app-content guards: indexes the + * app-accessory exemption already granted (an app's `inputAssistantItem` bar buttons ride the + * system assistant view above the keys), and editable text the app hosts inside an owned root + * outside the keyboard container, with its subtree. A keyboard's own `Key` nodes trip neither — + * they sit inside the container subtree, which the accessory pass never exempts — and the + * flattened interactive shape carries no window anchor, so the exemption is empty there and the + * walk claims the whole key plane exactly as the container-descendant walk always did. */ function collectOwnedKeyboardChromeIndexes( nodes: SnapshotNode[], byIndex: Map, ownedSubtreeRootIndexes: ReadonlySet, containerSubtreeIndexes: ReadonlySet, + accessoryExemptIndexes: ReadonlySet, ): Set { const chromeIndexes = collectSubtreeIndexes(nodes, byIndex, ownedSubtreeRootIndexes); - for (const node of nodes) { - if ( - chromeIndexes.has(node.index) && - EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) && - !containerSubtreeIndexes.has(node.index) - ) { - chromeIndexes.delete(node.index); - for (const descendant of nodes) { - if ( - chromeIndexes.has(descendant.index) && - hasAncestorIn(descendant, byIndex, new Set([node.index])) - ) { - chromeIndexes.delete(descendant.index); - } - } - } + // An exempted node re-joins chrome only when IT declares a keyboard-owned role: a system + // prediction cell is chrome wherever it sits, while an app's `inputAssistantItem` bar button + // under the assistant view declares no owned role and keeps its exemption. + for (const index of accessoryExemptIndexes) { + if (!ownedSubtreeRootIndexes.has(index)) chromeIndexes.delete(index); + } + for (const index of collectHostedEditableTextIndexes( + nodes, + byIndex, + chromeIndexes, + containerSubtreeIndexes, + )) { + chromeIndexes.delete(index); } return chromeIndexes; } +/** + * Editable text the app hosts under an owned root outside the keyboard container, and the + * chrome claimed beneath it — the inputAccessoryView guard in disguise: an app that rides its + * own composer on the assistant host keeps the field it is typing into. + */ +function collectHostedEditableTextIndexes( + nodes: SnapshotNode[], + byIndex: Map, + chromeIndexes: ReadonlySet, + containerSubtreeIndexes: ReadonlySet, +): Set { + const dropped = new Set(); + const hostedTextIndexes = new Set( + nodes + .filter( + (node) => + chromeIndexes.has(node.index) && + EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) && + !containerSubtreeIndexes.has(node.index), + ) + .map((node) => node.index), + ); + for (const node of nodes) { + if (!chromeIndexes.has(node.index)) continue; + if (hasAncestorIn(node, byIndex, hostedTextIndexes)) dropped.add(node.index); + } + for (const index of hostedTextIndexes) dropped.add(index); + return dropped; +} + /** * Indexes of app-owned accessory content in a keyboard window: non-keyboard * nodes whose center sits ABOVE the keyboard container's top edge (an diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index 85ad0c6cda..ef6645f46f 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -92,6 +92,35 @@ export async function runStableCaptureLoop( let lastCapture: CapturedSnapshot | undefined; let quietSinceMs = start; let requiredQuietMs = quietMs; + let lastQuietWindowBaseline: StableCaptureSignal | undefined; + let lastQuietWindowSignal: StableCaptureSignal | undefined; + // The broad-replacement verdict is a pure function of the armed baseline, the requested + // window, and this candidate signal, and the window it sets only latches upward — so the + // verdict cannot change while both inputs hold. Rebuilding the chrome-free projection (a + // full withoutSettleChrome pass plus a second signal) is owed on baseline arming (it can + // land after the first capture) and on candidate change, not on every 300ms poll of a + // quiet screen. + const nextQuietWindow = ( + snapshot: Parameters[0], + baseline: StableCaptureSignal | undefined, + signal: StableCaptureSignal, + ): number => { + if ( + baseline === lastQuietWindowBaseline && + stableCaptureSignalsEqual(lastQuietWindowSignal, signal) + ) { + return requiredQuietMs; + } + lastQuietWindowBaseline = baseline; + lastQuietWindowSignal = signal; + return stableCaptureNextQuietWindowMs({ + baseline, + snapshot, + appBundleId: session?.appBundleId, + requestedQuietMs: quietMs, + currentQuietMs: requiredQuietMs, + }); + }; while (now(runtime) < deadlineMs) { const capture = await captureStableSignalWithinDeadline( runtime, @@ -118,13 +147,7 @@ export async function runStableCaptureLoop( session?.appBundleId, ); const signal = stableCaptureSignal(capture.snapshot); - requiredQuietMs = stableCaptureNextQuietWindowMs({ - baseline: transitionBaseline, - snapshot: capture.snapshot, - appBundleId: session?.appBundleId, - requestedQuietMs: quietMs, - currentQuietMs: requiredQuietMs, - }); + requiredQuietMs = nextQuietWindow(capture.snapshot, transitionBaseline, signal); preferredBackend ??= preferredSnapshotBackendForVerdict(capture.snapshot.snapshotQuality); const nowMs = now(runtime); const recoveredDeadlineMs = extendedDeadlineAfterPrivateAxRecovery({ From 41c8a82775faff0defe59e83a4860486bb59f611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 13:27:21 +0200 Subject: [PATCH 4/5] fix(settle): drop the unsound memo key on the quiet-window verdict Keying the memo on stableCaptureSignalsEqual is unsound: that identity omits role, parent links, and sub-pixel geometry, all of which withoutSettleChrome reads, so a newly broad replacement could reuse a non-broad verdict and skip the confirmation window. Keying it on every classifier input would cost more than running the classifier, and the measurement says the memo was never owed: one full chrome-free pass is ~42us on the 83-node real keyboard capture and ~1.4ms on a 5k-node tree, against a 300ms poll cadence. The verdict is recomputed per capture again, and the doc-comment forbids re-memoizing on that key. --- .../interaction/runtime/stable-capture.ts | 41 +++++-------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index ef6645f46f..0451e3a4fc 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -92,35 +92,6 @@ export async function runStableCaptureLoop( let lastCapture: CapturedSnapshot | undefined; let quietSinceMs = start; let requiredQuietMs = quietMs; - let lastQuietWindowBaseline: StableCaptureSignal | undefined; - let lastQuietWindowSignal: StableCaptureSignal | undefined; - // The broad-replacement verdict is a pure function of the armed baseline, the requested - // window, and this candidate signal, and the window it sets only latches upward — so the - // verdict cannot change while both inputs hold. Rebuilding the chrome-free projection (a - // full withoutSettleChrome pass plus a second signal) is owed on baseline arming (it can - // land after the first capture) and on candidate change, not on every 300ms poll of a - // quiet screen. - const nextQuietWindow = ( - snapshot: Parameters[0], - baseline: StableCaptureSignal | undefined, - signal: StableCaptureSignal, - ): number => { - if ( - baseline === lastQuietWindowBaseline && - stableCaptureSignalsEqual(lastQuietWindowSignal, signal) - ) { - return requiredQuietMs; - } - lastQuietWindowBaseline = baseline; - lastQuietWindowSignal = signal; - return stableCaptureNextQuietWindowMs({ - baseline, - snapshot, - appBundleId: session?.appBundleId, - requestedQuietMs: quietMs, - currentQuietMs: requiredQuietMs, - }); - }; while (now(runtime) < deadlineMs) { const capture = await captureStableSignalWithinDeadline( runtime, @@ -147,7 +118,13 @@ export async function runStableCaptureLoop( session?.appBundleId, ); const signal = stableCaptureSignal(capture.snapshot); - requiredQuietMs = nextQuietWindow(capture.snapshot, transitionBaseline, signal); + requiredQuietMs = stableCaptureNextQuietWindowMs({ + baseline: transitionBaseline, + snapshot: capture.snapshot, + appBundleId: session?.appBundleId, + requestedQuietMs: quietMs, + currentQuietMs: requiredQuietMs, + }); preferredBackend ??= preferredSnapshotBackendForVerdict(capture.snapshot.snapshotQuality); const nowMs = now(runtime); const recoveredDeadlineMs = extendedDeadlineAfterPrivateAxRecovery({ @@ -264,7 +241,9 @@ function isBoundIosXCTestCapture( /** * The quiet window the candidate capture demands: the broad-replacement verdict runs on the * chrome-free candidate signal (see `stableCaptureReplacementSignal`), so a keyboard pop never - * reaches the confirmation branch at all. + * reaches the confirmation branch at all. Do not memoize this on `stableCaptureSignalsEqual`: + * the signal identity omits `role`, parent links, and >1px-free geometry, all of which the + * classifier reads, and one pass costs tens of microseconds against the 300 ms poll cadence. */ function stableCaptureNextQuietWindowMs(params: { baseline: StableCaptureSignal | undefined; From faf754aa622b6bd328d07d5ccd1238128d0cbf65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 12 Oct 2026 18:08:12 +0200 Subject: [PATCH 5/5] refactor(settle): one contracts iOS keyboard-role table; assistant hosts are markers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assistant view and its page view are keyboard hosts the APP can also fill (UITextField.inputAssistantItem bar buttons ride them), so they join the inputView placeholder as markers: chrome themselves, claiming no subtree. Subtree claims stay with the system-only hosts — the key plane (UIKeyboardLayoutStar) and the prediction bar — where no app content can ever sit. Surface roles are chrome per node. With no subtree walk able to reach app content, the accessory re-join guard and the hosted editable-text carve-out are dead weight and are removed. The IOS_KEYBOARD_* tables live in android-input-ownership — the module the settle classifier and the tap-occlusion band already evaluate, as the eager-closure gate (ADR 0027) requires for shared symbols; a dedicated module would grow every shared entry's eager closure. The prediction cell stays out of the surface table so QuickType geometry never enters the occlusion band. Accepts two inert noise survivors in the flattened capture (floating toolbar, QuickType scroll wrapper) — noise fails loudly, a subtree claim through an app-fillable host would not. Also reverts the broad-replacement right parameter to non-optional: the memo is gone and the only caller always passes a defined signal. --- .../capture-kit/src/snapshot-chrome.test.ts | 148 ++++++++++++++- packages/capture-kit/src/snapshot-chrome.ts | 169 ++++-------------- .../contracts/src/android-input-ownership.ts | 64 +++++++ .../contracts/src/tap-keyboard-occlusion.ts | 8 +- .../runtime/settle-keyboard-chrome.test.ts | 9 +- .../runtime/stable-capture-signal.ts | 4 +- 6 files changed, 254 insertions(+), 148 deletions(-) diff --git a/packages/capture-kit/src/snapshot-chrome.test.ts b/packages/capture-kit/src/snapshot-chrome.test.ts index 6f559ef2bc..486efd6d7e 100644 --- a/packages/capture-kit/src/snapshot-chrome.test.ts +++ b/packages/capture-kit/src/snapshot-chrome.test.ts @@ -234,11 +234,15 @@ test('the real interactive keyboard capture collapses to one [Keyboard] diff lin } // The stripped set collapses the ~50 keyboard-surface nodes; what survives is - // app content plus the lone keyboard plane line. + // app content plus the keyboard plane line and two accepted residues: the + // floating home-indicator Toolbar (31) and the inert QuickType UIScrollView + // (35) under the assistant markers, which declare no keyboard-owned role and + // are not under a system-only subtree host. Noise lines fail loudly; a + // subtree claim through the assistant host could swallow app controls. assert.equal(survivors.filter((node) => node.type === 'Key').length, 0); assert.deepEqual( survivors.filter((node) => node.index >= 31).map((node) => node.type), - ['Toolbar', 'Keyboard'], + ['Toolbar', 'ScrollView', 'Keyboard'], ); // App content never rides along: the search field the fill targeted survives. assert.equal(chromeRefs.has(refFor(nodes, 'Search products')), false); @@ -289,9 +293,10 @@ test('the inputView host frame is stripped without swallowing app input-view con ); }); -// An app that hosts editable text inside a keyboard-owned subtree (its own -// composer riding the input-assistant host) keeps the field it is typing into. -test('editable text hosted inside a keyboard-owned subtree survives classification', () => { +// An app that hosts editable text under a keyboard MARKER host (its own composer +// riding the input-assistant host) keeps the field it is typing into: the marker +// is chrome itself and owns no subtree. +test('editable text hosted under a keyboard marker host survives classification', () => { const nodes = attachRefs([ { index: 0, @@ -404,3 +409,136 @@ test('app inputAssistantItem buttons under the assistant view keep the accessory assert.equal(chromeRefs.has(refFor(nodes, 'Assistant Bar')), true); assert.equal(chromeRefs.has(refFor(nodes, 'q')), true); }); + +// The route #3469 targets: the interactive projection has NO Window anchor, so the +// app-accessory exemption is empty and only the marker rule protects app content. An +// app Button under the assistant view survives on its own; a prediction cell beside +// it, chrome by its own declared role, still collapses. +test('an app control under the assistant host survives on the Window-less interactive shape', () => { + const nodes = attachRefs([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'TextField', + label: 'Search', + rect: { x: 18, y: 100, width: 366, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Other', + role: 'TUISystemInputAssistantView', + rect: { x: 0, y: 546, width: 402, height: 44 }, + }, + { + index: 3, + depth: 2, + parentIndex: 2, + type: 'Button', + label: 'Paste', + rect: { x: 8, y: 550, width: 60, height: 36 }, + hittable: true, + }, + { + index: 4, + depth: 2, + parentIndex: 2, + type: 'Button', + role: 'TUIPredictionViewCell', + label: 'keep', + rect: { x: 200, y: 550, width: 60, height: 36 }, + hittable: true, + }, + { + index: 5, + depth: 1, + parentIndex: 0, + type: 'Keyboard', + role: 'UIKeyboardAutomatic', + rect: { x: 0, y: 606, width: 402, height: 268 }, + }, + { + index: 6, + depth: 2, + parentIndex: 5, + type: 'Key', + role: 'UIAccessibilityElementKBKey', + label: 'q', + rect: { x: 0, y: 620, width: 39, height: 54 }, + }, + ]); + const chromeRefs = collectSettleChromeRefs(nodes, undefined); + + assert.equal( + chromeRefs.has(refFor(nodes, 'Paste')), + false, + 'the assistant host is a marker: it claims no subtree, so an app control under it survives', + ); + assert.equal(chromeRefs.has(refFor(nodes, 'keep')), true); + assert.equal(chromeRefs.has(refFor(nodes, 'Search')), false); +}); + +// A capture that reports the key plane or the prediction bar WITHOUT any key/cell/marker +// sibling: the subtree-role hosts alone must still strip their subtrees, or the empty-set +// early return re-leaks the plane's inert children into the settle diff and tail budget. +test('a subtree-role host with no surface or marker sibling still strips its subtree', () => { + const nodes = attachRefs([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Other', + role: 'UIKeyboardLayoutStar', + rect: { x: 0, y: 606, width: 402, height: 268 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Other', + label: 'autoguess candidate row', + rect: { x: 0, y: 606, width: 402, height: 44 }, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'Other', + role: 'TUIPredictionView', + label: 'Typing Predictions', + rect: { x: 0, y: 546, width: 402, height: 44 }, + }, + { + index: 4, + depth: 2, + parentIndex: 3, + type: 'Other', + rect: { x: 0, y: 546, width: 402, height: 44 }, + }, + ]); + const survivors = withoutSettleChrome(nodes, undefined); + const chromeRefs = collectSettleChromeRefs(nodes, undefined); + + assert.deepEqual( + survivors.map((node) => node.index), + [0], + ); + assert.equal(chromeRefs.size, 4); +}); diff --git a/packages/capture-kit/src/snapshot-chrome.ts b/packages/capture-kit/src/snapshot-chrome.ts index 047c596fcd..34ce0da898 100644 --- a/packages/capture-kit/src/snapshot-chrome.ts +++ b/packages/capture-kit/src/snapshot-chrome.ts @@ -1,9 +1,16 @@ -import { isAndroidInputMethodNode } from '@agent-device/contracts/android-input-ownership'; import { ANDROID_SYSTEM_CHROME_PACKAGE, hasAndroidSystemChromeProvenance, isAndroidSystemChromeWindowResourceId, } from '@agent-device/contracts/android-system-chrome'; +import { + hasIosKeyboardRole, + IOS_KEYBOARD_CHROME_MARKER_ROLES, + IOS_KEYBOARD_SETTLE_CHROME_ROLES, + IOS_KEYBOARD_SUBTREE_CHROME_ROLES, + IOS_KEYBOARD_SURFACE_ROLES, + isAndroidInputMethodNode, +} from '@agent-device/contracts/android-input-ownership'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { normalizeType } from '@agent-device/contracts/snapshot'; @@ -74,64 +81,39 @@ type SettleChrome = { const EMPTY_SETTLE_CHROME: SettleChrome = { strippedIndexes: new Set(), refs: new Set() }; -/** - * UIKit classes the keyboard owns whose nodes the regular projection can flatten beside app - * content — the key plane and a key when no `[Keyboard]` container survives, the system - * input-assistant (QuickType) bar, and the dock's "Next keyboard" / "Dictate" buttons. Once the - * projection has dropped the keyboard's own window, neither the container-subtree nor the - * window walk can reach them, but each node still carries its producing class in `role`. - * - * Exact class names only, so a near miss fails closed rather than widening what counts as - * chrome (the same discipline `geometry-policy.ts` applies to private UIKit roles). Classes are - * locale-independent, unlike every label these nodes carry, and private to the keyboard - * surfaces — an app cannot host one in its own view tree. - */ -const IOS_KEYBOARD_OWNED_ROLES: ReadonlySet = new Set([ - 'UIKeyboardLayoutStar', - 'UIAccessibilityElementKBKey', - 'UIKeyboardDockItemButton', - 'TUISystemInputAssistantView', - 'TUISystemInputAssistantPageView', - 'TUIPredictionView', - 'TUIPredictionViewCell', -]); - -/** - * The system `inputView` host frame: chrome itself, but it OWNS NO SUBTREE — an app's custom - * `UITextField.inputView` content (a drawn keypad, a composer) renders under this placeholder and - * stays a control the agent must see. Stripping only the marker keeps the keyboard's appearance - * at one visible line without swallowing app content; if the class is ever renamed, the inert - * marker line returns as noise rather than app content vanishing — noise fails loudly. - */ -const IOS_KEYBOARD_CHROME_MARKER_ROLES: ReadonlySet = new Set([ - '_UIRemoteKeyboardPlaceholderView', -]); - -function hasRole(node: SnapshotNode, roles: ReadonlySet): boolean { - return node.role !== undefined && roles.has(node.role); -} - -function isKeyboardOwnedRoleNode(node: SnapshotNode): boolean { - return hasRole(node, IOS_KEYBOARD_OWNED_ROLES); -} - function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { const byIndex = new Map(nodes.map((node) => [node.index, node])); const containerIndexes = new Set( nodes.filter((node) => normalizeType(node.type ?? '') === 'keyboard').map((node) => node.index), ); - const ownedSubtreeRootIndexes = new Set( - nodes.filter(isKeyboardOwnedRoleNode).map((node) => node.index), + // Role-declared keyboard surfaces, reached because the interactive projection flattens them + // beside app content: chrome PER NODE (surface roles), chrome by SUBTREE (system-only hosts), + // or chrome AS A MARKER with no subtree (hosts the app can also fill — its custom inputView, + // its inputAssistantItem bar buttons — so the app content under a marker survives). + const surfaceIndexes = new Set( + nodes + .filter( + (node) => + hasIosKeyboardRole(node, IOS_KEYBOARD_SURFACE_ROLES) || + hasIosKeyboardRole(node, IOS_KEYBOARD_SETTLE_CHROME_ROLES), + ) + .map((node) => node.index), ); const markerIndexes = new Set( nodes - .filter((node) => hasRole(node, IOS_KEYBOARD_CHROME_MARKER_ROLES)) + .filter((node) => hasIosKeyboardRole(node, IOS_KEYBOARD_CHROME_MARKER_ROLES)) + .map((node) => node.index), + ); + const subtreeChromeRootIndexes = new Set( + nodes + .filter((node) => hasIosKeyboardRole(node, IOS_KEYBOARD_SUBTREE_CHROME_ROLES)) .map((node) => node.index), ); if ( containerIndexes.size === 0 && - ownedSubtreeRootIndexes.size === 0 && - markerIndexes.size === 0 + surfaceIndexes.size === 0 && + markerIndexes.size === 0 && + subtreeChromeRootIndexes.size === 0 ) { return EMPTY_SETTLE_CHROME; } @@ -143,36 +125,23 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { } // Preserve app-owned accessory controls hosted above the keyboard: they were // swept in by the whole-window pass but are not keyboard chrome. - const accessoryExemptIndexes = collectKeyboardAccessoryIndexes( + for (const index of collectKeyboardAccessoryIndexes( nodes, byIndex, windowIndexes, containerIndexes, containerSubtreeIndexes, - ); - for (const index of accessoryExemptIndexes) { + )) { chromeIndexes.delete(index); } - // Keyboard-owned classes the projection flattened out of every reachable subtree are chrome - // by their own declaration — but the walk may NOT outrank the app-accessory exemption above: - // an app's `UITextField.inputAssistantItem` bar buttons render UNDER the system assistant - // view, above the keys, and the window pass just exempted them; re-claiming them through the - // assistant's subtree would hide real app controls from the settle diff. An exempted node - // only re-joins chrome when IT declares a keyboard-owned role (a prediction cell is - // system-owned wherever it sits); an app node under an owned host stays exempt. In the - // flattened interactive shape there is no window anchor, the exemption is empty, and the - // walk claims the key plane exactly as the container-descendant walk always did. - for (const index of collectOwnedKeyboardChromeIndexes( - nodes, - byIndex, - ownedSubtreeRootIndexes, - containerSubtreeIndexes, - accessoryExemptIndexes, - )) { + // No role claim may reach app content: surface roles are chrome per node, subtree claims only + // start at system-only hosts the app can never fill, and markers own no subtree. That is why + // these passes need no interaction with the accessory exemption or an editable-text guard — + // an app node under the assistant view or the inputView placeholder is simply never claimed. + for (const index of surfaceIndexes) chromeIndexes.add(index); + for (const index of collectSubtreeIndexes(nodes, byIndex, subtreeChromeRootIndexes)) { chromeIndexes.add(index); } - // The system inputView host frame is itself chrome, but owning no subtree: an app's custom - // input view content under it survives classification. for (const index of markerIndexes) chromeIndexes.add(index); const refs = new Set( nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref), @@ -185,70 +154,6 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { return { strippedIndexes, refs }; } -/** - * The subtree under each keyboard-owned-class root, minus two app-content guards: indexes the - * app-accessory exemption already granted (an app's `inputAssistantItem` bar buttons ride the - * system assistant view above the keys), and editable text the app hosts inside an owned root - * outside the keyboard container, with its subtree. A keyboard's own `Key` nodes trip neither — - * they sit inside the container subtree, which the accessory pass never exempts — and the - * flattened interactive shape carries no window anchor, so the exemption is empty there and the - * walk claims the whole key plane exactly as the container-descendant walk always did. - */ -function collectOwnedKeyboardChromeIndexes( - nodes: SnapshotNode[], - byIndex: Map, - ownedSubtreeRootIndexes: ReadonlySet, - containerSubtreeIndexes: ReadonlySet, - accessoryExemptIndexes: ReadonlySet, -): Set { - const chromeIndexes = collectSubtreeIndexes(nodes, byIndex, ownedSubtreeRootIndexes); - // An exempted node re-joins chrome only when IT declares a keyboard-owned role: a system - // prediction cell is chrome wherever it sits, while an app's `inputAssistantItem` bar button - // under the assistant view declares no owned role and keeps its exemption. - for (const index of accessoryExemptIndexes) { - if (!ownedSubtreeRootIndexes.has(index)) chromeIndexes.delete(index); - } - for (const index of collectHostedEditableTextIndexes( - nodes, - byIndex, - chromeIndexes, - containerSubtreeIndexes, - )) { - chromeIndexes.delete(index); - } - return chromeIndexes; -} - -/** - * Editable text the app hosts under an owned root outside the keyboard container, and the - * chrome claimed beneath it — the inputAccessoryView guard in disguise: an app that rides its - * own composer on the assistant host keeps the field it is typing into. - */ -function collectHostedEditableTextIndexes( - nodes: SnapshotNode[], - byIndex: Map, - chromeIndexes: ReadonlySet, - containerSubtreeIndexes: ReadonlySet, -): Set { - const dropped = new Set(); - const hostedTextIndexes = new Set( - nodes - .filter( - (node) => - chromeIndexes.has(node.index) && - EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) && - !containerSubtreeIndexes.has(node.index), - ) - .map((node) => node.index), - ); - for (const node of nodes) { - if (!chromeIndexes.has(node.index)) continue; - if (hasAncestorIn(node, byIndex, hostedTextIndexes)) dropped.add(node.index); - } - for (const index of hostedTextIndexes) dropped.add(index); - return dropped; -} - /** * Indexes of app-owned accessory content in a keyboard window: non-keyboard * nodes whose center sits ABOVE the keyboard container's top edge (an diff --git a/packages/contracts/src/android-input-ownership.ts b/packages/contracts/src/android-input-ownership.ts index 55c299fde4..1e916da134 100644 --- a/packages/contracts/src/android-input-ownership.ts +++ b/packages/contracts/src/android-input-ownership.ts @@ -1,3 +1,12 @@ +// Input-surface ownership for both platforms: Android IME package provenance below, and the +// iOS keyboard role table (`IOS_KEYBOARD_*`) at the bottom. The iOS table lives HERE rather +// than in an ios-keyboard-ownership sibling because the eager-import closure gate +// (ADR-0027, eager-closure-budgets) requires shared symbols to sit in a module every entry +// that needs them already evaluates — a new static edge to a new module grows the daemon, +// replay-port, and capture-kit closures red. This module is that shared home: snapshot-chrome +// and tap-keyboard-occlusion already evaluate it, and renaming the published subpath would +// itself move every consumer's closure. + import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; export type AndroidInputOwner = 'app' | 'ime' | 'unknown'; @@ -135,3 +144,58 @@ function normalizeAndroidPackageName(value: string | null | undefined): string | const normalized = (value ?? '').trim().toLowerCase(); return normalized || undefined; } + +/** + * iOS keyboard ownership — the `IOS_KEYBOARD_*` twin of the Android provenance above, shared + * by the tap occlusion band (`tap-keyboard-occlusion.ts`) and the settle chrome classifier + * (`@agent-device/capture-kit/snapshot-chrome`). + * + * The interactive projection flattens the keyboard surfaces beside app content, so ancestry + * cannot reach them; each node still carries its producing UIKit class in `role`. The table is + * exact class names only, so a near miss fails closed rather than widening what counts as + * keyboard-owned (the same discipline the snapshot geometry policy applies to private UIKit + * roles). Classes are locale-independent, unlike every label these nodes carry, and private to + * the keyboard surfaces — an app cannot host one in its own view tree. + * + * Four ownership shapes, because "produced by the keyboard" is not one question: + * - {@link IOS_KEYBOARD_SURFACE_ROLES}: nodes the keyboard owns PER NODE, safe for geometry + * (occlusion band). + * - {@link IOS_KEYBOARD_SETTLE_CHROME_ROLES}: per-node chrome for the settle classifier only — + * the QuickType prediction cell. Kept out of the surface table so a QuickType candidate never + * joins the occlusion band's geometry. + * - {@link IOS_KEYBOARD_SUBTREE_CHROME_ROLES}: system-only hosts whose WHOLE subtree is + * keyboard chrome — the key plane and the prediction bar. An app can never fill these, so a + * subtree claim here cannot swallow app content. + * - {@link IOS_KEYBOARD_CHROME_MARKER_ROLES}: hosts the app CAN also fill (its custom + * `inputView`, its `inputAssistantItem` bar buttons ride the assistant view). The marker is + * chrome itself and owns NO subtree: app content under it survives classification. + */ + +/** Keyboard-owned per node; contributes geometry wherever the projection places it. */ +export const IOS_KEYBOARD_SURFACE_ROLES: ReadonlySet = new Set([ + 'UIKeyboardDockItemButton', + 'UIAccessibilityElementKBKey', +]); + +/** Per-node settle chrome beyond the surface table; never occlusion geometry. */ +export const IOS_KEYBOARD_SETTLE_CHROME_ROLES: ReadonlySet = new Set([ + 'TUIPredictionViewCell', +]); + +/** System-only keyboard hosts; their entire subtree is keyboard chrome. */ +export const IOS_KEYBOARD_SUBTREE_CHROME_ROLES: ReadonlySet = new Set([ + 'UIKeyboardLayoutStar', + 'TUIPredictionView', +]); + +/** Keyboard hosts the app can also fill: chrome itself, subtree left to the app. */ +export const IOS_KEYBOARD_CHROME_MARKER_ROLES: ReadonlySet = new Set([ + '_UIRemoteKeyboardPlaceholderView', + 'TUISystemInputAssistantView', + 'TUISystemInputAssistantPageView', +]); + +/** True when this node's own UIKit class is one of `roles`. */ +export function hasIosKeyboardRole(node: RawSnapshotNode, roles: ReadonlySet): boolean { + return node.role !== undefined && roles.has(node.role); +} diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 6ea6504eca..92a1731800 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -5,7 +5,7 @@ import type { SnapshotKeyboardBandFact, } from '@agent-device/kernel/snapshot'; import { containsPoint, isPositiveFiniteRect } from '@agent-device/kernel/rect'; -import { isAndroidInputMethodNode } from './android-input-ownership.ts'; +import { IOS_KEYBOARD_SURFACE_ROLES, isAndroidInputMethodNode } from './android-input-ownership.ts'; import { normalizeType } from './snapshot-text.ts'; /** @@ -106,12 +106,8 @@ function isKeyboardAnchorNode(node: RawSnapshotNode): boolean { * iOS names the keyboard's own surface elements by accessibility role, and the interactive projection * flattens them beside app content, so ancestry cannot reach them. The dock strip is the clearest * case: its buttons sit below the reported keys, inside the band, and are the keyboard rather than - * app content waiting behind it. + * app content waiting behind it. The table is shared with the settle chrome classifier. */ -const IOS_KEYBOARD_SURFACE_ROLES: ReadonlySet = new Set([ - 'UIKeyboardDockItemButton', - 'UIAccessibilityElementKBKey', -]); /** Every node the keyboard owns, whether or not it contributes geometry to the band. */ function isKeyboardSurfaceNode(node: RawSnapshotNode): boolean { diff --git a/src/commands/interaction/runtime/settle-keyboard-chrome.test.ts b/src/commands/interaction/runtime/settle-keyboard-chrome.test.ts index f3005e293e..59245aef6c 100644 --- a/src/commands/interaction/runtime/settle-keyboard-chrome.test.ts +++ b/src/commands/interaction/runtime/settle-keyboard-chrome.test.ts @@ -48,8 +48,11 @@ test('the settled diff after a fill shows the keyboard as one line, not one per assert.match(texts, /\[keyboard\]/); assert.ok(!diff.lines.some((line) => /\[key\]/.test(line.text)), texts); assert.ok(!/Next keyboard|Dictate|Typing Predictions|keep|kind/.test(texts), texts); - // App content is unchanged; the keyboard line and the floating toolbar the - // keyboard-window capture also reports are the only additions. - assert.equal(diff.summary.additions, 2); + // App content is unchanged; the additions are the keyboard plane line plus + // two accepted noise residues: the floating toolbar the keyboard-window + // capture also reports, and the inert QuickType UIScrollView under the + // assistant markers (no keyboard-owned role of its own; the assistant host + // is a marker and claims no subtree, so nothing under it is subtree-claimed). + assert.equal(diff.summary.additions, 3); assert.equal(diff.summary.removals, 0); }); diff --git a/src/commands/interaction/runtime/stable-capture-signal.ts b/src/commands/interaction/runtime/stable-capture-signal.ts index 07b6299691..36377c3141 100644 --- a/src/commands/interaction/runtime/stable-capture-signal.ts +++ b/src/commands/interaction/runtime/stable-capture-signal.ts @@ -57,12 +57,12 @@ export function stableCaptureSignalsEqual( export function stableCaptureSignalsHaveBroadReplacement( left: StableCaptureSignal | undefined, - right: StableCaptureSignal | undefined, + right: StableCaptureSignal, ): boolean { // A backend transition must reset the ordinary quiet window, but it must not conceal a broad // semantic replacement. Both signals carry the same chrome-free visible projection the settle // diff collapses, and the transition baseline is only armed for XCTest sessions. - if (!left || !right || left.nodes.length === 0 || right.nodes.length === 0) return false; + if (!left || left.nodes.length === 0 || right.nodes.length === 0) return false; const remainingIdentities = new Map(); for (const node of left.nodes) { remainingIdentities.set(node.identity, (remainingIdentities.get(node.identity) ?? 0) + 1);