From b3c479b0b7d85620464e3e6a23e74425c8846870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Sat, 3 Oct 2026 13:53:50 +0200 Subject: [PATCH 1/4] fix(ios): fill succeeds when app removes input after last character Reading frame of an input the app removed after delivery recorded an XCTest failure, which the runner converts to XCTEST_RECORDED_FAILURE and the daemon treats as session-fatal. Bind targets through a throwing snapshot and skip verify/repair once the delivered input is gone, so it never verifies or repairs into the next screen's input. --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 11 +++ .../RunnerTests+TextEntry.swift | 35 +++++++++- .../RunnerTests+TextTyping.swift | 9 +++ .../RunnerTests+TextEntryPolicyTests.swift | 22 ++++++ .../RunnerTests+TextTypingTests.swift | 67 +++++++++++++++++++ 5 files changed, 141 insertions(+), 3 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 1b71d6ce4d..15db8d8c1d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -277,6 +277,7 @@ static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) { // Edits further apart than this belong to different bursts: one runner command's characters arrive // well inside it, and two commands are separated by at least a commit-wait poll and a status read. static const NSTimeInterval AgentDeviceTextEntryBurstBreakSeconds = 1.0; +static const NSUInteger AgentDeviceTextEntryAutoSubmitLength = 6; - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { // A field whose app owns its value, the way a controlled React Native `TextInput` does. A burst @@ -310,6 +311,16 @@ - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { textField.text.length > 0) { [textField removeFromSuperview]; } + // An auto-submitting one-time-code field: the last digit navigates to a screen whose own input sits + // where the code field was. + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-auto-submit"] && + textField.text.length >= AgentDeviceTextEntryAutoSubmitLength && textField.superview != nil) { + UITextField *nextScreenField = [[UITextField alloc] initWithFrame:textField.frame]; + nextScreenField.accessibilityIdentifier = @"agent-device-auto-submit-next-screen-input"; + nextScreenField.borderStyle = UITextBorderStyleRoundedRect; + [textField.superview addSubview:nextScreenField]; + [textField removeFromSuperview]; + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 9791d1e075..47108f62ff 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -115,39 +115,68 @@ extension RunnerTests { var unconfirmed: TextEntryUnconfirmedEvidence? = nil } + /// Which input an element query resolved to. A query bound by index or point re-resolves to + /// whatever input occupies that slot now, so `exists` alone cannot tell the typed input from the + /// next screen's. + struct TextEntryInputIdentity: Equatable { + let elementType: XCUIElement.ElementType + let identifier: String + } + struct TextEntryTarget { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool let fromTapWitness: Bool + /// The input `element` resolved to when it was last bound, or nil when it did not resolve. + let inputIdentity: TextEntryInputIdentity? init( element: XCUIElement?, refreshPoint: CGPoint?, prefersFocusedElement: Bool, - fromTapWitness: Bool = false + fromTapWitness: Bool = false, + inputIdentity: TextEntryInputIdentity? = nil ) { self.element = element self.refreshPoint = refreshPoint self.prefersFocusedElement = prefersFocusedElement self.fromTapWitness = fromTapWitness + self.inputIdentity = inputIdentity } func withElement(_ nextElement: XCUIElement?) -> TextEntryTarget { guard let nextElement else { return self } - let frame = nextElement.frame + // An app may remove the input in reaction to the text just typed. Reading `frame` from a + // removed element records an XCTest failure; a snapshot that cannot resolve only throws. + let snapshot = try? nextElement.snapshot() + let frame = snapshot?.frame ?? .zero let point = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY) return TextEntryTarget( element: nextElement, refreshPoint: point, prefersFocusedElement: prefersFocusedElement, - fromTapWitness: fromTapWitness + fromTapWitness: fromTapWitness, + inputIdentity: snapshot.map { + TextEntryInputIdentity(elementType: $0.elementType, identifier: $0.identifier) + } ) } } + /// Whether the input text was delivered to is gone once delivery finished: removed, or replaced + /// in its query slot by another input, as when an auto-submitting code field navigates away. + /// Unknown when the input never resolved, so that case keeps verifying. + static func textEntryInputRemovedAfterDelivery( + deliveredTo: TextEntryInputIdentity?, + afterDelivery: TextEntryInputIdentity? + ) -> Bool { + guard let deliveredTo else { return false } + return afterDelivery != deliveredTo + } + struct TextEntryStabilization { let element: XCUIElement? let focusConfirmed: Bool diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index f5d59e2a53..f56a8dcce8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -85,6 +85,7 @@ extension RunnerTests { let initialResolveStartedAt = Date() let initialTarget = resolveTextEntryElement(app: app, target: activeTarget) activeTarget = activeTarget.withElement(initialTarget) + let deliveredTo = activeTarget.inputIdentity let currentText = editableTextValue(for: initialTarget, treatingPlaceholderAsEmpty: true) let initialText = repairMode == .append ? currentText : nil let expectedText = expectedTextEntryValue(typedText: text, mode: repairMode, initialText: initialText) @@ -345,6 +346,7 @@ extension RunnerTests { var result = verifyTextEntryWithRepairIfNeeded( app: app, target: activeTarget, + deliveredTo: deliveredTo, expectedText: expectedText, repairMode: repairMode, baseline: entryBaseline @@ -392,10 +394,17 @@ extension RunnerTests { private func verifyTextEntryWithRepairIfNeeded( app: XCUIApplication, target: TextEntryTarget, + deliveredTo: TextEntryInputIdentity?, expectedText: String?, repairMode: TextTypingRepairMode, baseline: TextEntryObservation? ) -> TextEntryResult { + // The app reacted to the delivered text by removing the input. Its value can no longer be read + // back, and re-resolving by point or focus would verify, and possibly repair, another input. + if Self.textEntryInputRemovedAfterDelivery(deliveredTo: deliveredTo, afterDelivery: target.inputIdentity) { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_INPUT_REMOVED_AFTER_DELIVERY") + return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil) + } let initialResult = verifyTextEntry( app: app, target: target, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index ab1c28afb1..b837b17663 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -198,6 +198,28 @@ extension RunnerTests { XCTAssertFalse(TextEntryFailure.commitNotObserved.message.contains("only part")) } + func testInputCountsAsRemovedOnlyWhenItsDeliveryIdentityIsGone() { + let codeField = TextEntryInputIdentity(elementType: .textField, identifier: "code") + let nextScreenField = TextEntryInputIdentity(elementType: .textField, identifier: "name") + let cases: [(deliveredTo: TextEntryInputIdentity?, after: TextEntryInputIdentity?, expected: Bool)] = [ + (codeField, codeField, false), + (codeField, nil, true), + (codeField, nextScreenField, true), + (nil, nil, false), + (nil, nextScreenField, false), + ] + for testCase in cases { + XCTAssertEqual( + Self.textEntryInputRemovedAfterDelivery( + deliveredTo: testCase.deliveredTo, + afterDelivery: testCase.after + ), + testCase.expected, + String(describing: testCase) + ) + } + } + #if os(iOS) func testSynthesizedTextEntryFallsBackOnlyWhenPrivateSynthesisIsUnavailable() { XCTAssertEqual( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 46d59bdeea..78ecf0cc8e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -153,6 +153,46 @@ extension RunnerTests { XCTAssertFalse(textField.exists) } + // An auto-submitting code field: the last digit navigates to a screen with its own input where the + // code field was. The fill delivered every character, so it succeeds unverified instead of + // recording an XCTest failure that the runner would convert into XCTEST_RECORDED_FAILURE and a + // session restart, and it neither verifies nor repairs into the next screen's input. + @MainActor + func testFillSucceedsWhenAppRemovesInputAfterLastCharacter() throws { + let textField = try launchRemovableInputFixture("--agent-device-text-entry-auto-submit") + + let failureCountBefore = currentXCTestFailureCount() + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-auto-submit", text: "123456", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertFalse(textField.exists) + let nextScreenField = app.textFields["agent-device-auto-submit-next-screen-input"] + XCTAssertTrue(nextScreenField.exists) + XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "") + } + + // The closest negative: the input is removed after the first character, so the rest was never + // delivered. That stays a typed failure, still without an XCTest failure. + @MainActor + func testFillFailsWhenAppRemovesInputBeforeTextIsDelivered() throws { + let textField = try launchRemovableInputFixture("--agent-device-text-entry-disappear-after-input") + + let failureCountBefore = currentXCTestFailureCount() + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-disappearing-input", text: "123456", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertFalse(textField.exists) + } + // Text past the delivery budget cannot be paced into a field the runner cannot resolve, so it goes // through application-wide typing. The budget is charged the whole command, warmup split included: // an append peels its first character for warmup, so a per-dispatch charge would find both of its @@ -241,6 +281,33 @@ extension RunnerTests { return textField } + /// Launches the soft-keyboard text-entry fixture with `removalArgument` choosing when the app + /// removes its input, and returns that input. + private func launchRemovableInputFixture(_ removalArgument: String) throws -> XCUIElement { + app.launchArguments = [ + "--agent-device-text-entry-regression", + "--agent-device-text-entry-soft-keyboard", + removalArgument, + ] + app.launch() + addTeardownBlock { [self] in + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + return textField + } + + /// The `type` command the daemon sends for `fill`: replace mode, addressed by the input's center. + private func fillCommandFixture(commandId: String, text: String, at element: XCUIElement) throws -> Command { + let frame = element.frame + return try runnerCommandFixture( + #"{"command":"type","commandId":"\#(commandId)","text":"\#(text)","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# + ) + } + @MainActor private func tapHardwareKeyboardInput(commandId: String) throws { let tapCommand = try runnerCommandFixture( From 91b23b20f7e638d9053adc7d10cec099c9495f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Sat, 3 Oct 2026 17:33:44 +0200 Subject: [PATCH 2/4] fix(ios): refuse text entry into an input that replaced the bound one Bind the input's type and identifier at the first resolve and check it in resolveTextEntryElement, which every post, warmup read-back, verify poll and repair call. Count the input as removed only on XCTest's no-match (10008). Refuse repair when the bound input has no identifier. --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 26 +++- .../RunnerTests+TextEntry.swift | 144 +++++++++++++++--- .../RunnerTests+TextTyping.swift | 29 +++- .../RunnerTests+TextEntryPolicyTests.swift | 37 ++--- .../RunnerTests+TextTypingTests.swift | 85 +++++++++-- 5 files changed, 261 insertions(+), 60 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 15db8d8c1d..32f11186ef 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -311,15 +311,27 @@ - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { textField.text.length > 0) { [textField removeFromSuperview]; } - // An auto-submitting one-time-code field: the last digit navigates to a screen whose own input sits - // where the code field was. - if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-auto-submit"] && - textField.text.length >= AgentDeviceTextEntryAutoSubmitLength && textField.superview != nil) { + // An auto-submitting one-time-code field: the last digit navigates away, to a screen whose own + // input takes the code field's place or to one without an input. `replace-after-input` is the + // same navigation landing before the code is complete. + NSArray *arguments = NSProcessInfo.processInfo.arguments; + BOOL codeComplete = textField.text.length >= AgentDeviceTextEntryAutoSubmitLength; + if (textField.superview == nil) { + return; + } + if ((codeComplete && [arguments containsObject:@"--agent-device-text-entry-auto-submit"]) || + (textField.text.length > 0 && [arguments containsObject:@"--agent-device-text-entry-replace-after-input"])) { UITextField *nextScreenField = [[UITextField alloc] initWithFrame:textField.frame]; - nextScreenField.accessibilityIdentifier = @"agent-device-auto-submit-next-screen-input"; + if (![arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) { + nextScreenField.accessibilityIdentifier = @"agent-device-auto-submit-next-screen-input"; + } nextScreenField.borderStyle = UITextBorderStyleRoundedRect; [textField.superview addSubview:nextScreenField]; [textField removeFromSuperview]; + [nextScreenField becomeFirstResponder]; + } else if (codeComplete && + [arguments containsObject:@"--agent-device-text-entry-auto-submit-without-successor"]) { + [textField removeFromSuperview]; } } #endif @@ -367,7 +379,9 @@ - (void)viewDidLoad { [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-count-value"]; UITextField *textField = digitCountValue ? [[AgentDeviceDigitCountTextField alloc] init] : [[UITextField alloc] init]; - textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input"; + if (![NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) { + textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input"; + } textField.borderStyle = UITextBorderStyleRoundedRect; // An empty input view keeps the software keyboard down, which is the hardware-keyboard responder // these routes are addressed to. `--agent-device-text-entry-soft-keyboard` leaves the real input diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 47108f62ff..24ae52dae4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -115,12 +115,14 @@ extension RunnerTests { var unconfirmed: TextEntryUnconfirmedEvidence? = nil } - /// Which input an element query resolved to. A query bound by index or point re-resolves to - /// whatever input occupies that slot now, so `exists` alone cannot tell the typed input from the - /// next screen's. + /// The input a text entry command bound to. XCUIElement exposes no per-instance id, and a query + /// bound by index or point re-resolves to whatever input occupies that slot now, so element type + /// plus identifier is the identity. It is unique only when the identifier is non-empty. struct TextEntryInputIdentity: Equatable { let elementType: XCUIElement.ElementType let identifier: String + + var isDistinguishable: Bool { !identifier.isEmpty } } struct TextEntryTarget { @@ -128,30 +130,31 @@ extension RunnerTests { let refreshPoint: CGPoint? let prefersFocusedElement: Bool let fromTapWitness: Bool - /// The input `element` resolved to when it was last bound, or nil when it did not resolve. - let inputIdentity: TextEntryInputIdentity? + /// The input the first resolved element was. Once bound, resolution refuses any other input. + let boundIdentity: TextEntryInputIdentity? init( element: XCUIElement?, refreshPoint: CGPoint?, prefersFocusedElement: Bool, fromTapWitness: Bool = false, - inputIdentity: TextEntryInputIdentity? = nil + boundIdentity: TextEntryInputIdentity? = nil ) { self.element = element self.refreshPoint = refreshPoint self.prefersFocusedElement = prefersFocusedElement self.fromTapWitness = fromTapWitness - self.inputIdentity = inputIdentity + self.boundIdentity = boundIdentity } func withElement(_ nextElement: XCUIElement?) -> TextEntryTarget { guard let nextElement else { return self } - // An app may remove the input in reaction to the text just typed. Reading `frame` from a - // removed element records an XCTest failure; a snapshot that cannot resolve only throws. - let snapshot = try? nextElement.snapshot() + // Reading `frame` from an input the app has removed records an XCTest failure; a snapshot + // only throws, after XCTest waits about two seconds for a match, so it is skipped when the + // input is already gone. + let snapshot = nextElement.exists ? try? nextElement.snapshot() : nil let frame = snapshot?.frame ?? .zero let point = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY) return TextEntryTarget( @@ -159,22 +162,39 @@ extension RunnerTests { refreshPoint: point, prefersFocusedElement: prefersFocusedElement, fromTapWitness: fromTapWitness, - inputIdentity: snapshot.map { + boundIdentity: boundIdentity ?? snapshot.map { TextEntryInputIdentity(elementType: $0.elementType, identifier: $0.identifier) } ) } } - /// Whether the input text was delivered to is gone once delivery finished: removed, or replaced - /// in its query slot by another input, as when an auto-submitting code field navigates away. - /// Unknown when the input never resolved, so that case keeps verifying. - static func textEntryInputRemovedAfterDelivery( - deliveredTo: TextEntryInputIdentity?, - afterDelivery: TextEntryInputIdentity? - ) -> Bool { - guard let deliveredTo else { return false } - return afterDelivery != deliveredTo + /// What one snapshot of a candidate element proved. + enum TextEntryInputProbe: Equatable { + case input(TextEntryInputIdentity) + /// XCTest found no element for the query: the only error that proves absence. + case noMatch + /// Any other snapshot failure, such as a transient accessibility error or multiple matches. + case unavailable + } + + static let xCTestUITestingErrorDomain = "com.apple.dt.xctest.ui-testing.error" + /// `snapshot()` of a query that matches nothing, as for an input the app removed. A query with + /// several matches throws 10006 and one into an app that is not running throws 10001. + static let xCTestNoMatchesErrorCode = 10008 + + /// Classifies a candidate's `snapshot()` error: only XCTest's no-match proves the input is gone. + static func textEntryInputProbe(snapshotError error: Error) -> TextEntryInputProbe { + let error = error as NSError + return error.domain == xCTestUITestingErrorDomain && error.code == xCTestNoMatchesErrorCode + ? .noMatch + : .unavailable + } + + /// Repair clears and retypes whatever input resolves, so it needs an identity that a + /// successor input cannot share. An unbound target keeps its unguarded repair. + static func textEntryRepairCanTarget(boundIdentity: TextEntryInputIdentity?) -> Bool { + boundIdentity?.isDistinguishable ?? true } struct TextEntryStabilization { @@ -233,7 +253,91 @@ extension RunnerTests { } } + /// Resolves the target's input. A bound target accepts only an element whose snapshot carries + /// its bound identity, so every post, read-back, verification poll, and repair refuses an input + /// that took the bound one's place. func resolveTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? { + guard let boundIdentity = target.boundIdentity else { + return resolveUnboundTextEntryElement(app: app, target: target) + } + func isBoundInput(_ candidate: XCUIElement) -> Bool { + candidate.exists && probeTextEntryInput(candidate) == .input(boundIdentity) + } + if target.prefersFocusedElement, let focused = focusedTextInput(app: app), isBoundInput(focused) { + return focused + } + if let element = target.element, isBoundInput(element) { + return element + } + if let refreshPoint = target.refreshPoint, + case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint), + let match = candidates.first(where: isBoundInput) { + return match + } + if let focused = focusedTextInput(app: app), isBoundInput(focused) { + return focused + } + if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity), isBoundInput(byIdentifier) { + return byIdentifier + } + return nil + } + + /// Whether a bound target's input is proven gone: XCTest found no match for it, or a different + /// input answers in its place, and nothing at its point carries its identity. A failed probe + /// proves nothing. Unbound targets and unidentified inputs without an element cannot be proven + /// gone. + func boundTextEntryInputIsGone(app: XCUIApplication, target: TextEntryTarget) -> Bool { + guard let boundIdentity = target.boundIdentity, + let query = boundTextEntryInputQuery(app: app, identity: boundIdentity) ?? target.element + else { + return false + } + switch probeTextEntryInput(query) { + case .input(let identity) where identity != boundIdentity: + break + case .noMatch: + break + case .input, .unavailable: + return false + } + guard let refreshPoint = target.refreshPoint else { + return true + } + switch probeTextInputs(app: app, point: refreshPoint) { + case .absent: + return true + case .matches(let candidates): + return candidates.allSatisfy { probeTextEntryInput($0) != .input(boundIdentity) } + case .unavailable: + return false + } + } + + /// The app-wide query for an input whose identifier is unique enough to search by. + private func boundTextEntryInputQuery(app: XCUIApplication, identity: TextEntryInputIdentity) -> XCUIElement? { + guard identity.isDistinguishable else { + return nil + } + return app.descendants(matching: identity.elementType).matching(identifier: identity.identifier).element + } + + /// Snapshots one candidate: its identity, a proven no-match, or a failure that proves nothing. + private func probeTextEntryInput(_ element: XCUIElement) -> TextEntryInputProbe { + var probe = TextEntryInputProbe.unavailable + let (_, exception) = catchingObjCException(fallback: ()) { + do { + let snapshot = try element.snapshot() + probe = .input(TextEntryInputIdentity(elementType: snapshot.elementType, identifier: snapshot.identifier)) + } catch { + probe = Self.textEntryInputProbe(snapshotError: error) + } + } + return exception == nil ? probe : .unavailable + } + + /// Resolution for a target not bound to an input yet: the first element that exists. + private func resolveUnboundTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? { if target.prefersFocusedElement { if let focused = focusedTextInput(app: app) { return focused diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index f56a8dcce8..ec88d4bbd8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -85,7 +85,6 @@ extension RunnerTests { let initialResolveStartedAt = Date() let initialTarget = resolveTextEntryElement(app: app, target: activeTarget) activeTarget = activeTarget.withElement(initialTarget) - let deliveredTo = activeTarget.inputIdentity let currentText = editableTextValue(for: initialTarget, treatingPlaceholderAsEmpty: true) let initialText = repairMode == .append ? currentText : nil let expectedText = expectedTextEntryValue(typedText: text, mode: repairMode, initialText: initialText) @@ -197,7 +196,7 @@ extension RunnerTests { textEntryRoute = "xctest-element" currentTarget.typeText(value) return (currentTarget, nil) - } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) { + } else if activeTarget.boundIdentity == nil && activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) { #if os(iOS) // Two ways this post leaves the synthesized channel, and both hand the text to // application-wide typing: the command's own budget refused it, or XCTest's private synthesis @@ -346,7 +345,6 @@ extension RunnerTests { var result = verifyTextEntryWithRepairIfNeeded( app: app, target: activeTarget, - deliveredTo: deliveredTo, expectedText: expectedText, repairMode: repairMode, baseline: entryBaseline @@ -394,14 +392,22 @@ extension RunnerTests { private func verifyTextEntryWithRepairIfNeeded( app: XCUIApplication, target: TextEntryTarget, - deliveredTo: TextEntryInputIdentity?, expectedText: String?, repairMode: TextTypingRepairMode, baseline: TextEntryObservation? ) -> TextEntryResult { - // The app reacted to the delivered text by removing the input. Its value can no longer be read - // back, and re-resolving by point or focus would verify, and possibly repair, another input. - if Self.textEntryInputRemovedAfterDelivery(deliveredTo: deliveredTo, afterDelivery: target.inputIdentity) { + if target.boundIdentity != nil, resolveTextEntryElement(app: app, target: target) == nil { + guard boundTextEntryInputIsGone(app: app, target: target) else { + return TextEntryResult( + verified: nil, + repaired: false, + expectedText: expectedText, + observedText: nil, + failure: .commitNotObserved + ) + } + // Every character was delivered and the app then removed the input, as an auto-submitting + // code field does. Its value can no longer be read back. NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_INPUT_REMOVED_AFTER_DELIVERY") return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil) } @@ -440,6 +446,15 @@ extension RunnerTests { repaired: false ) } + guard Self.textEntryRepairCanTarget(boundIdentity: target.boundIdentity) else { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_REPAIR_REFUSED reason=unidentified-input") + return verifyTextEntry( + app: app, + target: target, + expectedText: expectedText, + repaired: false + ) + } guard let repairTarget = resolveTextEntryElement(app: app, target: target) else { return initialResult diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index b837b17663..0f5b7a86c1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -198,28 +198,31 @@ extension RunnerTests { XCTAssertFalse(TextEntryFailure.commitNotObserved.message.contains("only part")) } - func testInputCountsAsRemovedOnlyWhenItsDeliveryIdentityIsGone() { - let codeField = TextEntryInputIdentity(elementType: .textField, identifier: "code") - let nextScreenField = TextEntryInputIdentity(elementType: .textField, identifier: "name") - let cases: [(deliveredTo: TextEntryInputIdentity?, after: TextEntryInputIdentity?, expected: Bool)] = [ - (codeField, codeField, false), - (codeField, nil, true), - (codeField, nextScreenField, true), - (nil, nil, false), - (nil, nextScreenField, false), + // Codes observed from `snapshot()` on an iOS 27.0 simulator: 10008 for an input the app removed, + // 10006 for a query with several matches, 10001 for an app that is not running. + func testOnlyXCTestNoMatchProvesATextEntryInputIsGone() { + let domain = Self.xCTestUITestingErrorDomain + let cases: [(error: NSError, expected: TextEntryInputProbe)] = [ + (NSError(domain: domain, code: 10008), .noMatch), + (NSError(domain: domain, code: 10006), .unavailable), + (NSError(domain: domain, code: 10001), .unavailable), + (NSError(domain: "com.example.other", code: 10008), .unavailable), ] for testCase in cases { - XCTAssertEqual( - Self.textEntryInputRemovedAfterDelivery( - deliveredTo: testCase.deliveredTo, - afterDelivery: testCase.after - ), - testCase.expected, - String(describing: testCase) - ) + XCTAssertEqual(Self.textEntryInputProbe(snapshotError: testCase.error), testCase.expected, String(describing: testCase)) } } + func testTextEntryRepairNeedsAnIdentityASuccessorCannotShare() { + XCTAssertTrue(Self.textEntryRepairCanTarget(boundIdentity: nil)) + XCTAssertTrue( + Self.textEntryRepairCanTarget(boundIdentity: TextEntryInputIdentity(elementType: .textField, identifier: "code")) + ) + XCTAssertFalse( + Self.textEntryRepairCanTarget(boundIdentity: TextEntryInputIdentity(elementType: .textField, identifier: "")) + ) + } + #if os(iOS) func testSynthesizedTextEntryFallsBackOnlyWhenPrivateSynthesisIsUnavailable() { XCTAssertEqual( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 78ecf0cc8e..961ea44aa6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -154,11 +154,11 @@ extension RunnerTests { } // An auto-submitting code field: the last digit navigates to a screen with its own input where the - // code field was. The fill delivered every character, so it succeeds unverified instead of - // recording an XCTest failure that the runner would convert into XCTEST_RECORDED_FAILURE and a - // session restart, and it neither verifies nor repairs into the next screen's input. + // code field was. Every character was delivered, so the fill succeeds unverified, without an + // XCTest failure (XCTEST_RECORDED_FAILURE and a session restart), and without verifying or + // repairing into the next screen's input. @MainActor - func testFillSucceedsWhenAppRemovesInputAfterLastCharacter() throws { + func testFillSucceedsWhenAppReplacesInputAfterLastCharacter() throws { let textField = try launchRemovableInputFixture("--agent-device-text-entry-auto-submit") let failureCountBefore = currentXCTestFailureCount() @@ -169,12 +169,31 @@ extension RunnerTests { XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(response.data?.message, "typed") XCTAssertFalse(textField.exists) let nextScreenField = app.textFields["agent-device-auto-submit-next-screen-input"] XCTAssertTrue(nextScreenField.exists) XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "") } + // The reported shape: the last digit navigates to a screen without an input. + @MainActor + func testFillSucceedsWhenAppRemovesInputAfterLastCharacter() throws { + let textField = try launchRemovableInputFixture("--agent-device-text-entry-auto-submit-without-successor") + + let failureCountBefore = currentXCTestFailureCount() + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-auto-submit-no-successor", text: "123456", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(response.data?.message, "typed") + XCTAssertFalse(textField.exists) + XCTAssertEqual(app.textFields.count, 0) + } + // The closest negative: the input is removed after the first character, so the rest was never // delivered. That stays a typed failure, still without an XCTest failure. @MainActor @@ -193,6 +212,51 @@ extension RunnerTests { XCTAssertFalse(textField.exists) } + // The navigation lands after the first character and focuses a successor input in the same spot. + // The remaining posts resolve that successor by point, so they must refuse it rather than type the + // rest of the code into the next screen. + @MainActor + func testFillFailsWhenAppReplacesInputBeforeTextIsDelivered() throws { + let textField = try launchRemovableInputFixture("--agent-device-text-entry-replace-after-input") + + let failureCountBefore = currentXCTestFailureCount() + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-replaced-input", text: "123456", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertFalse(textField.exists) + let nextScreenField = app.textFields["agent-device-auto-submit-next-screen-input"] + XCTAssertTrue(nextScreenField.exists) + XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "") + } + + // Neither input has an identifier, so the successor carries the same identity as the code field. + // The runner cannot tell them apart and must not clear and retype the successor: it reports the + // mismatch it read instead of repairing. + @MainActor + func testFillDoesNotRepairIntoAnIndistinguishableSuccessorInput() throws { + let textField = try launchRemovableInputFixture( + "--agent-device-text-entry-auto-submit", + "--agent-device-text-entry-unnamed-input" + ) + + let failureCountBefore = currentXCTestFailureCount() + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-unnamed-successor", text: "123456", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_ENTRY_MISMATCH") + XCTAssertEqual(app.textFields.count, 1) + XCTAssertEqual(editableTextValue(for: app.textFields.element(boundBy: 0), treatingPlaceholderAsEmpty: true), "") + } + // Text past the delivery budget cannot be paced into a field the runner cannot resolve, so it goes // through application-wide typing. The budget is charged the whole command, warmup split included: // an append peels its first character for warmup, so a per-dispatch charge would find both of its @@ -281,21 +345,22 @@ extension RunnerTests { return textField } - /// Launches the soft-keyboard text-entry fixture with `removalArgument` choosing when the app - /// removes its input, and returns that input. - private func launchRemovableInputFixture(_ removalArgument: String) throws -> XCUIElement { + /// Launches the soft-keyboard text-entry fixture with `arguments` choosing when the app removes or + /// replaces its input, and returns that input bound by index, since it may have no identifier. + private func launchRemovableInputFixture(_ arguments: String...) throws -> XCUIElement { app.launchArguments = [ "--agent-device-text-entry-regression", "--agent-device-text-entry-soft-keyboard", - removalArgument, - ] + ] + arguments app.launch() addTeardownBlock { [self] in invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) - let textField = app.textFields["agent-device-hardware-keyboard-input"] + let textField = arguments.contains("--agent-device-text-entry-unnamed-input") + ? app.textFields.element(boundBy: 0) + : app.textFields["agent-device-hardware-keyboard-input"] XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) return textField } From ef3bb3eb9ff1a3803471cb12afb3efa0021a8c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Sat, 3 Oct 2026 23:17:12 +0200 Subject: [PATCH 3/4] test(ios): pin typed failure when an unnamed input replaces the field mid-fill --- .../RunnerTests+TextTypingTests.swift | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 961ea44aa6..6ecca07392 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -257,6 +257,30 @@ extension RunnerTests { XCTAssertEqual(editableTextValue(for: app.textFields.element(boundBy: 0), treatingPlaceholderAsEmpty: true), "") } + // The known limit: neither input has an identifier, and the successor takes the code field's index + // in the query that bound it, so the remaining posts resolve to it. Refusing point and focus + // re-resolution would not help, because the index-bound query itself returns the successor. The + // command must still fail typed, never report success or repair into the successor. + @MainActor + func testFillFailsTypedWhenAnIndistinguishableInputReplacesItMidDelivery() throws { + let textField = try launchRemovableInputFixture( + "--agent-device-text-entry-replace-after-input", + "--agent-device-text-entry-unnamed-input" + ) + + let failureCountBefore = currentXCTestFailureCount() + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-unnamed-replaced", text: "123456", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_ENTRY_MISMATCH") + XCTAssertEqual(app.textFields.count, 1) + XCTAssertEqual(editableTextValue(for: app.textFields.element(boundBy: 0), treatingPlaceholderAsEmpty: true), "23456") + } + // Text past the delivery budget cannot be paced into a field the runner cannot resolve, so it goes // through application-wide typing. The budget is charged the whole command, warmup split included: // an append peels its first character for warmup, so a per-dispatch charge would find both of its From 0eef3bb26fffd67191e1f1bea8abffb7e79cb2dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Sun, 4 Oct 2026 09:35:59 +0200 Subject: [PATCH 4/4] test(ios): cover auto-submit of a digit-count field and frame the unnamed-successor limit --- .../RunnerTests+TextEntry.swift | 4 +- .../RunnerTests+TextTypingTests.swift | 38 ++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 24ae52dae4..4c039d18ec 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -117,7 +117,9 @@ extension RunnerTests { /// The input a text entry command bound to. XCUIElement exposes no per-instance id, and a query /// bound by index or point re-resolves to whatever input occupies that slot now, so element type - /// plus identifier is the identity. It is unique only when the identifier is non-empty. + /// plus identifier is the identity. It is unique only when the identifier is non-empty. Unlike + /// `TextEntryElementIdentity` it leaves out the frame: it must hold across every post of a fill, + /// while the field can still be moving with the keyboard. struct TextEntryInputIdentity: Equatable { let elementType: XCUIElement.ElementType let identifier: String diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 6ecca07392..8eade0870f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -257,12 +257,40 @@ extension RunnerTests { XCTAssertEqual(editableTextValue(for: app.textFields.element(boundBy: 0), treatingPlaceholderAsEmpty: true), "") } - // The known limit: neither input has an identifier, and the successor takes the code field's index - // in the query that bound it, so the remaining posts resolve to it. Refusing point and focus - // re-resolution would not help, because the index-bound query itself returns the successor. The - // command must still fail typed, never report success or repair into the successor. + // A one-time-code field whose value is a digit-count summary auto-submits on the last digit and a + // named input takes its place. The bound field is gone, so the fill is the plain unverified + // "typed": no unconfirmed evidence read off the successor, and no text in it. @MainActor - func testFillFailsTypedWhenAnIndistinguishableInputReplacesItMidDelivery() throws { + func testFillIntoAutoSubmittingDigitCountFieldReportsNoEvidenceFromItsSuccessor() throws { + let textField = try launchRemovableInputFixture( + "--agent-device-text-entry-digit-count-value", + "--agent-device-text-entry-auto-submit" + ) + + let failureCountBefore = currentXCTestFailureCount() + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-digit-count-auto-submit", text: "123456", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(response.data?.message, "typed") + XCTAssertNil(response.data?.verification) + XCTAssertFalse(textField.exists) + let nextScreenField = app.textFields["agent-device-auto-submit-next-screen-input"] + XCTAssertTrue(nextScreenField.exists) + XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "") + } + + // Pins a known limit, not desired behavior: neither input has an identifier, and the successor + // takes the code field's index in the query that bound it, so the remaining posts resolve to it + // and land in the successor. Refusing point and focus re-resolution would not help, because the + // index-bound query itself returns the successor. What must hold is a typed failure, never success + // or a repair into the successor. The successor's "23456" records the wrong-target side effect; a + // per-instance identity would make that assertion fail, and the test should then change with it. + @MainActor + func testFillPinsKnownLimitWhenAnIndistinguishableInputReplacesItMidDelivery() throws { let textField = try launchRemovableInputFixture( "--agent-device-text-entry-replace-after-input", "--agent-device-text-entry-unnamed-input"