From f04602e664bb8f5e73663a87b3b4a674fab8d27c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 12 Sep 2026 16:17:20 -0700 Subject: [PATCH] fix: separate screen capture and accessibility ownership --- CHANGELOG.md | 2 + ...kabooBridgeDesktopObservationBinding.swift | 58 +++++++- ...oBridgeScreenObservationBindingTests.swift | 135 ++++++++++++++++++ docs/commands/see.md | 4 + 4 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeScreenObservationBindingTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index c9ea76cfb..15e640aae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Fix host-routed screen observations with Accessibility elements by validating their semantic owner separately from the screen raster target. #715, #710. + ## 4.3.4 - 2026-09-11 **Highlights:** Restore provider-compatible MCP tools and frontmost daemon captures. diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift index 794a26861..dad02342d 100644 --- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift +++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift @@ -484,11 +484,16 @@ enum PeekabooBridgeDesktopObservationBinding { private static func validateApplicationEvidence(_ result: DesktopObservationResult) -> String? { guard let targetApp = result.target.app else { if result.capture.metadata.applicationInfo != nil || - result.target.detectionContext?.applicationProcessId != nil || - result.elements?.metadata.windowContext?.applicationProcessId != nil + result.target.detectionContext?.applicationProcessId != nil { return "unexpected application evidence" } + if case .screen = result.target.kind { + return self.validateScreenSemanticOwner(result.elements?.metadata.windowContext) + } + if result.elements?.metadata.windowContext?.applicationProcessId != nil { + return "unexpected application evidence" + } return nil } @@ -541,14 +546,59 @@ enum PeekabooBridgeDesktopObservationBinding { return nil } + private static func validateScreenSemanticOwner(_ context: WindowContext?) -> String? { + guard let context, + context.applicationProcessId != nil || context.applicationProcessStartIdentity != nil || + context.windowID != nil || context.windowMutationIdentity != nil || context.focusedElement != nil + else { return nil } + let mismatch = "screen accessibility owner" + guard context.applicationProcessId != nil else { return mismatch } + if let windowID = context.windowID, windowID <= 0 || UInt32(exactly: windowID) == nil { + return mismatch + } + if let bounds = context.windowBounds, !self.isValidBounds(bounds) { + return mismatch + } + if let focused = context.focusedElement { + guard focused.processIdentifier == context.applicationProcessId, + focused.windowID > 0, UInt32(exactly: focused.windowID) != nil, + !focused.role.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + self.isValidBounds(focused.frame) + else { return mismatch } + if let windowID = context.windowID, focused.windowID != windowID { + return mismatch + } + if let bounds = context.windowBounds, + !bounds.contains(CGPoint(x: focused.frame.midX, y: focused.frame.midY)) + { + return mismatch + } + } + do { + // Screen pixels have no app target. AX carries its own generation-bound semantic owner; + // unreceipted window hints must never promote those pixels to exact-window authority. + guard try DesktopTargetPlanning.DesktopTargetIdentityCoalescer.resolve([ + DesktopTargetEvidenceAdapter.evidence(context: context), + ]) != nil else { return mismatch } + } catch { + return mismatch + } + return nil + } + private static func validateWindowEvidence(_ result: DesktopObservationResult) -> String? { guard let targetWindow = result.target.window else { if result.capture.metadata.windowInfo != nil || - result.target.detectionContext?.windowID != nil || - result.elements?.metadata.windowContext?.windowID != nil + result.target.detectionContext?.windowID != nil { return "unexpected window evidence" } + if case .screen = result.target.kind { + return nil + } + if result.elements?.metadata.windowContext?.windowID != nil { + return "unexpected window evidence" + } return nil } diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeScreenObservationBindingTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeScreenObservationBindingTests.swift new file mode 100644 index 000000000..d1f46d644 --- /dev/null +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeScreenObservationBindingTests.swift @@ -0,0 +1,135 @@ +import CoreGraphics +import Foundation +import PeekabooAutomationKit +import PeekabooFoundation +import Testing +@testable import PeekabooBridge + +@Suite(.serialized) +struct PeekabooBridgeScreenObservationBindingTests: DesktopObservationBindingFixtureProviding { + @Test(arguments: [false, true]) + @MainActor + func `screen raster and generation bound AX owner remain distinct signed evidence`( + includesFocusedElement: Bool) async throws + { + let request = Self.request + let result = Self.result(context: Self.context(focusedElement: includesFocusedElement ? Self.focus() : nil)) + #expect(PeekabooBridgeDesktopObservationBinding.mismatch(request: request, result: result) == nil) + #expect(result.target.app == nil) + #expect(result.target.window == nil) + #expect(result.capture.metadata.applicationInfo == nil) + #expect(result.elements?.metadata.windowContext?.applicationProcessId == 42) + + let provider = ObservationProvider(result: result) + let server = Self.server(provider: provider) + _ = try await PeekabooBridgeRequestContext.$usesAttestedOperationResultSemantics.withValue(true) { + try await server.handleAuthorized(.desktopObservation(request), peer: nil, permissions: Self.permissions) + } + #expect(provider.observationCount == 1) + + let bundle = try await Self.makeBundle( + request: .desktopObservation(request), response: .desktopObservation(result), target: .global) + try bundle.validateIntegrity() + } + + @Test + @MainActor + func `screen AX owner rejects missing generation and contradictory window receipts`() async throws { + let wrongOwner = WindowMutationIdentity( + windowID: 73, + ownerProcessIdentifier: 99, + ownerProcessStartIdentity: 1001, + capturedBounds: Self.windowBounds) + let wrongGeneration = WindowMutationIdentity( + windowID: 73, + ownerProcessIdentifier: 42, + ownerProcessStartIdentity: 1002, + capturedBounds: Self.windowBounds) + let wrongBounds = WindowMutationIdentity( + windowID: 73, + ownerProcessIdentifier: 42, + ownerProcessStartIdentity: 1001, + capturedBounds: CGRect(x: 10, y: 10, width: 400, height: 300)) + let contexts = [ + Self.context(generation: nil), + Self.context(generation: 0), + Self.context(processIdentifier: 0), + Self.context(receipt: wrongOwner), + Self.context(receipt: wrongGeneration), + Self.context(receipt: wrongBounds), + Self.context(focusedElement: Self.focus(processIdentifier: 99)), + Self.context(focusedElement: Self.focus(windowID: 74)), + Self.context(focusedElement: Self.focus(role: "")), + Self.context(focusedElement: Self.focus(frame: CGRect(x: 1000, y: 1000, width: 10, height: 10))), + ] + for context in contexts { + let result = Self.result(context: context) + #expect(PeekabooBridgeDesktopObservationBinding.mismatch( + request: Self.request, result: result) != nil) + let bundle = try await Self.makeBundle( + request: .desktopObservation(Self.request), response: .desktopObservation(result), target: .global) + #expect(throws: PeekabooBridgeOperationReceiptError.self) { + try bundle.validateIntegrity() + } + } + } + + @Test + func `screen semantics accept a consistent exact window receipt without changing raster scope`() { + let receipt = WindowMutationIdentity( + windowID: 73, + ownerProcessIdentifier: 42, + ownerProcessStartIdentity: 1001, + capturedBounds: Self.windowBounds) + let result = Self.result(context: Self.context(receipt: receipt)) + #expect(PeekabooBridgeDesktopObservationBinding.mismatch(request: Self.request, result: result) == nil) + #expect(result.capture.metadata.windowInfo == nil) + } + + private static var request: DesktopObservationRequest { + DesktopObservationRequest(target: .screen(index: 0), detection: .init(traversalBudget: AXTraversalBudget())) + } + + private static let windowBounds = CGRect(x: 20, y: 30, width: 400, height: 300) + + private static func context( + processIdentifier: Int32 = 42, + generation: UInt64? = 1001, + receipt: WindowMutationIdentity? = nil, + focusedElement: FocusedElementIdentity? = nil) -> WindowContext + { + WindowContext( + applicationName: "Fixture", + applicationBundleId: "example.fixture", + applicationProcessId: processIdentifier, + applicationProcessStartIdentity: generation, + windowTitle: "Fixture window", + windowID: 73, + windowBounds: self.windowBounds, + windowMutationIdentity: receipt, + focusedElement: focusedElement, + shouldFocusWebContent: false, + includeMenuBarElements: false, + traversalBudget: AXTraversalBudget()) + } + + private static func focus( + processIdentifier: Int32 = 42, + windowID: Int = 73, + role: String = "AXButton", + frame: CGRect = CGRect(x: 30, y: 40, width: 10, height: 10)) -> FocusedElementIdentity + { + FocusedElementIdentity(processIdentifier: processIdentifier, windowID: windowID, role: role, frame: frame) + } + + private static func result(context: WindowContext) -> DesktopObservationResult { + replacingElements( + screenResult(index: 0), + with: ElementDetectionResult( + snapshotId: "screen-semantic-fixture", + screenshotPath: "", + elements: .init(buttons: [.init(id: "B1", type: .button, bounds: self.windowBounds)]), + metadata: .init( + detectionTime: 0, elementCount: 1, method: "fixture", windowContext: context, isDialog: false))) + } +} diff --git a/docs/commands/see.md b/docs/commands/see.md index f44fb6386..4345cb88b 100644 --- a/docs/commands/see.md +++ b/docs/commands/see.md @@ -137,6 +137,10 @@ This fallback only runs inside the resolved window (it won’t hop between windo ## JSON output primer +Screen captures keep a display-scoped raster and separate application-scoped Accessibility evidence. The AX map +carries its own process generation; it does not make the screenshot an exact-window capture. Use an explicit +`--window-id` observation when a follow-up coordinate action needs a window-bound capture receipt. + When `--json` is supplied, the CLI prints: - `snapshot_id` – producer-bound `ps1_` reference for subsequent `click --snapshot …` and `type --snapshot …`.