From 342eac4b7bc26f0cf747f3a5c495480ae7b40e9f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 27 Aug 2026 01:40:32 -0700 Subject: [PATCH 1/2] fix(cli): honor verify unknown exit status --- Apps/CLI/CHANGELOG.md | 1 + .../Commands/AI/VerifyCommand.swift | 5 +++-- .../Commands/MCP/MCPToolCommandOutput.swift | 5 +++-- .../CoreCLITests/VerifyCommandTests.swift | 20 +++++++++++++++++++ CHANGELOG.md | 1 + 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/Apps/CLI/CHANGELOG.md b/Apps/CLI/CHANGELOG.md index 700588338..cb6362ecc 100644 --- a/Apps/CLI/CHANGELOG.md +++ b/Apps/CLI/CHANGELOG.md @@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Warm ScreenCaptureKit ownership validation off the main actor before Bridge socket/capability publication, with explicit publication and daemon-readiness reserves beyond the bounded scan. - Claim and generation-check the host's ScreenCaptureKit lease before trying the concurrent engine first for background Bridge full-screen automatic capture, preserving legacy fallback after modern failure and automatic fallback on every claim failure or competing owner. - Prevent agent-spawned exec children from retaining the global ScreenCaptureKit transaction lock after an interrupted capture owner exits. +- Return exit status 2 when `verify` cannot evaluate state because its underlying tool fails. - Report background text, editable special keys, and clears with their actual AXValue, event, or composite delivery; count only real key events as key presses; preserve the planned receiver literal after escape processing; and require protocol 1.36 before AX-capable remote type requests. - Revalidate exact-window focused elements and the application's internal key window before typing, reject parent targets with attached sheets while preserving independently identified exact sheet targets, confirm clear-plus-literal text only from a generation-bound value change after bounded event settlement, keep pixel-focus setup confirmation separate from its typing leaf, and stop reporting no-change, missing, or dispatched-but-unverified outcomes as typed characters. - Require explicit standalone CLI foreground consent for application focus/switch and Dock visibility changes, and reject contradictory app-switch selectors before runtime discovery. diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/VerifyCommand.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/VerifyCommand.swift index 3dedc27f4..b6e93364b 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/VerifyCommand.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/VerifyCommand.swift @@ -48,9 +48,10 @@ struct VerifyCommand: ErrorHandlingCommand, OutputFormattable, RuntimeBackedComm tool: tool.name, response: response, jsonOutput: self.jsonOutput, - logger: self.outputLogger + logger: self.outputLogger, + errorExitCode: ExitCode(2) ) - return + throw ExitCode(2) } let screenshotPath = try self.saveScreenshot(from: response) diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/MCP/MCPToolCommandOutput.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/MCP/MCPToolCommandOutput.swift index f29dec3a8..e100a8e06 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/MCP/MCPToolCommandOutput.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/MCP/MCPToolCommandOutput.swift @@ -108,7 +108,8 @@ enum MCPToolCommandOutput { tool: String, response: ToolResponse, jsonOutput: Bool, - logger: Logger + logger: Logger, + errorExitCode: ExitCode = ExitCode(1) ) throws { let payload = self.payload(tool: tool, response: response) if jsonOutput { @@ -123,7 +124,7 @@ enum MCPToolCommandOutput { } if response.isError { - throw ExitCode(1) + throw errorExitCode } } diff --git a/Apps/CLI/Tests/CoreCLITests/VerifyCommandTests.swift b/Apps/CLI/Tests/CoreCLITests/VerifyCommandTests.swift index 03424a0c2..ee967ed99 100644 --- a/Apps/CLI/Tests/CoreCLITests/VerifyCommandTests.swift +++ b/Apps/CLI/Tests/CoreCLITests/VerifyCommandTests.swift @@ -1,3 +1,6 @@ +import Commander +import PeekabooAgentRuntime +import PeekabooCore import Testing @testable import PeekabooCLI @@ -47,4 +50,21 @@ struct VerifyCommandTests { } #expect(category == .vision) } + + @Test + func `tool failure exits with unknown error status`() async throws { + var command = try VerifyCommand.parse([ + "--app", "Fixture", "--window-exists", "--screenshot", "/tmp/unused-verify-state.png", + ]) + let runtime = CommandRuntime( + configuration: .init(verbose: false, jsonOutput: true, logLevel: nil), + services: PeekabooServices(initializeAgentService: false), + toolCapturePreflightRefusal: MCPToolCapturePreflightRefusal(message: "fixture capture refusal") + ) + + let exitCode = await #expect(throws: ExitCode.self) { + try await command.run(using: runtime) + } + #expect(exitCode == ExitCode(2)) + } } diff --git a/CHANGELOG.md b/CHANGELOG.md index db6f0a0ee..0b277f29d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - Warm ScreenCaptureKit ownership validation off the main actor before Bridge socket/capability publication, with explicit publication and daemon-readiness reserves beyond the bounded scan. - Claim and generation-check the host's ScreenCaptureKit lease before trying the concurrent engine first for background Bridge full-screen automatic capture, preserving legacy fallback after modern failure and automatic fallback on every claim failure or competing owner. - Prevent agent-spawned exec children from retaining the global ScreenCaptureKit transaction lock after an interrupted capture owner exits. +- Return exit status 2 when `verify` cannot evaluate state because its underlying tool fails. - Report background text, editable special keys, and clears with their actual AXValue, event, or composite delivery; count only real key events as key presses; preserve the planned receiver literal after escape processing; and require protocol 1.36 before AX-capable remote type requests. - Revalidate exact-window focused elements and the application's internal key window before typing, reject parent targets with attached sheets while preserving independently identified exact sheet targets, confirm clear-plus-literal text only from a generation-bound value change after bounded event settlement, keep pixel-focus setup confirmation separate from its typing leaf, and stop reporting no-change, missing, or dispatched-but-unverified outcomes as typed characters. - Require process-generation receipts for process-scoped `action` and `set-value` snapshots, revalidate them before dispatch, and preserve their canonical target metadata through MCP and signed Bridge results. From 9e4d1c5f439665f9d6d48eb6b1839607e3c9505f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 27 Aug 2026 20:19:52 -0700 Subject: [PATCH 2/2] test(cli): isolate verify error regressions --- .../MCPToolCommandOutputTests.swift | 39 +++++++++++ .../CoreCLITests/VerifyCommandTests.swift | 66 +++++++++++++++++-- docs/commands/verify.md | 4 +- 3 files changed, 102 insertions(+), 7 deletions(-) diff --git a/Apps/CLI/Tests/CoreCLITests/MCPToolCommandOutputTests.swift b/Apps/CLI/Tests/CoreCLITests/MCPToolCommandOutputTests.swift index 3e85a80a3..bf7cbc47e 100644 --- a/Apps/CLI/Tests/CoreCLITests/MCPToolCommandOutputTests.swift +++ b/Apps/CLI/Tests/CoreCLITests/MCPToolCommandOutputTests.swift @@ -1,3 +1,4 @@ +import Commander import MCP import PeekabooFoundation import TachikomaMCP @@ -5,6 +6,44 @@ import Testing @testable import PeekabooCLI struct MCPToolCommandOutputTests { + @Test(arguments: [false, true]) + func `Tool errors use exit status one by default`(jsonOutput: Bool) { + let exitCode = #expect(throws: ExitCode.self) { + try MCPToolCommandOutput.output( + tool: "fixture", + response: .error("fixture failure"), + jsonOutput: jsonOutput, + logger: .shared + ) + } + #expect(exitCode == ExitCode(1)) + } + + @Test(arguments: [false, true]) + func `Tool errors honor an explicit exit status`(jsonOutput: Bool) { + let exitCode = #expect(throws: ExitCode.self) { + try MCPToolCommandOutput.output( + tool: "fixture", + response: .error("fixture failure"), + jsonOutput: jsonOutput, + logger: .shared, + errorExitCode: ExitCode(2) + ) + } + #expect(exitCode == ExitCode(2)) + } + + @Test(arguments: [false, true]) + func `Successful tool output does not throw`(jsonOutput: Bool) throws { + try MCPToolCommandOutput.output( + tool: "fixture", + response: .text("fixture success"), + jsonOutput: jsonOutput, + logger: .shared, + errorExitCode: ExitCode(2) + ) + } + @Test func `Browser CLI envelope projects canonical failure and exact target metadata`() throws { let outcome = DesktopActionOutcome.indeterminate( diff --git a/Apps/CLI/Tests/CoreCLITests/VerifyCommandTests.swift b/Apps/CLI/Tests/CoreCLITests/VerifyCommandTests.swift index ee967ed99..c76725146 100644 --- a/Apps/CLI/Tests/CoreCLITests/VerifyCommandTests.swift +++ b/Apps/CLI/Tests/CoreCLITests/VerifyCommandTests.swift @@ -1,5 +1,8 @@ import Commander +import Foundation import PeekabooAgentRuntime +import PeekabooAutomationKit +import PeekabooBridge import PeekabooCore import Testing @testable import PeekabooCLI @@ -51,20 +54,71 @@ struct VerifyCommandTests { #expect(category == .vision) } - @Test - func `tool failure exits with unknown error status`() async throws { + @Test(arguments: [false, true]) + func `tool failure exits with unknown error status`(jsonOutput: Bool) async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("verify-preflight-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: directory) } var command = try VerifyCommand.parse([ - "--app", "Fixture", "--window-exists", "--screenshot", "/tmp/unused-verify-state.png", + "--app", "Fixture", "--window-exists", "--screenshot", directory.appendingPathComponent("unused.png").path, ]) let runtime = CommandRuntime( - configuration: .init(verbose: false, jsonOutput: true, logLevel: nil), - services: PeekabooServices(initializeAgentService: false), - toolCapturePreflightRefusal: MCPToolCapturePreflightRefusal(message: "fixture capture refusal") + configuration: .init(verbose: false, jsonOutput: jsonOutput, logLevel: nil), + services: VerifyPreflightServices(directory: directory), + toolCapturePreflightRefusal: MCPToolCapturePreflightRefusal(message: "fixture capture refusal"), + interactionMutationTracker: InteractionMutationTracker( + desktopMutationWatermarkStore: DesktopMutationWatermarkStore(directoryURL: directory) + ) ) let exitCode = await #expect(throws: ExitCode.self) { try await command.run(using: runtime) } #expect(exitCode == ExitCode(2)) + #expect(!FileManager.default.fileExists(atPath: directory.appendingPathComponent("unused.png").path)) + } +} + +@MainActor +private final class VerifyPreflightServices: PeekabooServiceProviding { + let automation: any UIAutomationServiceProtocol = MockAutomationService() + let windows: any WindowManagementServiceProtocol = MockWindowService(result: []) + let menu: any MenuServiceProtocol = MockMenuService(barItems: []) + let dock: any DockServiceProtocol = MockDockService(items: []) + let snapshots: any SnapshotManagerProtocol = InMemorySnapshotManager() + let permissions = PermissionsService() + let screens: any ScreenServiceProtocol = ScreenService() + let clipboard: any ClipboardServiceProtocol = ClipboardService() + let agent: (any AgentServiceProtocol)? = nil + let screenCapture: any ScreenCaptureServiceProtocol + let applications: any ApplicationServiceProtocol + let dialogs: any DialogServiceProtocol + let browser: any BrowserMCPClientProviding + + init(directory: URL) { + // These adapters are inert until called; preflight must refuse before any Bridge request. + let client = PeekabooBridgeClient(socketPath: directory.appendingPathComponent("absent.sock").path) + self.screenCapture = RemoteScreenCaptureService(client: client) + self.applications = RemoteApplicationService(client: client) + self.dialogs = RemoteDialogService(client: client) + self.browser = RemoteBrowserMCPClient(client: client) + } + + var configuration: PeekabooCore.ConfigurationManager { + fatalError("Verification preflight must not load shared configuration") } + + var audioInput: AudioInputService { + fatalError("Verification preflight must not initialize AI providers") + } + + var logging: any LoggingServiceProtocol { + fatalError("Verification preflight uses only the CLI logger") + } + + var files: any FileServiceProtocol { + fatalError("Verification preflight must not access files") + } + + func ensureVisualizerConnection() {} } diff --git a/docs/commands/verify.md b/docs/commands/verify.md index 52c6dfab5..fcdb92ce3 100644 --- a/docs/commands/verify.md +++ b/docs/commands/verify.md @@ -9,7 +9,9 @@ read_when: `peekaboo verify` polls fresh native window and accessibility state until every requested predicate is stable or the timeout expires. It is the deterministic replacement for sleep-based polling: the command never focuses, clicks, types, or treats an incomplete observation as success. -Results are ternary. `satisfied` exits 0, `unsatisfied` exits 1, and `unknown` exits 2. JSON output includes every predicate result and an `unknown_reason` field; it is `null` when the result is not unknown. +Results are ternary. `satisfied` exits 0, `unsatisfied` exits 1, and `unknown` exits 2. Evaluated results in JSON output include every predicate result and an `unknown_reason` field; it is `null` when the result is not unknown. + +Tool failures that prevent evaluation also exit 2. These failures use the standard error envelope in JSON mode, without predicate results or an `unknown_reason` field. ## Key options