From 6b0b438c4679c0e7e9108cadbca20584b0988bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 15:55:05 +0200 Subject: [PATCH 1/6] fix(macos): reject --fullscreen on desktop and menubar surfaces desktop and menubar always capture the main display through the macOS helper, so an explicit --fullscreen on either named a frame the capture could never vary. It was parsed, ignored, and echoed back unchanged. Refuse it before any capture with a typed INVALID_ARGS reason instead. macOS app sessions (runner path) and every other platform keep accepting --fullscreen unchanged. Remove the now-dead --fullscreen plumbing: the helper's argument parse, its capture parameter, and the fullscreen response field; the TS helper argv and its return type. --- apple/macos-helper/Package.swift | 4 + .../Sources/AgentDeviceMacOSHelper/main.swift | 9 +- .../ScreenshotResponseTests.swift | 15 ++++ packages/contracts/src/facades/capture.ts | 2 + packages/contracts/src/screenshot.ts | 13 +++ .../screenshot-macos-surface.test.ts | 86 +++++++++++++++++++ packages/platform-apple/src/interactor.ts | 18 +++- .../src/os/macos/helper.test.ts | 26 +++++- .../platform-apple/src/os/macos/helper.ts | 6 +- src/commands/capture/screenshot.ts | 2 +- 10 files changed, 167 insertions(+), 14 deletions(-) create mode 100644 apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/ScreenshotResponseTests.swift create mode 100644 packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts diff --git a/apple/macos-helper/Package.swift b/apple/macos-helper/Package.swift index 3a472a3ca2..844c6cb2bb 100644 --- a/apple/macos-helper/Package.swift +++ b/apple/macos-helper/Package.swift @@ -22,5 +22,9 @@ let package = Package( name: "AgentDeviceMacOSInputTests", dependencies: ["AgentDeviceMacOSInput"] ), + .testTarget( + name: "AgentDeviceMacOSHelperTests", + dependencies: ["AgentDeviceMacOSHelper"] + ), ] ) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index cc9be1260a..1dc3445312 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -75,7 +75,6 @@ struct PressResponse: Encodable { struct ScreenshotResponse: Encodable { let path: String let surface: String? - let fullscreen: Bool } struct AgentDeviceMacOSHelper { @@ -436,9 +435,8 @@ struct AgentDeviceMacOSHelper { } let surface = optionValue(arguments: arguments, name: "--surface") - let fullscreen = arguments.contains("--fullscreen") - try captureSurfaceScreenshot(surface: surface, outPath: outPath, fullscreen: fullscreen) - return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface, fullscreen: fullscreen)) + try captureSurfaceScreenshot(surface: surface, outPath: outPath) + return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface)) } static func handleAudioProbe(arguments: [String]) throws -> any Encodable { @@ -547,8 +545,7 @@ private func pressAtPosition(_ request: MouseClickRequest) throws { } } -private func captureSurfaceScreenshot(surface: String?, outPath: String, fullscreen: Bool) throws { - _ = fullscreen +private func captureSurfaceScreenshot(surface: String?, outPath: String) throws { guard #available(macOS 15.2, *) else { throw HelperError.commandFailed( "screenshot on macOS desktop and menubar surfaces requires macOS 15.2 or newer" diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/ScreenshotResponseTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/ScreenshotResponseTests.swift new file mode 100644 index 0000000000..a5471c009f --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/ScreenshotResponseTests.swift @@ -0,0 +1,15 @@ +import XCTest + +@testable import AgentDeviceMacOSHelper + +/// `captureSurfaceScreenshot` always reads the main display, so the response describes only +/// what was actually captured: no field for an argument the helper does not read. +final class ScreenshotResponseTests: XCTestCase { + func testResponseNeverCarriesAFullscreenField() throws { + let response = ScreenshotResponse(path: "/tmp/out.png", surface: "desktop") + let data = try JSONEncoder().encode(response) + let decoded = try JSONSerialization.jsonObject(with: data) as? [String: Any] + + XCTAssertEqual(Set(decoded?.keys.map { $0 } ?? []), ["path", "surface"]) + } +} diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index 9b10f17115..a25a0c2258 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -4,6 +4,7 @@ export { SCREENSHOT_ACTION_FLAG_KEYS, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_CROP_REASONS, + SCREENSHOT_FULLSCREEN_REASONS, SCREENSHOT_SCALE_LIMITS, SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS, appendScreenshotScriptFlags, @@ -18,6 +19,7 @@ export { export type { ScreenshotCropReason, ScreenshotDispatchFlags, + ScreenshotFullscreenReason, ScreenshotPublicOptions, ScreenshotRequestFlags, ScreenshotRuntimeFlags, diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index a759c80073..6afa8f0b78 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -67,6 +67,19 @@ export const SCREENSHOT_CROP_REASONS = { export type ScreenshotCropReason = (typeof SCREENSHOT_CROP_REASONS)[keyof typeof SCREENSHOT_CROP_REASONS]; +/** + * Machine-readable `screenshot --fullscreen` refusal reason. macOS `desktop` and `menubar` + * surfaces capture through the helper, which always reads the main display; an explicit + * `--fullscreen` on one of them names a frame the capture cannot vary, so it is refused + * with this reason in `error.details.reason` rather than silently ignored. + */ +export const SCREENSHOT_FULLSCREEN_REASONS = { + macOsHelperSurfaceFixedFrame: 'SCREENSHOT_FULLSCREEN_MACOS_HELPER_SURFACE_FIXED_FRAME', +} as const; + +export type ScreenshotFullscreenReason = + (typeof SCREENSHOT_FULLSCREEN_REASONS)[keyof typeof SCREENSHOT_FULLSCREEN_REASONS]; + export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'out', 'overlayRefs', diff --git a/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts b/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts new file mode 100644 index 0000000000..5d69a09915 --- /dev/null +++ b/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts @@ -0,0 +1,86 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; + +vi.mock('../os/macos/helper.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + runMacOsScreenshotAction: vi.fn(async (outPath: string) => ({ path: outPath })), + }; +}); + +vi.mock('../core/screenshot.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + captureScreenshotViaRunner: vi.fn(), + screenshotIos: vi.fn(), + }; +}); + +import { createAppleInteractor } from '../interactor.ts'; +import { runMacOsScreenshotAction } from '../os/macos/helper.ts'; +import { screenshotIos } from '../core/screenshot.ts'; + +const macOsDevice: DeviceInfo = { + platform: 'apple', + appleOs: 'macos', + id: 'host-mac', + name: 'Host Mac', + kind: 'device', + target: 'desktop', + booted: true, +}; + +beforeEach(() => { + vi.mocked(runMacOsScreenshotAction).mockClear(); + vi.mocked(screenshotIos).mockClear(); +}); + +test.each(['desktop', 'menubar'] as const)( + 'refuses an explicit --fullscreen on the macOS %s surface before any capture', + async (surface) => { + const interactor = createAppleInteractor(macOsDevice, {}); + + await expect( + interactor.screenshot('/tmp/out.png', { surface, fullscreen: true }), + ).rejects.toMatchObject({ + code: 'INVALID_ARGS', + details: expect.objectContaining({ + reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, + surface, + }), + }); + + expect(runMacOsScreenshotAction).not.toHaveBeenCalled(); + }, +); + +test.each(['desktop', 'menubar'] as const)( + 'captures the %s surface through the helper when --fullscreen is not requested', + async (surface) => { + const interactor = createAppleInteractor(macOsDevice, {}); + + await interactor.screenshot('/tmp/out.png', { surface }); + + expect(runMacOsScreenshotAction).toHaveBeenCalledOnce(); + const [, options] = vi.mocked(runMacOsScreenshotAction).mock.calls[0]!; + expect(options).toEqual({ surface }); + expect(Object.hasOwn(options ?? {}, 'fullscreen')).toBe(false); + }, +); + +test('keeps a macOS app session on the runner path with --fullscreen unchanged', async () => { + const interactor = createAppleInteractor(macOsDevice, {}); + + await interactor.screenshot('/tmp/out.png', { surface: 'app', fullscreen: true }); + + expect(runMacOsScreenshotAction).not.toHaveBeenCalled(); + expect(screenshotIos).toHaveBeenCalledOnce(); + expect(screenshotIos).toHaveBeenCalledWith( + macOsDevice, + '/tmp/out.png', + expect.objectContaining({ fullscreen: true }), + ); +}); diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 313c797765..995f9890fb 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -13,6 +13,7 @@ import { type AppleRunnerProvider, } from './runner/index.ts'; import { toAppleTvRemoteButton } from '@agent-device/contracts/tv-remote'; +import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; import type { SessionSurface } from '@agent-device/contracts/session'; import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; import { normalizeSnapshotScope } from '@agent-device/contracts/snapshot'; @@ -386,9 +387,15 @@ async function runAppleScreenshot( runnerOpts: RunnerCallOptions, ): Promise { if (usesMacOsSurfaceScreenshot(device, options.surface)) { + if (options.fullscreen && rejectsMacOsHelperFullscreen(options.surface)) { + throw new AppError( + 'INVALID_ARGS', + `screenshot --fullscreen is not accepted on the macOS ${options.surface} surface: it always captures the main display`, + { reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, surface: options.surface }, + ); + } await runMacOsScreenshotAction(outPath, { surface: options.surface, - fullscreen: options.fullscreen, }); return; } @@ -421,6 +428,15 @@ function usesMacOsSurfaceScreenshot( return isMacOs(device) && surface !== undefined && surface !== 'app'; } +/** + * `desktop` and `menubar` always capture the main display through the helper; an explicit + * `--fullscreen` on either names a frame the capture cannot vary, so it is refused rather than + * silently ignored. + */ +function rejectsMacOsHelperFullscreen(surface: ScreenshotOptions['surface']): boolean { + return surface === 'desktop' || surface === 'menubar'; +} + /** Only non-app macOS surfaces are helper-read; an app session is runner-read like any leaf. */ function usesMacOsHelperSurface(device: DeviceInfo, surface: SessionSurface | undefined): boolean { return isMacOs(device) && surface !== undefined && surface !== 'app'; diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index eb8d24b0e9..483483d494 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,7 +1,12 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; -import { macOsClickScheduleMs, runMacOsPressAction, runMacOsSnapshotAction } from './helper.ts'; +import { + macOsClickScheduleMs, + runMacOsPressAction, + runMacOsScreenshotAction, + runMacOsSnapshotAction, +} from './helper.ts'; test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); @@ -207,3 +212,22 @@ test('macOS helper press stays a single held click when nothing is repeated', as assert.equal(receivedArgs.includes('--hold-ms'), false); assert.equal(receivedArgs.includes('--interval-ms'), false); }); + +test('macOS helper screenshot argv carries only --out and --surface', async () => { + let receivedArgs: string[] = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (args) => { + receivedArgs = args; + return helperReturn({ path: '/tmp/out.png', surface: 'desktop' }); + }, + }, + }); + + await withAppleToolProvider( + provider, + async () => await runMacOsScreenshotAction('/tmp/out.png', { surface: 'desktop' }), + ); + + assert.deepEqual(receivedArgs, ['screenshot', '--out', '/tmp/out.png', '--surface', 'desktop']); +}); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index a0bd002176..b807ab1589 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -473,16 +473,12 @@ export async function runMacOsPressAction( export async function runMacOsScreenshotAction( outPath: string, - options: { surface?: SessionSurface; fullscreen?: boolean } = {}, + options: { surface?: SessionSurface } = {}, ): Promise<{ path: string; surface?: SessionSurface; - fullscreen: boolean; }> { const args = ['screenshot', '--out', outPath]; appendMacOsHelperContextArgs(args, options); - if (options.fullscreen) { - args.push('--fullscreen'); - } return await runMacOsHelper(args); } diff --git a/src/commands/capture/screenshot.ts b/src/commands/capture/screenshot.ts index 67a86c38d3..e0687d2cc7 100644 --- a/src/commands/capture/screenshot.ts +++ b/src/commands/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops.', + 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. macOS --surface desktop and --surface menubar always capture the main display and refuse an explicit --fullscreen.', }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), From d58f087ca9a24dba6c8f0540e8acfa80bbaf8cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 16:06:42 +0200 Subject: [PATCH 2/6] test(macos): prove --fullscreen desktop refusal through the daemon router The provider-scenario suite still sent screenshotFullscreen:true on the desktop surface and asserted a --fullscreen helper argv, so it exercised the pre-fix contract and was the only test reaching the refusal through daemon -> bound screenshot runtime -> interactor. Add a direct refusal check (INVALID_ARGS, details.reason SCREENSHOT_FULLSCREEN_MACOS_HELPER_SURFACE_FIXED_FRAME, no helper call), strip screenshotFullscreen from the two desktop capture steps and their expected argv, and stop echoing the retired fullscreen field from the fake helper. Update the desktop-inspection recipe and add a prose note documenting that desktop/menubar refuse --fullscreen. Record the behavior change in the changelog, including its effect on recorded scripts and project config that set screenshotFullscreen for those surfaces. --- CHANGELOG.md | 7 ++++ packages/platform-apple/src/interactor.ts | 5 ++- .../provider-scenarios/macos-desktop.test.ts | 42 ++++++++++++++++--- .../provider-scenarios/macos-world.ts | 1 - website/docs/docs/commands.md | 3 +- 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 688e453617..f48d6bd5bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ the caller was told it had failed (#2546's late tap, from the banner side). Alert activation now opts out of XCTest's interruption handling, which also stops that handler from pressing a button of its own choosing on an unrelated system alert while the command answers the alert it resolved. +- Fixed (macos): `screenshot --fullscreen` on the `desktop` or `menubar` surface now refuses with + `INVALID_ARGS` (`details.reason: SCREENSHOT_FULLSCREEN_MACOS_HELPER_SURFACE_FIXED_FRAME`) instead + of being accepted and silently ignored. Both surfaces always capture the main display through the + macOS helper, so the flag never named a frame the capture could vary. A `.ad` script or project + config that sets `screenshotFullscreen` for a desktop/menubar capture now fails instead of + succeeding with the same image it always produced; drop the flag for those surfaces. macOS app + sessions and every other platform keep accepting `--fullscreen` unchanged. (#2799) - Fixed (ios): a local Simulator snapshot taken through the host AX bridge once again publishes the geometric `hittable` fact, so `is hittable` and a `hittable:` selector resolve the same controls on the bridge and the XCTest runner. The snapshot capability table has declared `hittable = diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 995f9890fb..30191d7443 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -391,7 +391,10 @@ async function runAppleScreenshot( throw new AppError( 'INVALID_ARGS', `screenshot --fullscreen is not accepted on the macOS ${options.surface} surface: it always captures the main display`, - { reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, surface: options.surface }, + { + reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, + surface: options.surface, + }, ); } await runMacOsScreenshotAction(outPath, { diff --git a/test/integration/provider-scenarios/macos-desktop.test.ts b/test/integration/provider-scenarios/macos-desktop.test.ts index 9df52ff9d6..06f9925ac9 100644 --- a/test/integration/provider-scenarios/macos-desktop.test.ts +++ b/test/integration/provider-scenarios/macos-desktop.test.ts @@ -1,7 +1,13 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import { test } from 'vitest'; -import { assertFlatToolCall, assertPngDimensions, assertPngFile } from './assertions.ts'; +import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; +import { + assertFlatToolCall, + assertPngDimensions, + assertPngFile, + assertRpcError, +} from './assertions.ts'; import { PROVIDER_SCENARIO_MACOS } from './fixtures.ts'; import { createProviderScenarioTempPath, withProviderScenarioResource } from './harness.ts'; import { createMacOsDesktopWorld } from './macos-world.ts'; @@ -304,6 +310,34 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help appBundleId: undefined, }, }, + ]); + + const helperScreenshotCallsBeforeRefusal = appleTool.calls.filter( + (call) => call[0] === 'macos-helper' && call[1] === 'screenshot', + ).length; + const fullscreenRefusal = await daemon.callCommand('screenshot', [], { + out: screenshotPath, + screenshotFullscreen: true, + }); + const refusalErrorData = assertRpcError( + fullscreenRefusal, + 'INVALID_ARGS', + /--fullscreen is not accepted on the macOS desktop surface/, + ); + const refusalDetails = refusalErrorData.details as Record | undefined; + assert.equal( + refusalDetails?.reason, + SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, + ); + assert.equal(refusalDetails?.surface, 'desktop'); + assert.equal( + appleTool.calls.filter((call) => call[0] === 'macos-helper' && call[1] === 'screenshot') + .length, + helperScreenshotCallsBeforeRefusal, + 'A refused --fullscreen desktop screenshot must not reach the macos-helper', + ); + + await runProviderScenario(daemon, [ { name: 'read desktop surface state', command: 'appstate', @@ -316,11 +350,10 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help }, }, { - name: 'capture fullscreen desktop screenshot', + name: 'capture desktop screenshot', command: 'screenshot', flags: { out: screenshotPath, - screenshotFullscreen: true, }, expectData: { path: screenshotPath }, assert: () => { @@ -332,7 +365,6 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help command: 'screenshot', flags: { out: scaledScreenshotPath, - screenshotFullscreen: true, screenshotScale: 0.5, }, expectData: { path: scaledScreenshotPath }, @@ -493,7 +525,6 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help screenshotPath, '--surface', 'desktop', - '--fullscreen', ]); assertFlatToolCall(appleTool.calls, [ 'macos-helper', @@ -502,7 +533,6 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help scaledScreenshotPath, '--surface', 'desktop', - '--fullscreen', ]); assertFlatToolCall(appleTool.calls, [ 'macos-helper', diff --git a/test/integration/provider-scenarios/macos-world.ts b/test/integration/provider-scenarios/macos-world.ts index 66338a228b..3fbb9897cc 100644 --- a/test/integration/provider-scenarios/macos-world.ts +++ b/test/integration/provider-scenarios/macos-world.ts @@ -181,7 +181,6 @@ function runScriptedMacOsHelper(args: readonly string[]): { return helperOk({ path: outPath, surface: args.includes('--surface') ? args[args.indexOf('--surface') + 1] : 'frontmost-app', - fullscreen: args.includes('--fullscreen'), }); } if (args[0] === 'press') { diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 5a675ccdc6..708d76b6a1 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -347,7 +347,7 @@ agent-device close agent-device open --platform macos --surface desktop agent-device snapshot -i agent-device is visible 'role="window" label="Notes"' -agent-device screenshot desktop.png --fullscreen +agent-device screenshot desktop.png agent-device close # Menubar / menu-extra inspection @@ -1040,6 +1040,7 @@ agent-device record stop # Stop active recording - Keep the scale default unset, or use `--scale 1`, when full-resolution screenshots are required for reusable pixel-diff baselines. - `screenshot --overlay-refs` captures a fresh full snapshot and burns visible `@eN` refs plus their target rectangles into the saved PNG. - `screenshot --crop-on ` captures a fresh full snapshot of the same screen and crops the saved PNG to the frame the selector resolves to. The crop is re-encoded, so byte-comparing it against an older crop of the same frame is unreliable; a crop whose pixels are all opaque is written as truecolor RGB, while one containing transparency keeps RGBA. The selector must resolve to exactly one framed node; the result carries a `warnings` entry when the frame is clipped to the image. Currently accepted on iOS simulators and Android emulators — every other target is refused before any device work, and the flag cannot be combined with `--overlay-refs` or `--fullscreen` because both move the captured frame away from the snapshot viewport the crop is measured against. +- `screenshot --fullscreen` on macOS applies only to app sessions. The `desktop` and `menubar` surfaces always capture the main display through the macOS helper, so an explicit `--fullscreen` on either names a frame the capture cannot vary; it is refused with `INVALID_ARGS` before any capture runs, rather than accepted and ignored. - `screenshot --normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for deterministic screenshot baselines; ordinary screenshots leave the simulator's current chrome visible. - `screenshot --scale --overlay-refs` writes a smaller image and draws refs for that final image size; avoid very small scales when text, icons, or labels need to remain readable. - `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Each input is decoded from its own bytes, so `--baseline` and a saved current image may be PNG or JPEG whatever their extension says. Most `agent-device screenshot` artifacts are PNG; a HarmonyOS capture is the JPEG its device serves, stored under the requested name. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. The earlier best-effort `ocr` and `nonTextDeltas` analyzers are retired; their optional result fields remain for source compatibility but are no longer emitted, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. From e77dbad16afd4dd8fd2df53afd43e6ea3ea97971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 18:32:40 +0200 Subject: [PATCH 3/6] fix(macos): derive --fullscreen refusal from the helper-routing predicate rejectsMacOsHelperFullscreen hand-listed desktop/menubar as a second, independently maintained copy of usesMacOsSurfaceScreenshot's routing condition, so it silently missed frontmost-app: that surface also captures through the macOS helper's fixed main-display frame but kept accepting and ignoring --fullscreen. Key the refusal directly off usesMacOsSurfaceScreenshot instead, so every surface it routes to the helper today or in the future refuses the flag, and delete the separate list. --- packages/contracts/src/screenshot.ts | 8 ++++---- packages/platform-apple/src/interactor.ts | 16 ++++++---------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index 6afa8f0b78..e3610f8a5d 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -68,10 +68,10 @@ export type ScreenshotCropReason = (typeof SCREENSHOT_CROP_REASONS)[keyof typeof SCREENSHOT_CROP_REASONS]; /** - * Machine-readable `screenshot --fullscreen` refusal reason. macOS `desktop` and `menubar` - * surfaces capture through the helper, which always reads the main display; an explicit - * `--fullscreen` on one of them names a frame the capture cannot vary, so it is refused - * with this reason in `error.details.reason` rather than silently ignored. + * Machine-readable `screenshot --fullscreen` refusal reason. Every macOS surface captured through + * the helper (every `--surface` other than `app`) always reads the main display; an explicit + * `--fullscreen` on one of them names a frame the capture cannot vary, so it is refused with this + * reason in `error.details.reason` rather than silently ignored. */ export const SCREENSHOT_FULLSCREEN_REASONS = { macOsHelperSurfaceFixedFrame: 'SCREENSHOT_FULLSCREEN_MACOS_HELPER_SURFACE_FIXED_FRAME', diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 30191d7443..945a32309f 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -387,7 +387,7 @@ async function runAppleScreenshot( runnerOpts: RunnerCallOptions, ): Promise { if (usesMacOsSurfaceScreenshot(device, options.surface)) { - if (options.fullscreen && rejectsMacOsHelperFullscreen(options.surface)) { + if (options.fullscreen) { throw new AppError( 'INVALID_ARGS', `screenshot --fullscreen is not accepted on the macOS ${options.surface} surface: it always captures the main display`, @@ -424,6 +424,11 @@ async function runAppleScreenshot( }); } +/** + * Every surface this admits captures through the macOS helper's fixed main-display frame, so + * `runAppleScreenshot` also keys its `--fullscreen` refusal directly off this predicate: whichever + * surface routes here cannot vary its captured frame, helper-routed today or added later. + */ function usesMacOsSurfaceScreenshot( device: DeviceInfo, surface: ScreenshotOptions['surface'], @@ -431,15 +436,6 @@ function usesMacOsSurfaceScreenshot( return isMacOs(device) && surface !== undefined && surface !== 'app'; } -/** - * `desktop` and `menubar` always capture the main display through the helper; an explicit - * `--fullscreen` on either names a frame the capture cannot vary, so it is refused rather than - * silently ignored. - */ -function rejectsMacOsHelperFullscreen(surface: ScreenshotOptions['surface']): boolean { - return surface === 'desktop' || surface === 'menubar'; -} - /** Only non-app macOS surfaces are helper-read; an app session is runner-read like any leaf. */ function usesMacOsHelperSurface(device: DeviceInfo, surface: SessionSurface | undefined): boolean { return isMacOs(device) && surface !== undefined && surface !== 'app'; From fb1c56ae683c656dbc539a9099bcfe2b8e4f466a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 18:32:45 +0200 Subject: [PATCH 4/6] test(macos): pin every helper-routed surface refuses --fullscreen Derive the covered surfaces from SESSION_SURFACES filtered to non-'app' (the routing predicate's own domain) instead of a hand list, so screenshot-macos-surface.test.ts now exercises frontmost-app alongside desktop/menubar and stays complete if a future surface is added. Add a matching frontmost-app refusal check to the provider-scenario daemon-router test, next to the existing desktop one. --- .../screenshot-macos-surface.test.ts | 10 +++++-- .../provider-scenarios/macos-desktop.test.ts | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts b/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts index 5d69a09915..3d41e5d0d1 100644 --- a/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts +++ b/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts @@ -1,6 +1,7 @@ import { beforeEach, expect, test, vi } from 'vitest'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; +import { SESSION_SURFACES } from '@agent-device/contracts/session'; vi.mock('../os/macos/helper.ts', async (importOriginal) => { const actual = await importOriginal(); @@ -38,7 +39,12 @@ beforeEach(() => { vi.mocked(screenshotIos).mockClear(); }); -test.each(['desktop', 'menubar'] as const)( +// The helper-routed domain, derived from the same condition `usesMacOsSurfaceScreenshot` applies +// (every session surface except `app`) rather than a hand-picked list — so adding a surface to +// `SESSION_SURFACES` extends this coverage automatically instead of silently falling outside it. +const helperRoutedSurfaces = SESSION_SURFACES.filter((surface) => surface !== 'app'); + +test.each(helperRoutedSurfaces)( 'refuses an explicit --fullscreen on the macOS %s surface before any capture', async (surface) => { const interactor = createAppleInteractor(macOsDevice, {}); @@ -57,7 +63,7 @@ test.each(['desktop', 'menubar'] as const)( }, ); -test.each(['desktop', 'menubar'] as const)( +test.each(helperRoutedSurfaces)( 'captures the %s surface through the helper when --fullscreen is not requested', async (surface) => { const interactor = createAppleInteractor(macOsDevice, {}); diff --git a/test/integration/provider-scenarios/macos-desktop.test.ts b/test/integration/provider-scenarios/macos-desktop.test.ts index 06f9925ac9..e982520fa5 100644 --- a/test/integration/provider-scenarios/macos-desktop.test.ts +++ b/test/integration/provider-scenarios/macos-desktop.test.ts @@ -298,6 +298,36 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help flags: { doubleTap: true }, expectData: { x: 116, y: 80, doubleTap: true }, }, + ]); + + const helperScreenshotCallsBeforeFrontmostRefusal = appleTool.calls.filter( + (call) => call[0] === 'macos-helper' && call[1] === 'screenshot', + ).length; + const frontmostFullscreenRefusal = await daemon.callCommand('screenshot', [], { + out: screenshotPath, + screenshotFullscreen: true, + }); + const frontmostRefusalErrorData = assertRpcError( + frontmostFullscreenRefusal, + 'INVALID_ARGS', + /--fullscreen is not accepted on the macOS frontmost-app surface/, + ); + const frontmostRefusalDetails = frontmostRefusalErrorData.details as + | Record + | undefined; + assert.equal( + frontmostRefusalDetails?.reason, + SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, + ); + assert.equal(frontmostRefusalDetails?.surface, 'frontmost-app'); + assert.equal( + appleTool.calls.filter((call) => call[0] === 'macos-helper' && call[1] === 'screenshot') + .length, + helperScreenshotCallsBeforeFrontmostRefusal, + 'A refused --fullscreen frontmost-app screenshot must not reach the macos-helper', + ); + + await runProviderScenario(daemon, [ { name: 'switch to desktop surface', command: 'open', From 7713b7d4ea3f9c5b4a26762a4ccbada306e7f4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 18:32:50 +0200 Subject: [PATCH 5/6] docs(macos): scope --fullscreen refusal wording to every helper-routed surface The screenshot command's cliDetail, the commands.md prose, and the Unreleased CHANGELOG entry named only desktop/menubar; widen them to match the fix now that frontmost-app also refuses. --- CHANGELOG.md | 15 ++++++++------- src/commands/capture/screenshot.ts | 2 +- website/docs/docs/commands.md | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f48d6bd5bf..2cdcb0b8eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,13 +17,14 @@ the caller was told it had failed (#2546's late tap, from the banner side). Alert activation now opts out of XCTest's interruption handling, which also stops that handler from pressing a button of its own choosing on an unrelated system alert while the command answers the alert it resolved. -- Fixed (macos): `screenshot --fullscreen` on the `desktop` or `menubar` surface now refuses with - `INVALID_ARGS` (`details.reason: SCREENSHOT_FULLSCREEN_MACOS_HELPER_SURFACE_FIXED_FRAME`) instead - of being accepted and silently ignored. Both surfaces always capture the main display through the - macOS helper, so the flag never named a frame the capture could vary. A `.ad` script or project - config that sets `screenshotFullscreen` for a desktop/menubar capture now fails instead of - succeeding with the same image it always produced; drop the flag for those surfaces. macOS app - sessions and every other platform keep accepting `--fullscreen` unchanged. (#2799) +- Fixed (macos): `screenshot --fullscreen` on the `desktop`, `menubar`, or `frontmost-app` surface + now refuses with `INVALID_ARGS` (`details.reason: + SCREENSHOT_FULLSCREEN_MACOS_HELPER_SURFACE_FIXED_FRAME`) instead of being accepted and silently + ignored. Every one of those surfaces captures through the macOS helper, which always reads the + main display, so the flag never named a frame the capture could vary. A `.ad` script or project + config that sets `screenshotFullscreen` for one of those surfaces now fails instead of succeeding + with the same image it always produced; drop the flag there. macOS app sessions and every other + platform keep accepting `--fullscreen` unchanged. (#2799) - Fixed (ios): a local Simulator snapshot taken through the host AX bridge once again publishes the geometric `hittable` fact, so `is hittable` and a `hittable:` selector resolve the same controls on the bridge and the XCTest runner. The snapshot capability table has declared `hittable = diff --git a/src/commands/capture/screenshot.ts b/src/commands/capture/screenshot.ts index e0687d2cc7..bb62815e6e 100644 --- a/src/commands/capture/screenshot.ts +++ b/src/commands/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. macOS --surface desktop and --surface menubar always capture the main display and refuse an explicit --fullscreen.', + 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. on macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen.', }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 708d76b6a1..3d4f3ed519 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1040,7 +1040,7 @@ agent-device record stop # Stop active recording - Keep the scale default unset, or use `--scale 1`, when full-resolution screenshots are required for reusable pixel-diff baselines. - `screenshot --overlay-refs` captures a fresh full snapshot and burns visible `@eN` refs plus their target rectangles into the saved PNG. - `screenshot --crop-on ` captures a fresh full snapshot of the same screen and crops the saved PNG to the frame the selector resolves to. The crop is re-encoded, so byte-comparing it against an older crop of the same frame is unreliable; a crop whose pixels are all opaque is written as truecolor RGB, while one containing transparency keeps RGBA. The selector must resolve to exactly one framed node; the result carries a `warnings` entry when the frame is clipped to the image. Currently accepted on iOS simulators and Android emulators — every other target is refused before any device work, and the flag cannot be combined with `--overlay-refs` or `--fullscreen` because both move the captured frame away from the snapshot viewport the crop is measured against. -- `screenshot --fullscreen` on macOS applies only to app sessions. The `desktop` and `menubar` surfaces always capture the main display through the macOS helper, so an explicit `--fullscreen` on either names a frame the capture cannot vary; it is refused with `INVALID_ARGS` before any capture runs, rather than accepted and ignored. +- `screenshot --fullscreen` on macOS applies only to app sessions. Every other `--surface` (`desktop`, `menubar`, `frontmost-app`) captures through the macOS helper, which always captures the main display, so an explicit `--fullscreen` on any of them names a frame the capture cannot vary; it is refused with `INVALID_ARGS` before any capture runs, rather than accepted and ignored. - `screenshot --normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for deterministic screenshot baselines; ordinary screenshots leave the simulator's current chrome visible. - `screenshot --scale --overlay-refs` writes a smaller image and draws refs for that final image size; avoid very small scales when text, icons, or labels need to remain readable. - `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Each input is decoded from its own bytes, so `--baseline` and a saved current image may be PNG or JPEG whatever their extension says. Most `agent-device screenshot` artifacts are PNG; a HarmonyOS capture is the JPEG its device serves, stored under the requested name. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. The earlier best-effort `ocr` and `nonTextDeltas` analyzers are retired; their optional result fields remain for source compatibility but are no longer emitted, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. From 477bcd120d9541f2e3ddf632f549748b14b97499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 19:37:53 +0200 Subject: [PATCH 6/6] docs(macos): capitalize the --fullscreen surface refusal sentence in screenshot help --- src/commands/capture/screenshot.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/capture/screenshot.ts b/src/commands/capture/screenshot.ts index bb62815e6e..283fc30659 100644 --- a/src/commands/capture/screenshot.ts +++ b/src/commands/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. on macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen.', + 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen.', }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input),