Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@
the caller was told it had failed (#2546's late tap, from the banner side). Alert activation now
opts out of XCTest's interruption handling, which also stops that handler from pressing a button of
its own choosing on an unrelated system alert while the command answers the alert it resolved.
- Fixed (macos): `screenshot --fullscreen` on the `desktop`, `menubar`, or `frontmost-app` surface
now refuses with `INVALID_ARGS` (`details.reason:
SCREENSHOT_FULLSCREEN_MACOS_HELPER_SURFACE_FIXED_FRAME`) instead of being accepted and silently
ignored. Every one of those surfaces captures through the macOS helper, which always reads the
main display, so the flag never named a frame the capture could vary. A `.ad` script or project
config that sets `screenshotFullscreen` for one of those surfaces now fails instead of succeeding
with the same image it always produced; drop the flag there. macOS app sessions and every other
platform keep accepting `--fullscreen` unchanged. (#2799)
- Fixed (ios): a local Simulator snapshot taken through the host AX bridge once again publishes the
geometric `hittable` fact, so `is hittable` and a `hittable:` selector resolve the same controls on
the bridge and the XCTest runner. The snapshot capability table has declared `hittable =
Expand Down
4 changes: 4 additions & 0 deletions apple/macos-helper/Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,9 @@ let package = Package(
name: "AgentDeviceMacOSInputTests",
dependencies: ["AgentDeviceMacOSInput"]
),
.testTarget(
name: "AgentDeviceMacOSHelperTests",
dependencies: ["AgentDeviceMacOSHelper"]
),
]
)
9 changes: 3 additions & 6 deletions apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ struct PressResponse: Encodable {
struct ScreenshotResponse: Encodable {
let path: String
let surface: String?
let fullscreen: Bool
}

struct AgentDeviceMacOSHelper {
Expand Down Expand Up @@ -436,9 +435,8 @@ struct AgentDeviceMacOSHelper {
}

let surface = optionValue(arguments: arguments, name: "--surface")
let fullscreen = arguments.contains("--fullscreen")
try captureSurfaceScreenshot(surface: surface, outPath: outPath, fullscreen: fullscreen)
return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface, fullscreen: fullscreen))
try captureSurfaceScreenshot(surface: surface, outPath: outPath)
return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface))
}

static func handleAudioProbe(arguments: [String]) throws -> any Encodable {
Expand Down Expand Up @@ -547,8 +545,7 @@ private func pressAtPosition(_ request: MouseClickRequest) throws {
}
}

private func captureSurfaceScreenshot(surface: String?, outPath: String, fullscreen: Bool) throws {
_ = fullscreen
private func captureSurfaceScreenshot(surface: String?, outPath: String) throws {
guard #available(macOS 15.2, *) else {
throw HelperError.commandFailed(
"screenshot on macOS desktop and menubar surfaces requires macOS 15.2 or newer"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import XCTest

@testable import AgentDeviceMacOSHelper

/// `captureSurfaceScreenshot` always reads the main display, so the response describes only
/// what was actually captured: no field for an argument the helper does not read.
final class ScreenshotResponseTests: XCTestCase {
func testResponseNeverCarriesAFullscreenField() throws {
let response = ScreenshotResponse(path: "/tmp/out.png", surface: "desktop")
let data = try JSONEncoder().encode(response)
let decoded = try JSONSerialization.jsonObject(with: data) as? [String: Any]

XCTAssertEqual(Set(decoded?.keys.map { $0 } ?? []), ["path", "surface"])
}
}
2 changes: 2 additions & 0 deletions packages/contracts/src/facades/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ export {
SCREENSHOT_ACTION_FLAG_KEYS,
SCREENSHOT_COMMAND_FLAG_KEYS,
SCREENSHOT_CROP_REASONS,
SCREENSHOT_FULLSCREEN_REASONS,
SCREENSHOT_SCALE_LIMITS,
SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS,
appendScreenshotScriptFlags,
Expand All @@ -18,6 +19,7 @@ export {
export type {
ScreenshotCropReason,
ScreenshotDispatchFlags,
ScreenshotFullscreenReason,
ScreenshotPublicOptions,
ScreenshotRequestFlags,
ScreenshotRuntimeFlags,
Expand Down
13 changes: 13 additions & 0 deletions packages/contracts/src/screenshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,19 @@ export const SCREENSHOT_CROP_REASONS = {
export type ScreenshotCropReason =
(typeof SCREENSHOT_CROP_REASONS)[keyof typeof SCREENSHOT_CROP_REASONS];

/**
* Machine-readable `screenshot --fullscreen` refusal reason. Every macOS surface captured through
* the helper (every `--surface` other than `app`) always reads the main display; an explicit
* `--fullscreen` on one of them names a frame the capture cannot vary, so it is refused with this
* reason in `error.details.reason` rather than silently ignored.
*/
export const SCREENSHOT_FULLSCREEN_REASONS = {
macOsHelperSurfaceFixedFrame: 'SCREENSHOT_FULLSCREEN_MACOS_HELPER_SURFACE_FIXED_FRAME',
} as const;

export type ScreenshotFullscreenReason =
(typeof SCREENSHOT_FULLSCREEN_REASONS)[keyof typeof SCREENSHOT_FULLSCREEN_REASONS];

export const SCREENSHOT_COMMAND_FLAG_KEYS = [
'out',
'overlayRefs',
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import { beforeEach, expect, test, vi } from 'vitest';
import type { DeviceInfo } from '@agent-device/kernel/device';
import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture';
import { SESSION_SURFACES } from '@agent-device/contracts/session';

vi.mock('../os/macos/helper.ts', async (importOriginal) => {
const actual = await importOriginal<typeof import('../os/macos/helper.ts')>();
return {
...actual,
runMacOsScreenshotAction: vi.fn(async (outPath: string) => ({ path: outPath })),
};
});

vi.mock('../core/screenshot.ts', async (importOriginal) => {
const actual = await importOriginal<typeof import('../core/screenshot.ts')>();
return {
...actual,
captureScreenshotViaRunner: vi.fn(),
screenshotIos: vi.fn(),
};
});

import { createAppleInteractor } from '../interactor.ts';
import { runMacOsScreenshotAction } from '../os/macos/helper.ts';
import { screenshotIos } from '../core/screenshot.ts';

const macOsDevice: DeviceInfo = {
platform: 'apple',
appleOs: 'macos',
id: 'host-mac',
name: 'Host Mac',
kind: 'device',
target: 'desktop',
booted: true,
};

beforeEach(() => {
vi.mocked(runMacOsScreenshotAction).mockClear();
vi.mocked(screenshotIos).mockClear();
});

// The helper-routed domain, derived from the same condition `usesMacOsSurfaceScreenshot` applies
// (every session surface except `app`) rather than a hand-picked list — so adding a surface to
// `SESSION_SURFACES` extends this coverage automatically instead of silently falling outside it.
const helperRoutedSurfaces = SESSION_SURFACES.filter((surface) => surface !== 'app');

test.each(helperRoutedSurfaces)(
'refuses an explicit --fullscreen on the macOS %s surface before any capture',
async (surface) => {
const interactor = createAppleInteractor(macOsDevice, {});

await expect(
interactor.screenshot('/tmp/out.png', { surface, fullscreen: true }),
).rejects.toMatchObject({
code: 'INVALID_ARGS',
details: expect.objectContaining({
reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame,
surface,
}),
});

expect(runMacOsScreenshotAction).not.toHaveBeenCalled();
},
);

test.each(helperRoutedSurfaces)(
'captures the %s surface through the helper when --fullscreen is not requested',
async (surface) => {
const interactor = createAppleInteractor(macOsDevice, {});

await interactor.screenshot('/tmp/out.png', { surface });

expect(runMacOsScreenshotAction).toHaveBeenCalledOnce();
const [, options] = vi.mocked(runMacOsScreenshotAction).mock.calls[0]!;
expect(options).toEqual({ surface });
expect(Object.hasOwn(options ?? {}, 'fullscreen')).toBe(false);
},
);

test('keeps a macOS app session on the runner path with --fullscreen unchanged', async () => {
const interactor = createAppleInteractor(macOsDevice, {});

await interactor.screenshot('/tmp/out.png', { surface: 'app', fullscreen: true });

expect(runMacOsScreenshotAction).not.toHaveBeenCalled();
expect(screenshotIos).toHaveBeenCalledOnce();
expect(screenshotIos).toHaveBeenCalledWith(
macOsDevice,
'/tmp/out.png',
expect.objectContaining({ fullscreen: true }),
);
});
17 changes: 16 additions & 1 deletion packages/platform-apple/src/interactor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
type AppleRunnerProvider,
} from './runner/index.ts';
import { toAppleTvRemoteButton } from '@agent-device/contracts/tv-remote';
import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture';
import type { SessionSurface } from '@agent-device/contracts/session';
import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device';
import { normalizeSnapshotScope } from '@agent-device/contracts/snapshot';
Expand Down Expand Up @@ -386,9 +387,18 @@ async function runAppleScreenshot(
runnerOpts: RunnerCallOptions,
): Promise<void> {
if (usesMacOsSurfaceScreenshot(device, options.surface)) {
if (options.fullscreen) {
throw new AppError(
'INVALID_ARGS',
`screenshot --fullscreen is not accepted on the macOS ${options.surface} surface: it always captures the main display`,
{
reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame,
surface: options.surface,
},
);
}
await runMacOsScreenshotAction(outPath, {
surface: options.surface,
fullscreen: options.fullscreen,
});
return;
}
Expand All @@ -414,6 +424,11 @@ async function runAppleScreenshot(
});
}

/**
* Every surface this admits captures through the macOS helper's fixed main-display frame, so
* `runAppleScreenshot` also keys its `--fullscreen` refusal directly off this predicate: whichever
* surface routes here cannot vary its captured frame, helper-routed today or added later.
*/
function usesMacOsSurfaceScreenshot(
device: DeviceInfo,
surface: ScreenshotOptions['surface'],
Expand Down
26 changes: 25 additions & 1 deletion packages/platform-apple/src/os/macos/helper.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts';
import { macOsClickScheduleMs, runMacOsPressAction, runMacOsSnapshotAction } from './helper.ts';
import {
macOsClickScheduleMs,
runMacOsPressAction,
runMacOsScreenshotAction,
runMacOsSnapshotAction,
} from './helper.ts';

test('macOS helper snapshot passes cancellation to the helper process', async () => {
const controller = new AbortController();
Expand Down Expand Up @@ -207,3 +212,22 @@ test('macOS helper press stays a single held click when nothing is repeated', as
assert.equal(receivedArgs.includes('--hold-ms'), false);
assert.equal(receivedArgs.includes('--interval-ms'), false);
});

test('macOS helper screenshot argv carries only --out and --surface', async () => {
let receivedArgs: string[] = [];
const provider = createLocalAppleToolProvider({
macosHelper: {
run: async (args) => {
receivedArgs = args;
return helperReturn({ path: '/tmp/out.png', surface: 'desktop' });
},
},
});

await withAppleToolProvider(
provider,
async () => await runMacOsScreenshotAction('/tmp/out.png', { surface: 'desktop' }),
);

assert.deepEqual(receivedArgs, ['screenshot', '--out', '/tmp/out.png', '--surface', 'desktop']);
});
6 changes: 1 addition & 5 deletions packages/platform-apple/src/os/macos/helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -473,16 +473,12 @@ export async function runMacOsPressAction(

export async function runMacOsScreenshotAction(
outPath: string,
options: { surface?: SessionSurface; fullscreen?: boolean } = {},
options: { surface?: SessionSurface } = {},
): Promise<{
path: string;
surface?: SessionSurface;
fullscreen: boolean;
}> {
const args = ['screenshot', '--out', outPath];
appendMacOsHelperContextArgs(args, options);
if (options.fullscreen) {
args.push('--fullscreen');
}
return await runMacOsHelper(args);
}
2 changes: 1 addition & 1 deletion src/commands/capture/screenshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({
text: {
summary: 'Capture a screenshot',
cliDetail:
'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on <selector> to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops.',
'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on <selector> to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen.',
},
metadata: screenshotCommandMetadata,
run: (client, input) => client.capture.screenshot(input),
Expand Down
Loading
Loading