diff --git a/contracts/fixtures/screenshot-crop-geometry.json b/contracts/fixtures/screenshot-crop-geometry.json index 5fc2de762a..bb8db5e6df 100644 --- a/contracts/fixtures/screenshot-crop-geometry.json +++ b/contracts/fixtures/screenshot-crop-geometry.json @@ -17,6 +17,15 @@ "expectedProjection": { "x": 48, "y": 880, "width": 1110, "height": 156 }, "expectedIntersection": { "x": 48, "y": 880, "width": 1110, "height": 156 } }, + { + "name": "iOS physical 3x capture (iPhone 12): a fractional points rect scales by imgW/bounds.width and rounds", + "space": "viewport-points", + "bounds": { "x": 0, "y": 0, "width": 390, "height": 844 }, + "rect": { "x": 255.6667, "y": 73.6667, "width": 54.3333, "height": 14.3333 }, + "image": { "width": 1170, "height": 2532 }, + "expectedProjection": { "x": 767, "y": 221, "width": 163, "height": 43 }, + "expectedIntersection": { "x": 767, "y": 221, "width": 163, "height": 43 } + }, { "name": "Android: device-pixel rects project 1:1 with no bounds lookup", "space": "device-pixels", diff --git a/packages/capture-kit/src/snapshot-rect-projection.test.ts b/packages/capture-kit/src/snapshot-rect-projection.test.ts index 6278127803..b318ea96b3 100644 --- a/packages/capture-kit/src/snapshot-rect-projection.test.ts +++ b/packages/capture-kit/src/snapshot-rect-projection.test.ts @@ -14,7 +14,7 @@ import { } from './snapshot-rect-projection.ts'; // Golden geometry table for `screenshot --crop-on`: the numbers are the live -// evidence baseline (iOS simulator 402/1206, Android 1080, macOS 586/1172), +// evidence baseline (iOS simulator 402/1206, iOS physical 390/1170, Android 1080, macOS 586/1172), // so a projection-law change that moves any accepted cell's crop box turns this // red before the pixel-identity cross-check could ever notice. diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index a17438a9aa..ad3ba6ff71 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -74,7 +74,11 @@ export type ScreenshotOptions = { * rendered the pixels in, so a landscape value pairs with a landscape image; it is absent when the * owner has no rotation reading it can stand behind for this capture. */ -export type ScreenshotCaptureFacts = Readonly<{ displayRotation?: DeviceRotation }>; +export type ScreenshotCaptureFacts = Readonly<{ + displayRotation?: DeviceRotation; + /** Actual physical iOS capture route, including any fallback from CoreDevice to XCTest. */ + iosPhysicalCaptureBackend?: 'coredevice' | 'xctest'; +}>; export type ElementSelectorKey = 'id' | 'label' | 'text' | 'value'; diff --git a/packages/daemon/src/execution/__tests__/screenshot-crop-target.test.ts b/packages/daemon/src/execution/__tests__/screenshot-crop-target.test.ts index 0003a3c103..52bf1f8306 100644 --- a/packages/daemon/src/execution/__tests__/screenshot-crop-target.test.ts +++ b/packages/daemon/src/execution/__tests__/screenshot-crop-target.test.ts @@ -3,6 +3,7 @@ import { ANDROID_EMULATOR, IOS_DEVICE, IOS_SIMULATOR, + IPADOS_SIMULATOR, LINUX_DEVICE, MACOS_DEVICE, TVOS_SIMULATOR, @@ -28,6 +29,23 @@ type CropTargetDevice = Readonly<{ surface: SessionSurface | undefined; }>; +const IPADOS_DEVICE: DeviceInfo = { ...IPADOS_SIMULATOR, kind: 'device' }; +const IOS_DEVICE_COREDEVICE: DeviceInfo = { + ...IOS_DEVICE, + iosPhysicalDeviceBackend: 'coredevice', +}; +const IOS_DEVICE_XCTEST: DeviceInfo = { + ...IOS_DEVICE, + iosPhysicalDeviceBackend: 'xctest', +}; + +/** The cells with live pixel-identity evidence; every other cell stays rejected. */ +const ACCEPTED_CROP_TARGETS: readonly CropTargetDevice['target'][] = [ + 'ios-simulator', + 'android-emulator', + 'ios-physical', +]; + /** One device per matrix cell: the classifier's whole reachable output, named by its cell. */ const CROP_TARGET_DEVICES: readonly CropTargetDevice[] = [ { target: 'ios-simulator', device: IOS_SIMULATOR, surface: undefined }, @@ -35,6 +53,8 @@ const CROP_TARGET_DEVICES: readonly CropTargetDevice[] = [ { target: 'android-device', device: ANDROID_DEVICE, surface: undefined }, { target: 'macos-app-window', device: MACOS_DEVICE, surface: 'app' }, { target: 'ios-physical', device: IOS_DEVICE, surface: undefined }, + { target: 'ios-physical-xctest', device: IOS_DEVICE_XCTEST, surface: undefined }, + { target: 'ipados-physical', device: IPADOS_DEVICE, surface: undefined }, { target: 'macos-helper', device: MACOS_DEVICE, surface: 'desktop' }, { target: 'web', device: WEB_DESKTOP_DEVICE, surface: undefined }, { target: 'linux', device: LINUX_DEVICE, surface: undefined }, @@ -59,7 +79,7 @@ test('the classifier and the acceptance matrix agree one-to-one, and the accepte } expect(CROP_TARGET_DEVICES.map((row) => row.target)).toEqual(matrixTargets); for (const cell of SCREENSHOT_CROP_TARGET_CELLS) { - if (cell.target === 'ios-simulator' || cell.target === 'android-emulator') { + if (ACCEPTED_CROP_TARGETS.includes(cell.target)) { expect(cell.status).toBe('accepted'); } else { expect(cell).toEqual({ @@ -71,6 +91,66 @@ test('the classifier and the acceptance matrix agree one-to-one, and the accepte } }); +test('a physical iPad has its own cell, and an iPad simulator stays in the simulator cell', () => { + expect(classifyScreenshotCropTarget(IPADOS_DEVICE, undefined)).toBe('ipados-physical'); + expect(classifyScreenshotCropTarget(IPADOS_SIMULATOR, undefined)).toBe('ios-simulator'); +}); + +test('a physical iPhone on CoreDevice passes the policy', () => { + expect(classifyScreenshotCropTarget(IOS_DEVICE_COREDEVICE, undefined)).toBe('ios-physical'); + expect(() => + assertScreenshotCropPolicy({ + device: IOS_DEVICE_COREDEVICE, + surface: undefined, + cropOn: 'label="Continue"', + overlayRefs: false, + fullscreen: false, + }), + ).not.toThrow(); +}); + +test('a physical iPhone on the XCTest backend is refused with the pending-evidence reason, naming its own cell', () => { + expect(() => + assertScreenshotCropPolicy({ + device: IOS_DEVICE_XCTEST, + surface: undefined, + cropOn: 'label="Continue"', + overlayRefs: false, + fullscreen: false, + }), + ).toThrow( + expect.objectContaining({ + code: 'UNSUPPORTED_OPERATION', + message: 'screenshot --crop-on is not accepted on ios-physical-xctest targets', + details: { + reason: SCREENSHOT_CROP_REASONS.targetNotAccepted, + rejectionReason: SCREENSHOT_CROP_REASONS.pendingPixelIdentityEvidence, + }, + }), + ); +}); + +test('a physical iPad is refused with the pending-evidence reason, naming its own cell', () => { + expect(() => + assertScreenshotCropPolicy({ + device: IPADOS_DEVICE, + surface: undefined, + cropOn: 'label="Continue"', + overlayRefs: false, + fullscreen: false, + }), + ).toThrow( + expect.objectContaining({ + code: 'UNSUPPORTED_OPERATION', + message: 'screenshot --crop-on is not accepted on ipados-physical targets', + details: { + reason: SCREENSHOT_CROP_REASONS.targetNotAccepted, + rejectionReason: SCREENSHOT_CROP_REASONS.pendingPixelIdentityEvidence, + }, + }), + ); +}); + const MACOS_SURFACE_BACKENDS: Record = { app: 'xctest', 'frontmost-app': 'macos-helper', @@ -168,7 +248,9 @@ test('a matrix-rejected target is refused with the pending-evidence reason', () }); test('an accepted target with a valid selector passes the policy', () => { - expect(() => - assertScreenshotCropPolicy({ device: ANDROID_EMULATOR, surface: undefined, ...POLICY_BASE }), - ).not.toThrow(); + for (const device of [IOS_SIMULATOR, ANDROID_EMULATOR]) { + expect(() => + assertScreenshotCropPolicy({ device, surface: undefined, ...POLICY_BASE }), + ).not.toThrow(); + } }); diff --git a/packages/daemon/src/execution/screenshot-crop-target.ts b/packages/daemon/src/execution/screenshot-crop-target.ts index bf98152312..1e476e123c 100644 --- a/packages/daemon/src/execution/screenshot-crop-target.ts +++ b/packages/daemon/src/execution/screenshot-crop-target.ts @@ -1,3 +1,4 @@ +import type { ScreenshotCaptureFacts } from '@agent-device/contracts/screenshot-runtime'; import { SCREENSHOT_CROP_REASONS, type ScreenshotCropReason, @@ -19,6 +20,8 @@ type CropTarget = | 'android-device' | 'macos-app-window' | 'ios-physical' + | 'ios-physical-xctest' + | 'ipados-physical' | 'macos-helper' | 'web' | 'linux' @@ -46,7 +49,9 @@ export const SCREENSHOT_CROP_TARGET_CELLS: readonly CropTargetCell[] = [ { target: 'android-emulator', status: 'accepted' }, { target: 'android-device', status: 'rejected', rejectionReason: PENDING }, { target: 'macos-app-window', status: 'rejected', rejectionReason: PENDING }, - { target: 'ios-physical', status: 'rejected', rejectionReason: PENDING }, + { target: 'ios-physical', status: 'accepted' }, + { target: 'ios-physical-xctest', status: 'rejected', rejectionReason: PENDING }, + { target: 'ipados-physical', status: 'rejected', rejectionReason: PENDING }, { target: 'macos-helper', status: 'rejected', rejectionReason: PENDING }, { target: 'web', status: 'rejected', rejectionReason: PENDING }, { target: 'linux', status: 'rejected', rejectionReason: PENDING }, @@ -83,8 +88,12 @@ function classifyAppleCropTarget( const appleOs = resolveDeviceAppleOs(device); switch (appleOs) { case 'ios': + if (device.kind !== 'device') return 'ios-simulator'; + return device.iosPhysicalDeviceBackend === 'xctest' ? 'ios-physical-xctest' : 'ios-physical'; case 'ipados': - return device.kind === 'device' ? 'ios-physical' : 'ios-simulator'; + // Only a physical iPhone has crop evidence. An iPad's largest viewport root can be the app + // window rather than the screen, so its physical capture frame is not yet checked. + return device.kind === 'device' ? 'ipados-physical' : 'ios-simulator'; case 'tvos': return 'tvos'; case 'macos': @@ -147,3 +156,16 @@ export function assertScreenshotCropPolicy(params: { throw cropRefusal(target, cell?.status === 'rejected' ? cell.rejectionReason : undefined); } } + +/** Physical iPhone crop evidence applies to the image producer, not just the selected backend. */ +export function assertScreenshotCropCapture( + device: DeviceInfo, + facts: ScreenshotCaptureFacts, +): void { + if ( + classifyScreenshotCropTarget(device, undefined) === 'ios-physical' && + facts.iosPhysicalCaptureBackend !== 'coredevice' + ) { + throw cropRefusal('ios-physical-non-coredevice-capture', PENDING); + } +} diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index 9a7960a0a5..7c1d97abdb 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -5,7 +5,12 @@ import path from 'node:path'; import { expect, test } from 'vitest'; import { SCREENSHOT_CROP_REASONS } from '@agent-device/contracts/capture'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { ANDROID_EMULATOR, IOS_SIMULATOR, mkdtempForTestSync } from '@agent-device/test-kit'; +import { + ANDROID_EMULATOR, + IOS_DEVICE, + IOS_SIMULATOR, + mkdtempForTestSync, +} from '@agent-device/test-kit'; import { makeSession } from '../__tests__/support/session-factories.ts'; import type { GenericPlatformExecutionParams } from '../core/generic-leaf-execution.ts'; import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts'; @@ -428,3 +433,49 @@ test('the result record omits the display rotation when the capture reported non expect(await executeResult(resolved, session, req)).not.toHaveProperty('displayRotation'); }); + +for (const backend of ['coredevice', 'xctest', undefined] as const) { + for (const crop of [false, true]) { + test(`physical iPhone capture backend=${backend}, crop=${crop} enforces capture evidence`, async () => { + const fixture = screenshotRuntimeFixture({ + onCapture: ({ outPath }) => { + writeSolidPng(outPath); + return backend ? { iosPhysicalCaptureBackend: backend } : {}; + }, + snapshotResult: () => ({ nodes: CROP_TREE, backend: 'xctest', producer: 'apple-runner' }), + }); + const session = makeSession('iphone', { device: IOS_DEVICE }); + const outPath = path.join(mkdtempForTestSync('physical-crop'), 'screen.png'); + const req = screenshotRequest({ + positionals: [outPath], + flags: crop ? { screenshotCropOn: 'label="Save"' } : {}, + }); + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (crop && backend !== 'coredevice') { + await expect(executeResult(resolved, session, req)).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + message: + 'screenshot --crop-on is not accepted on ios-physical-non-coredevice-capture targets', + details: { + reason: SCREENSHOT_CROP_REASONS.targetNotAccepted, + rejectionReason: SCREENSHOT_CROP_REASONS.pendingPixelIdentityEvidence, + }, + }); + expect(fixture.captureSnapshot).not.toHaveBeenCalled(); + } else { + const result = await executeResult(resolved, session, req); + expect(result).toMatchObject(crop ? { width: 40, height: 20 } : { width: 100, height: 50 }); + expect(fixture.captureSnapshot).toHaveBeenCalledTimes(crop ? 1 : 0); + } + expect(fixture.captureScreenshot).toHaveBeenCalledTimes(1); + }); + } +} diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index e7d2ba054e..1df252ff60 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -30,7 +30,10 @@ import type { ResolvedGenericExecution, } from '../core/generic-leaf-execution.ts'; import { createReadonlyRuntimeSessionStore } from '../core/runtime-session.ts'; -import { assertScreenshotCropPolicy } from './screenshot-crop-target.ts'; +import { + assertScreenshotCropCapture, + assertScreenshotCropPolicy, +} from './screenshot-crop-target.ts'; import { buildScreenshotCropWarnings, cropScreenshotToSelector } from './screenshot-crop.ts'; import { annotateScreenshotWithRefs } from '@agent-device/capture-kit/screenshot-overlay'; import { @@ -373,6 +376,7 @@ function createBoundScreenshotBackend( execution: screenshotExecutionFromContext(dispatchContext), }); if (crop) { + assertScreenshotCropCapture(session.device, facts); const outcome = await cropScreenshotToSelector({ device: session.device, session, diff --git a/packages/platform-apple/src/core/__tests__/physical-device-control.test.ts b/packages/platform-apple/src/core/__tests__/physical-device-control.test.ts index ef2dbd2ec9..b0e49d719f 100644 --- a/packages/platform-apple/src/core/__tests__/physical-device-control.test.ts +++ b/packages/platform-apple/src/core/__tests__/physical-device-control.test.ts @@ -95,7 +95,7 @@ test('default interactor screenshots stay in-band without invoking devicectl', a const outPath = path.join(await mkdtempForTest('agent-device-xctest-screenshot-'), 'screen.png'); const toolCalls: string[][] = []; try { - await withAppleToolProvider( + const facts = await withAppleToolProvider( createLocalAppleToolProvider({ runCommand: async (_cmd, args) => { toolCalls.push(args); @@ -120,6 +120,7 @@ test('default interactor screenshots stay in-band without invoking devicectl', a ), ); + assert.deepEqual(facts, { iosPhysicalCaptureBackend: 'xctest' }); assert.equal(await fs.readFile(outPath, 'utf8'), 'png-bytes'); assert.equal( toolCalls.some((args) => args[0] === 'devicectl'), @@ -136,3 +137,41 @@ test('CoreDevice-only app inventory fails with targeted XCTest guidance', async /App inventory is unavailable on this XCTest-backed physical iOS device/, ); }); + +for (const fallback of [false, true]) { + test(`physical interactor reports the actual screenshot backend after fallback=${fallback}`, async () => { + const outPath = path.join(await mkdtempForTest('agent-device-capture-backend-'), 'screen.png'); + let runnerCalls = 0; + const facts = await withAppleToolProvider( + createLocalAppleToolProvider({ + runCommand: async (_cmd, args) => { + assert.deepEqual(args.slice(0, 4), ['devicectl', 'device', 'capture', 'screenshot']); + if (fallback) + return { exitCode: 1, stdout: '', stderr: "error: Unknown option '--device'" }; + await fs.writeFile(outPath, 'coredevice-png'); + return { exitCode: 0, stdout: '', stderr: '' }; + }, + }), + async () => + await withAppleRunnerProvider( + async () => { + runnerCalls += 1; + return { imageBase64: Buffer.from('runner-png').toString('base64') }; + }, + { deviceId: IOS_DEVICE.id }, + async () => + await createAppleInteractor( + IOS_DEVICE, + {}, + { + env: process.env, + cwd: process.cwd(), + }, + ).screenshot(outPath), + ), + ); + assert.deepEqual(facts, { iosPhysicalCaptureBackend: fallback ? 'xctest' : 'coredevice' }); + assert.equal(runnerCalls, fallback ? 1 : 0); + assert.equal(await fs.readFile(outPath, 'utf8'), fallback ? 'runner-png' : 'coredevice-png'); + }); +} diff --git a/packages/platform-apple/src/core/physical-device-control.ts b/packages/platform-apple/src/core/physical-device-control.ts index 763fbe4f32..9be1adc8b7 100644 --- a/packages/platform-apple/src/core/physical-device-control.ts +++ b/packages/platform-apple/src/core/physical-device-control.ts @@ -1,3 +1,4 @@ +import type { ScreenshotCaptureFacts } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { EnvMap } from '@agent-device/kernel/source-value'; import { AppError } from '@agent-device/kernel/errors'; @@ -61,7 +62,7 @@ export type IosPhysicalDeviceControl = IosPhysicalDeviceRunnerControl & { device: DeviceInfo, outPath: string, options: IosPhysicalDeviceScreenshotOptions, - ): Promise; + ): Promise; copyRunnerFile( device: DeviceInfo, remotePath: string, diff --git a/packages/platform-apple/src/core/physical-device-screenshot.ts b/packages/platform-apple/src/core/physical-device-screenshot.ts index 675904afd8..e1cc8f10a4 100644 --- a/packages/platform-apple/src/core/physical-device-screenshot.ts +++ b/packages/platform-apple/src/core/physical-device-screenshot.ts @@ -1,3 +1,4 @@ +import type { ScreenshotCaptureFacts } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { EnvMap } from '@agent-device/kernel/source-value'; import { AppError } from '@agent-device/kernel/errors'; @@ -22,7 +23,7 @@ export async function captureCoreDeviceScreenshot( device: DeviceInfo, outPath: string, options: IosPhysicalDeviceScreenshotOptions, -): Promise { +): Promise { if (!options.preferRunner) { try { await runIosDevicectl( @@ -32,7 +33,7 @@ export async function captureCoreDeviceScreenshot( deviceId: device.id, }, ); - return; + return { iosPhysicalCaptureBackend: 'coredevice' }; } catch (error) { if (!shouldFallbackToRunnerForIosScreenshot(error)) throw error; emitPhysicalScreenshotFallbackDiagnostic(device, error); @@ -40,19 +41,22 @@ export async function captureCoreDeviceScreenshot( } const result = await runRunnerScreenshot(device, options, false); - if (await writeInlineRunnerScreenshot(result, outPath)) return; + if (await writeInlineRunnerScreenshot(result, outPath)) + return { iosPhysicalCaptureBackend: 'xctest' }; const remoteFileName = readRunnerScreenshotPath(result); await copyCoreDeviceRunnerFile(device, remoteFileName, outPath, options.env); + return { iosPhysicalCaptureBackend: 'xctest' }; } export async function captureXctestDeviceScreenshot( device: DeviceInfo, outPath: string, options: IosPhysicalDeviceScreenshotOptions, -): Promise { +): Promise { const result = await runRunnerScreenshot(device, options, true); - if (await writeInlineRunnerScreenshot(result, outPath)) return; + if (await writeInlineRunnerScreenshot(result, outPath)) + return { iosPhysicalCaptureBackend: 'xctest' }; throw new AppError( 'COMMAND_FAILED', 'Failed to capture iOS screenshot: XCTest runner returned no inline image', diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index a612c59f3a..ba32e01d36 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -117,14 +117,13 @@ export async function screenshotIos( return await captureSimulatorScreenshotWithFallback(device, outPath, options); } - await resolveIosPhysicalDeviceControl(device).captureScreenshot(device, outPath, { + return await resolveIosPhysicalDeviceControl(device).captureScreenshot(device, outPath, { appBundleId: options.appBundleId, fullscreen: options.fullscreen, runnerOptions: options.runnerOptions, runRunnerCommand: runAppleRunnerCommand, env: options.env, }); - return {}; } export async function captureSimulatorScreenshotWithFallback( diff --git a/src/commands/capture/screenshot.ts b/src/commands/capture/screenshot.ts index a8149b92e9..ed6cc0c393 100644 --- a/src/commands/capture/screenshot.ts +++ b/src/commands/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/src/commands/schema/cli-help-command-usage.test.ts b/src/commands/schema/cli-help-command-usage.test.ts index ff4d7aaecd..a477702ee4 100644 --- a/src/commands/schema/cli-help-command-usage.test.ts +++ b/src/commands/schema/cli-help-command-usage.test.ts @@ -30,6 +30,10 @@ test('usageForCommand documents screenshot web aliases and stabilization flags', assert.match(help, /deterministic iOS simulator or Android chrome/); assert.match(help, /--crop-on /); assert.match(help, /crop the capture to the frame of the selector/); + assert.match( + help, + /currently iOS simulators, Android emulators, and CoreDevice physical iPhones/, + ); }); test('usageForCommand documents screenshot diff normalization', async () => { diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index c1c2b438e5..ac79457a85 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -985,7 +985,7 @@ agent-device record stop # Stop active recording - `--scale ` resizes both dimensions, from `0.01` to `1` (full resolution). Set `AGENT_DEVICE_SCREENSHOT_SCALE=0.3` or `screenshotScale` in config as a token-conscious default; an explicit `--scale` overrides it. Leave it unset or pass `--scale 1` for pixel-diff baselines. `--max-size` is refused everywhere (CLI, `.ad` scripts, Node options, config, `AGENT_DEVICE_SCREENSHOT_MAX_SIZE`) with guidance to use `--scale`. - ADB-backed Android screenshots capture immediately by default. Use `--normalize-status-bar` to set deterministic status-bar chrome and wait for transient UI to fade. In the Node client, `stabilize: true` also enables this wait. - `--overlay-refs` takes a fresh snapshot and burns visible `@eN` refs and their rectangles into the PNG. With `--scale`, refs are drawn at the final size, so avoid scales where labels become unreadable. In `--json` mode each ref also has a screenshot-space `center` for `press ` fallback. -- `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators and Android emulators, needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. +- `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. - On single-display iOS simulators and Android devices, the result may report `displayRotation` (`portrait`, `portrait-upside-down`, `landscape-left`, or `landscape-right`). It is absent when the device returns no reading in time, on other targets, and on multi-display devices such as foldables. The saved image is always upright.