feat(screenshot): support selector crops on CoreDevice iPhones

contracts/fixtures/screenshot-crop-geometry.json

@@ -17,6 +17,15 @@
     "expectedProjection": { "x": 48, "y": 880, "width": 1110, "height": 156 },
     "expectedIntersection": { "x": 48, "y": 880, "width": 1110, "height": 156 }
   },
+  {
+    "name": "iOS physical 3x capture (iPhone 12): a fractional points rect scales by imgW/bounds.width and rounds",
+    "space": "viewport-points",
+    "bounds": { "x": 0, "y": 0, "width": 390, "height": 844 },
+    "rect": { "x": 255.6667, "y": 73.6667, "width": 54.3333, "height": 14.3333 },
+    "image": { "width": 1170, "height": 2532 },
+    "expectedProjection": { "x": 767, "y": 221, "width": 163, "height": 43 },
+    "expectedIntersection": { "x": 767, "y": 221, "width": 163, "height": 43 }
+  },
   {
     "name": "Android: device-pixel rects project 1:1 with no bounds lookup",
     "space": "device-pixels",

packages/capture-kit/src/snapshot-rect-projection.test.ts

@@ -14,7 +14,7 @@ import {
 } from './snapshot-rect-projection.ts';
 
 // Golden geometry table for `screenshot --crop-on`: the numbers are the live
-// evidence baseline (iOS simulator 402/1206, Android 1080, macOS 586/1172),
+// evidence baseline (iOS simulator 402/1206, iOS physical 390/1170, Android 1080, macOS 586/1172),
 // so a projection-law change that moves any accepted cell's crop box turns this
 // red before the pixel-identity cross-check could ever notice.
 

packages/contracts/src/interactor-types.ts

@@ -74,7 +74,11 @@ export type ScreenshotOptions = {
  * rendered the pixels in, so a landscape value pairs with a landscape image; it is absent when the
  * owner has no rotation reading it can stand behind for this capture.
  */
-export type ScreenshotCaptureFacts = Readonly<{ displayRotation?: DeviceRotation }>;
+export type ScreenshotCaptureFacts = Readonly<{
+  displayRotation?: DeviceRotation;
+  /** Actual physical iOS capture route, including any fallback from CoreDevice to XCTest. */
+  iosPhysicalCaptureBackend?: 'coredevice' | 'xctest';
+}>;
 
 export type ElementSelectorKey = 'id' | 'label' | 'text' | 'value';
 

packages/daemon/src/execution/__tests__/screenshot-crop-target.test.ts

@@ -3,6 +3,7 @@ import {
   ANDROID_EMULATOR,
   IOS_DEVICE,
   IOS_SIMULATOR,
+  IPADOS_SIMULATOR,
   LINUX_DEVICE,
   MACOS_DEVICE,
   TVOS_SIMULATOR,
@@ -28,13 +29,32 @@ type CropTargetDevice = Readonly<{
   surface: SessionSurface | undefined;
 }>;
 
+const IPADOS_DEVICE: DeviceInfo = { ...IPADOS_SIMULATOR, kind: 'device' };
+const IOS_DEVICE_COREDEVICE: DeviceInfo = {
+  ...IOS_DEVICE,
+  iosPhysicalDeviceBackend: 'coredevice',
+};
+const IOS_DEVICE_XCTEST: DeviceInfo = {
+  ...IOS_DEVICE,
+  iosPhysicalDeviceBackend: 'xctest',
+};
+
+/** The cells with live pixel-identity evidence; every other cell stays rejected. */
+const ACCEPTED_CROP_TARGETS: readonly CropTargetDevice['target'][] = [
+  'ios-simulator',
+  'android-emulator',
+  'ios-physical',
+];
+
 /** One device per matrix cell: the classifier's whole reachable output, named by its cell. */
 const CROP_TARGET_DEVICES: readonly CropTargetDevice[] = [
   { target: 'ios-simulator', device: IOS_SIMULATOR, surface: undefined },
   { target: 'android-emulator', device: ANDROID_EMULATOR, surface: undefined },
   { target: 'android-device', device: ANDROID_DEVICE, surface: undefined },
   { target: 'macos-app-window', device: MACOS_DEVICE, surface: 'app' },
   { target: 'ios-physical', device: IOS_DEVICE, surface: undefined },
+  { target: 'ios-physical-xctest', device: IOS_DEVICE_XCTEST, surface: undefined },
+  { target: 'ipados-physical', device: IPADOS_DEVICE, surface: undefined },
   { target: 'macos-helper', device: MACOS_DEVICE, surface: 'desktop' },
   { target: 'web', device: WEB_DESKTOP_DEVICE, surface: undefined },
   { target: 'linux', device: LINUX_DEVICE, surface: undefined },
@@ -59,7 +79,7 @@ test('the classifier and the acceptance matrix agree one-to-one, and the accepte
   }
   expect(CROP_TARGET_DEVICES.map((row) => row.target)).toEqual(matrixTargets);
   for (const cell of SCREENSHOT_CROP_TARGET_CELLS) {
-    if (cell.target === 'ios-simulator' || cell.target === 'android-emulator') {
+    if (ACCEPTED_CROP_TARGETS.includes(cell.target)) {
       expect(cell.status).toBe('accepted');
     } else {
       expect(cell).toEqual({
@@ -71,6 +91,66 @@ test('the classifier and the acceptance matrix agree one-to-one, and the accepte
   }
 });
 
+test('a physical iPad has its own cell, and an iPad simulator stays in the simulator cell', () => {
+  expect(classifyScreenshotCropTarget(IPADOS_DEVICE, undefined)).toBe('ipados-physical');
+  expect(classifyScreenshotCropTarget(IPADOS_SIMULATOR, undefined)).toBe('ios-simulator');
+});
+
+test('a physical iPhone on CoreDevice passes the policy', () => {
+  expect(classifyScreenshotCropTarget(IOS_DEVICE_COREDEVICE, undefined)).toBe('ios-physical');
+  expect(() =>
+    assertScreenshotCropPolicy({
+      device: IOS_DEVICE_COREDEVICE,
+      surface: undefined,
+      cropOn: 'label="Continue"',
+      overlayRefs: false,
+      fullscreen: false,
+    }),
+  ).not.toThrow();
+});
+
+test('a physical iPhone on the XCTest backend is refused with the pending-evidence reason, naming its own cell', () => {
+  expect(() =>
+    assertScreenshotCropPolicy({
+      device: IOS_DEVICE_XCTEST,
+      surface: undefined,
+      cropOn: 'label="Continue"',
+      overlayRefs: false,
+      fullscreen: false,
+    }),
+  ).toThrow(
+    expect.objectContaining({
+      code: 'UNSUPPORTED_OPERATION',
+      message: 'screenshot --crop-on is not accepted on ios-physical-xctest targets',
+      details: {
+        reason: SCREENSHOT_CROP_REASONS.targetNotAccepted,
+        rejectionReason: SCREENSHOT_CROP_REASONS.pendingPixelIdentityEvidence,
+      },
+    }),
+  );
+});
+
+test('a physical iPad is refused with the pending-evidence reason, naming its own cell', () => {
+  expect(() =>
+    assertScreenshotCropPolicy({
+      device: IPADOS_DEVICE,
+      surface: undefined,
+      cropOn: 'label="Continue"',
+      overlayRefs: false,
+      fullscreen: false,
+    }),
+  ).toThrow(
+    expect.objectContaining({
+      code: 'UNSUPPORTED_OPERATION',
+      message: 'screenshot --crop-on is not accepted on ipados-physical targets',
+      details: {
+        reason: SCREENSHOT_CROP_REASONS.targetNotAccepted,
+        rejectionReason: SCREENSHOT_CROP_REASONS.pendingPixelIdentityEvidence,
+      },
+    }),
+  );
+});
+
 const MACOS_SURFACE_BACKENDS: Record<SessionSurface, MacOsSurfaceBackend> = {
   app: 'xctest',
   'frontmost-app': 'macos-helper',
@@ -168,7 +248,9 @@ test('a matrix-rejected target is refused with the pending-evidence reason', ()
 });
 
 test('an accepted target with a valid selector passes the policy', () => {
-  expect(() =>
-    assertScreenshotCropPolicy({ device: ANDROID_EMULATOR, surface: undefined, ...POLICY_BASE }),
-  ).not.toThrow();
+  for (const device of [IOS_SIMULATOR, ANDROID_EMULATOR]) {
+    expect(() =>
+      assertScreenshotCropPolicy({ device, surface: undefined, ...POLICY_BASE }),
+    ).not.toThrow();
+  }
 });

packages/daemon/src/execution/screenshot-crop-target.ts

@@ -1,3 +1,4 @@
+import type { ScreenshotCaptureFacts } from '@agent-device/contracts/screenshot-runtime';
 import {
   SCREENSHOT_CROP_REASONS,
   type ScreenshotCropReason,
@@ -19,6 +20,8 @@ type CropTarget =
   | 'android-device'
   | 'macos-app-window'
   | 'ios-physical'
+  | 'ios-physical-xctest'
+  | 'ipados-physical'
   | 'macos-helper'
   | 'web'
   | 'linux'
@@ -46,7 +49,9 @@ export const SCREENSHOT_CROP_TARGET_CELLS: readonly CropTargetCell[] = [
   { target: 'android-emulator', status: 'accepted' },
   { target: 'android-device', status: 'rejected', rejectionReason: PENDING },
   { target: 'macos-app-window', status: 'rejected', rejectionReason: PENDING },
-  { target: 'ios-physical', status: 'rejected', rejectionReason: PENDING },
+  { target: 'ios-physical', status: 'accepted' },
+  { target: 'ios-physical-xctest', status: 'rejected', rejectionReason: PENDING },
+  { target: 'ipados-physical', status: 'rejected', rejectionReason: PENDING },
   { target: 'macos-helper', status: 'rejected', rejectionReason: PENDING },
   { target: 'web', status: 'rejected', rejectionReason: PENDING },
   { target: 'linux', status: 'rejected', rejectionReason: PENDING },
@@ -83,8 +88,12 @@ function classifyAppleCropTarget(
   const appleOs = resolveDeviceAppleOs(device);
   switch (appleOs) {
     case 'ios':
+      if (device.kind !== 'device') return 'ios-simulator';
+      return device.iosPhysicalDeviceBackend === 'xctest' ? 'ios-physical-xctest' : 'ios-physical';
     case 'ipados':
-      return device.kind === 'device' ? 'ios-physical' : 'ios-simulator';
+      // Only a physical iPhone has crop evidence. An iPad's largest viewport root can be the app
+      // window rather than the screen, so its physical capture frame is not yet checked.
+      return device.kind === 'device' ? 'ipados-physical' : 'ios-simulator';
     case 'tvos':
       return 'tvos';
     case 'macos':
@@ -147,3 +156,16 @@ export function assertScreenshotCropPolicy(params: {
     throw cropRefusal(target, cell?.status === 'rejected' ? cell.rejectionReason : undefined);
   }
 }
+
+/** Physical iPhone crop evidence applies to the image producer, not just the selected backend. */
+export function assertScreenshotCropCapture(
+  device: DeviceInfo,
+  facts: ScreenshotCaptureFacts,
+): void {
+  if (
+    classifyScreenshotCropTarget(device, undefined) === 'ios-physical' &&
+    facts.iosPhysicalCaptureBackend !== 'coredevice'
+  ) {
+    throw cropRefusal('ios-physical-non-coredevice-capture', PENDING);
+  }
+}

packages/daemon/src/execution/screenshot-runtime-execution.test.ts

@@ -5,7 +5,12 @@ import path from 'node:path';
 import { expect, test } from 'vitest';
 import { SCREENSHOT_CROP_REASONS } from '@agent-device/contracts/capture';
 import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
-import { ANDROID_EMULATOR, IOS_SIMULATOR, mkdtempForTestSync } from '@agent-device/test-kit';
+import {
+  ANDROID_EMULATOR,
+  IOS_DEVICE,
+  IOS_SIMULATOR,
+  mkdtempForTestSync,
+} from '@agent-device/test-kit';
 import { makeSession } from '../__tests__/support/session-factories.ts';
 import type { GenericPlatformExecutionParams } from '../core/generic-leaf-execution.ts';
 import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts';
@@ -428,3 +433,49 @@ test('the result record omits the display rotation when the capture reported non
 
   expect(await executeResult(resolved, session, req)).not.toHaveProperty('displayRotation');
 });
+
+for (const backend of ['coredevice', 'xctest', undefined] as const) {
+  for (const crop of [false, true]) {
+    test(`physical iPhone capture backend=${backend}, crop=${crop} enforces capture evidence`, async () => {
+      const fixture = screenshotRuntimeFixture({
+        onCapture: ({ outPath }) => {
+          writeSolidPng(outPath);
+          return backend ? { iosPhysicalCaptureBackend: backend } : {};
+        },
+        snapshotResult: () => ({ nodes: CROP_TREE, backend: 'xctest', producer: 'apple-runner' }),
+      });
+      const session = makeSession('iphone', { device: IOS_DEVICE });
+      const outPath = path.join(mkdtempForTestSync('physical-crop'), 'screen.png');
+      const req = screenshotRequest({
+        positionals: [outPath],
+        flags: crop ? { screenshotCropOn: 'label="Save"' } : {},
+      });
+      const resolved = await resolveScreenshotGenericExecution({
+        hostCwd: process.cwd(),
+        createCommandSurface: testCreateCommandSurface,
+        platformServices: daemonPlatformServicesFixture(),
+        req,
+        session,
+        inspectFacts: fixture.inspectFacts,
+        bindDevice: fixture.bindDevice,
+      });
+      if (crop && backend !== 'coredevice') {
+        await expect(executeResult(resolved, session, req)).rejects.toMatchObject({
+          code: 'UNSUPPORTED_OPERATION',
+          message:
+            'screenshot --crop-on is not accepted on ios-physical-non-coredevice-capture targets',
+          details: {
+            reason: SCREENSHOT_CROP_REASONS.targetNotAccepted,
+            rejectionReason: SCREENSHOT_CROP_REASONS.pendingPixelIdentityEvidence,
+          },
+        });
+        expect(fixture.captureSnapshot).not.toHaveBeenCalled();
+      } else {
+        const result = await executeResult(resolved, session, req);
+        expect(result).toMatchObject(crop ? { width: 40, height: 20 } : { width: 100, height: 50 });
+        expect(fixture.captureSnapshot).toHaveBeenCalledTimes(crop ? 1 : 0);
+      }
+      expect(fixture.captureScreenshot).toHaveBeenCalledTimes(1);
+    });
+  }
+}

packages/daemon/src/execution/screenshot-runtime.ts

@@ -30,7 +30,10 @@ import type {
   ResolvedGenericExecution,
 } from '../core/generic-leaf-execution.ts';
 import { createReadonlyRuntimeSessionStore } from '../core/runtime-session.ts';
-import { assertScreenshotCropPolicy } from './screenshot-crop-target.ts';
+import {
+  assertScreenshotCropCapture,
+  assertScreenshotCropPolicy,
+} from './screenshot-crop-target.ts';
 import { buildScreenshotCropWarnings, cropScreenshotToSelector } from './screenshot-crop.ts';
 import { annotateScreenshotWithRefs } from '@agent-device/capture-kit/screenshot-overlay';
 import {
@@ -373,6 +376,7 @@ function createBoundScreenshotBackend(
         execution: screenshotExecutionFromContext(dispatchContext),
       });
       if (crop) {
+        assertScreenshotCropCapture(session.device, facts);
         const outcome = await cropScreenshotToSelector({
           device: session.device,
           session,

packages/platform-apple/src/core/__tests__/physical-device-control.test.ts

@@ -95,7 +95,7 @@ test('default interactor screenshots stay in-band without invoking devicectl', a
   const outPath = path.join(await mkdtempForTest('agent-device-xctest-screenshot-'), 'screen.png');
   const toolCalls: string[][] = [];
   try {
-    await withAppleToolProvider(
+    const facts = await withAppleToolProvider(
       createLocalAppleToolProvider({
         runCommand: async (_cmd, args) => {
           toolCalls.push(args);
@@ -120,6 +120,7 @@ test('default interactor screenshots stay in-band without invoking devicectl', a
         ),
     );
 
+    assert.deepEqual(facts, { iosPhysicalCaptureBackend: 'xctest' });
     assert.equal(await fs.readFile(outPath, 'utf8'), 'png-bytes');
     assert.equal(
       toolCalls.some((args) => args[0] === 'devicectl'),
@@ -136,3 +137,41 @@ test('CoreDevice-only app inventory fails with targeted XCTest guidance', async
     /App inventory is unavailable on this XCTest-backed physical iOS device/,
   );
 });
+
+for (const fallback of [false, true]) {
+  test(`physical interactor reports the actual screenshot backend after fallback=${fallback}`, async () => {
+    const outPath = path.join(await mkdtempForTest('agent-device-capture-backend-'), 'screen.png');
+    let runnerCalls = 0;
+    const facts = await withAppleToolProvider(
+      createLocalAppleToolProvider({
+        runCommand: async (_cmd, args) => {
+          assert.deepEqual(args.slice(0, 4), ['devicectl', 'device', 'capture', 'screenshot']);
+          if (fallback)
+            return { exitCode: 1, stdout: '', stderr: "error: Unknown option '--device'" };
+          await fs.writeFile(outPath, 'coredevice-png');
+          return { exitCode: 0, stdout: '', stderr: '' };
+        },
+      }),
+      async () =>
+        await withAppleRunnerProvider(
+          async () => {
+            runnerCalls += 1;
+            return { imageBase64: Buffer.from('runner-png').toString('base64') };
+          },
+          { deviceId: IOS_DEVICE.id },
+          async () =>
+            await createAppleInteractor(
+              IOS_DEVICE,
+              {},
+              {
+                env: process.env,
+                cwd: process.cwd(),
+              },
+            ).screenshot(outPath),
+        ),
+    );
+    assert.deepEqual(facts, { iosPhysicalCaptureBackend: fallback ? 'xctest' : 'coredevice' });
+    assert.equal(runnerCalls, fallback ? 1 : 0);
+    assert.equal(await fs.readFile(outPath, 'utf8'), fallback ? 'runner-png' : 'coredevice-png');
+  });
+}

packages/platform-apple/src/core/physical-device-control.ts

@@ -1,3 +1,4 @@
+import type { ScreenshotCaptureFacts } from '@agent-device/contracts/interactor-types';
 import type { DeviceInfo } from '@agent-device/kernel/device';
 import type { EnvMap } from '@agent-device/kernel/source-value';
 import { AppError } from '@agent-device/kernel/errors';
@@ -61,7 +62,7 @@ export type IosPhysicalDeviceControl = IosPhysicalDeviceRunnerControl & {
     device: DeviceInfo,
     outPath: string,
     options: IosPhysicalDeviceScreenshotOptions,
-  ): Promise<void>;
+  ): Promise<ScreenshotCaptureFacts>;
   copyRunnerFile(
     device: DeviceInfo,
     remotePath: string,