fix(limrun): write the iOS screenshot as PNG instead of Limrun's JPEG bytes

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+- Fixed (limrun): `screenshot` on Limrun iOS direct sessions writes a PNG. Limrun serves its capture
+  as JPEG and the interactor wrote those bytes straight to the `.png` path, so every capture failed
+  downstream with "Screenshot file is not a valid PNG". The bytes are now sniffed and a JPEG is
+  transcoded to PNG before it is written; a PNG passes through unchanged.
 - Changed (ios): a regular `snapshot --depth N` on the XCTest runner is a presentation cut over a
   full acquisition, not a bound on the walk. Acquisition publishes the frames the platform reported,
   one normalization pass turns geometry into the app's orientation space and recomputes `hittable`,

package.json

@@ -318,6 +318,7 @@
     "fallow": "^2.104.0",
     "fast-check": "^4.9.0",
     "ipaddr.js": "^2.5.0",
+    "jpeg-js": "^0.4.4",
     "oxc-parser": "^0.146.0",
     "oxfmt": "^0.64.0",
     "oxlint": "^1.79.0",

packages/capture-kit/package.json

@@ -290,6 +290,7 @@
   "devDependencies": {
     "@types/pngjs": "^6.0.5",
     "fast-check": "^4.9.0",
+    "jpeg-js": "^0.4.4",
     "pngjs": "^7.0.0"
   }
 }

packages/capture-kit/src/png-transcode.test.ts

@@ -0,0 +1,76 @@
+import { expect, test } from 'vitest';
+import { encode as encodeJpeg } from 'jpeg-js';
+import { PNG } from './png.ts';
+import { detectScreenshotImageFormat, transcodeScreenshotToPng } from './png-transcode.ts';
+
+function solidRgba(width: number, height: number, rgba: readonly [number, number, number, number]) {
+  const data = Buffer.alloc(width * height * 4);
+  for (let offset = 0; offset < data.length; offset += 4) data.set(rgba, offset);
+  return data;
+}
+
+/** The value a synchronous call throws, so one assertion can check its code and message together. */
+function thrownBy(run: () => unknown): unknown {
+  try {
+    run();
+  } catch (error) {
+    return error;
+  }
+  throw new Error('expected the call to throw');
+}
+
+test('a JPEG screenshot becomes a PNG of the same size with the same picture', () => {
+  const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [200, 30, 30, 255]) }, 100);
+  expect(detectScreenshotImageFormat(jpeg.data)).toBe('jpeg');
+
+  const png = transcodeScreenshotToPng(jpeg.data, 'test screenshot');
+
+  expect(detectScreenshotImageFormat(png)).toBe('png');
+  const decoded = PNG.sync.read(png);
+  expect([decoded.width, decoded.height]).toEqual([6, 4]);
+  const [r = -1, g = -1, b = -1, a = -1] = decoded.data.subarray(0, 4);
+  // JPEG is lossy; a flat field survives within a few levels per channel.
+  expect(Math.abs(r - 200)).toBeLessThanOrEqual(4);
+  expect(Math.abs(g - 30)).toBeLessThanOrEqual(4);
+  expect(Math.abs(b - 30)).toBeLessThanOrEqual(4);
+  expect(a).toBe(255);
+});
+
+test('a PNG screenshot passes through byte for byte', () => {
+  const png = PNG.sync.write(new PNG({ width: 3, height: 2 }));
+
+  expect(transcodeScreenshotToPng(png, 'test screenshot')).toBe(png);
+});
+
+test('bytes that are neither container are refused with a typed error carrying the label', () => {
+  expect(
+    thrownBy(() => transcodeScreenshotToPng(Buffer.from('GIF89a'), 'Limrun iOS screenshot')),
+  ).toMatchObject({
+    code: 'COMMAND_FAILED',
+    message: 'Limrun iOS screenshot is neither PNG nor JPEG',
+    details: { label: 'Limrun iOS screenshot', leadingBytes: '47494638' },
+  });
+});
+
+test('a valid SOI marker followed by garbage is refused with the typed decode error, not the decoder exception', () => {
+  const garbage = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]);
+  expect(detectScreenshotImageFormat(garbage)).toBe('jpeg');
+
+  expect(thrownBy(() => transcodeScreenshotToPng(garbage, 'Limrun iOS screenshot'))).toMatchObject({
+    code: 'COMMAND_FAILED',
+    message: 'Failed to decode Limrun iOS screenshot as JPEG',
+    details: { label: 'Limrun iOS screenshot', reason: expect.any(String) },
+  });
+});
+
+test('a truncated JPEG body is refused with the same typed decode error', () => {
+  const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [10, 200, 30, 255]) }, 100);
+  const truncated = jpeg.data.subarray(0, 24);
+
+  expect(
+    thrownBy(() => transcodeScreenshotToPng(truncated, 'Limrun iOS screenshot')),
+  ).toMatchObject({
+    code: 'COMMAND_FAILED',
+    message: 'Failed to decode Limrun iOS screenshot as JPEG',
+  });
+});

packages/capture-kit/src/png-transcode.ts

@@ -0,0 +1,47 @@
+import { AppError } from '@agent-device/kernel/errors';
+import { decode as decodeJpeg } from 'jpeg-js';
+import { hasPngSignature, PNG } from './png.ts';
+
+const JPEG_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]);
+
+export type ScreenshotImageFormat = 'png' | 'jpeg';
+
+/** Names the container a screenshot arrived in from its magic bytes, or `undefined` for neither. */
+export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageFormat | undefined {
+  if (hasPngSignature(bytes)) return 'png';
+  if (bytes.subarray(0, JPEG_SIGNATURE.length).equals(JPEG_SIGNATURE)) return 'jpeg';
+  return undefined;
+}
+
+/**
+ * Returns PNG bytes for a screenshot a provider handed back in whatever container it prefers. PNG
+ * passes through untouched; JPEG is decoded and re-encoded losslessly from the decoded pixels, so
+ * every PNG-only reader downstream (size, crop, overlay, diff) sees the format the path promises.
+ *
+ * Synchronous and CPU-bound: daemon request paths reach it through the PNG worker
+ * (`transcodeScreenshotToPngAsync` in `png-worker-client.ts`), like every other codec job.
+ */
+export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer {
+  const format = detectScreenshotImageFormat(bytes);
+  if (format === 'png') return bytes;
+  if (format === 'jpeg') return jpegToPng(bytes, label);
+  throw new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, {
+    label,
+    leadingBytes: bytes.subarray(0, 4).toString('hex'),
+  });
+}
+
+/** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */
+function jpegToPng(bytes: Buffer, label: string): Buffer {
+  try {
+    const decoded = decodeJpeg(bytes, { useTArray: true, formatAsRGBA: true });
+    const png = new PNG({ width: decoded.width, height: decoded.height });
+    png.data = Buffer.from(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength);
+    return PNG.sync.write(png);
+  } catch (error) {
+    throw new AppError('COMMAND_FAILED', `Failed to decode ${label} as JPEG`, {
+      label,
+      reason: error instanceof Error ? error.message : String(error),
+    });
+  }
+}

packages/capture-kit/src/png-worker-client.test.ts

@@ -8,6 +8,7 @@ import {
   decodePngAsync,
   encodePngAsync,
   terminatePngWorker,
+  transcodeScreenshotToPngAsync,
 } from './png-worker-client.ts';
 import { computeScreenshotDiffPixels } from './screenshot-diff-pixels.ts';
 
@@ -106,3 +107,34 @@ test('decodePngAsync rejects invalid PNG data with the canonical decode AppError
     },
   );
 });
+
+test('transcodeScreenshotToPngAsync matches the synchronous transcoder byte for byte', async () => {
+  const { encode } = await import('jpeg-js');
+  const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
+  const rgba = Buffer.alloc(9 * 7 * 4, 0x66);
+  const jpeg = encode({ width: 9, height: 7, data: rgba }, 90).data;
+
+  const fromWorker = await transcodeScreenshotToPngAsync(jpeg, 'fixture');
+
+  assert.deepEqual(fromWorker, transcodeScreenshotToPng(jpeg, 'fixture'));
+  assert.equal(PNG.sync.read(fromWorker).width, 9);
+});
+
+test('transcodeScreenshotToPngAsync returns a PNG untouched without a worker round trip', async () => {
+  const png = PNG.sync.write(new PNG({ width: 2, height: 2 }));
+
+  assert.equal(await transcodeScreenshotToPngAsync(png, 'fixture'), png);
+});
+
+test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical decode AppError', async () => {
+  const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(16, 0)]);
+
+  await assert.rejects(transcodeScreenshotToPngAsync(corrupt, 'fixture'), (error) => {
+    assert.equal(error instanceof AppError, true);
+    assert.equal((error as AppError).code, 'COMMAND_FAILED');
+    assert.match((error as AppError).message, /Failed to decode fixture as JPEG/);
+    assert.equal((error as AppError).details?.label, 'fixture');
+    assert.ok(String((error as AppError).details?.reason).length > 0);
+    return true;
+  });
+});

packages/capture-kit/src/png-worker-client.ts

@@ -3,7 +3,7 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
 import { AppError, toAppErrorCode } from '@agent-device/kernel/errors';
 import type { Rect } from '@agent-device/kernel/snapshot';
 import { resolveInternalEntryModulePath } from './internal-entry.ts';
-import { decodePng, PNG } from './png.ts';
+import { decodePng, hasPngSignature, PNG } from './png.ts';
 import {
   computeScreenshotDiffPixels,
   type ScreenshotDiffPixelsJob,
@@ -256,3 +256,19 @@ export async function computeScreenshotDiffPixelsAsync(
   }));
   return { ...result, diffData: toBuffer(result.diffData) };
 }
+
+/**
+ * PNG bytes for a provider screenshot in whatever container it arrived in. A PNG returns as is
+ * without a worker round trip; a JPEG is decoded and re-encoded on the worker so a full-resolution
+ * capture never blocks the daemon event loop. Decode failures carry the canonical `AppError`.
+ */
+export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string): Promise<Buffer> {
+  if (hasPngSignature(bytes)) return bytes;
+  const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => {
+    // Read on demand so the JPEG decoder stays out of the import closure of every entry that only
+    // needs the worker's other jobs.
+    const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
+    return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(bytes, label) };
+  });
+  return toBuffer(result.png);
+}

packages/capture-kit/src/png-worker-contract.ts

@@ -9,7 +9,7 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts';
 /**
  * Message contract between the daemon-side PNG worker client
  * (`png-worker-client.ts`) and the worker thread entry (`png-worker.ts`).
- * One message = one decode, encode, crop, or diff job. Binary payloads cross the
+ * One message = one decode, encode, crop, diff, or transcode job. Binary payloads cross the
  * thread boundary via structured clone (or transfer), so `Buffer` fields
  * arrive as plain `Uint8Array` views on the receiving side.
  */
@@ -19,15 +19,18 @@ export type PngWorkerJob =
   | { kind: 'encode'; width: number; height: number; data: Uint8Array }
   | { kind: 'crop'; png: Uint8Array; label: string; box: Rect }
   | { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string }
-  | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob);
+  | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob)
+  // A provider screenshot in whatever container it arrived in; answers PNG bytes.
+  | { kind: 'jpeg-to-png'; image: Uint8Array; label: string };
 
 export type PngWorkerJobResult =
   | { kind: 'decode'; width: number; height: number; data: Uint8Array }
   | { kind: 'encode'; png: Uint8Array }
   // A crop answers `null` when the box already covers the image, so the caller keeps the file.
   | { kind: 'crop'; png: Uint8Array | null }
   | ({ kind: 'rgb-difference' } & PngRgbDifferenceResult)
-  | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult);
+  | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult)
+  | { kind: 'jpeg-to-png'; png: Uint8Array };
 
 export type PngWorkerJobKind = PngWorkerJob['kind'];
 

packages/capture-kit/src/png-worker.ts

@@ -17,7 +17,8 @@ import {
  * `png-worker-client.ts`; published as the `internal/png-worker` build entry.
  */
 
-function runJob(request: PngWorkerRequest): PngWorkerJobResult {
+// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a transcode job.
+async function runJob(request: PngWorkerRequest): Promise<PngWorkerJobResult> {
   switch (request.kind) {
     case 'decode': {
       const png = decodePng(toBuffer(request.png), request.label);
@@ -42,6 +43,13 @@ function runJob(request: PngWorkerRequest): PngWorkerJobResult {
     case 'diff-pixels': {
       return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) };
     }
+    case 'jpeg-to-png': {
+      const { transcodeScreenshotToPng } = await import('./png-transcode.ts');
+      return {
+        kind: 'jpeg-to-png',
+        png: transcodeScreenshotToPng(toBuffer(request.image), request.label),
+      };
+    }
   }
 }
 
@@ -81,15 +89,17 @@ function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] {
       return [];
     case 'diff-pixels':
       return [result.diffData, result.diffMask];
+    case 'jpeg-to-png':
+      return [result.png];
   }
 }
 
 const port = parentPort;
 if (port) {
-  port.on('message', (request: PngWorkerRequest) => {
+  port.on('message', async (request: PngWorkerRequest) => {
     let response: PngWorkerResponse;
     try {
-      response = { id: request.id, ok: true, result: runJob(request) };
+      response = { id: request.id, ok: true, result: await runJob(request) };
     } catch (error) {
       response = { id: request.id, ok: false, error: normalizeError(error) };
     }

packages/capture-kit/src/png.ts

@@ -3,6 +3,13 @@ import { PNG } from 'pngjs';
 
 export { PNG };
 
+const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+
+/** Whether the bytes open with the PNG signature, before any decode is attempted. */
+export function hasPngSignature(bytes: Buffer): boolean {
+  return bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE);
+}
+
 /**
  * Decodes a PNG, wrapping failures in the canonical decode `AppError`. Shared
  * by the in-process sync path and the PNG worker thread (`png-worker.ts`), so

packages/provider-limrun/src/ios-interactor-screenshot.test.ts

@@ -0,0 +1,68 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { expect, test, vi } from 'vitest';
+import { readPngSize } from '@agent-device/capture-kit/png-size';
+import { createLimrunIosInteractor, type LimrunIosSession } from './ios.ts';
+import { mkdtempForTestSync } from './tmp-dir.fixtures.ts';
+
+/** An 8x6 mid-gray JPEG at quality 90: the container Limrun serves its captures in. */
+const JPEG_8X6_BASE64 = [
+  '/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4R',
+  'DgsLEBYQERMUFRUVDA8XGBYUGBIUFRQBAwQEBQQFCQUFCRQNCw0UFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU',
+  'FBQUFBQUFBQUFBQUFBQUFBQUFBQUFP/AABEIAAYACAMBEQACEQEDEQH/xAGiAAABBQEBAQEBAQAAAAAAAAAAAQID',
+  'BAUGBwgJCgsQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYX',
+  'GBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi',
+  'o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+gEAAwEBAQEBAQEB',
+  'AQAAAAAAAAECAwQFBgcICQoLEQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS',
+  '8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeI',
+  'iYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/',
+  '2gAMAwEAAhEDEQA/ACgD/9k=',
+].join('');
+
+function sessionWithScreenshot(base64: string) {
+  const client = {
+    screenshot: vi.fn(async () => ({ base64, width: 402, height: 874 })),
+  };
+  const session = {
+    platform: 'ios',
+    instanceId: 'limrun-screenshot-instance',
+    client,
+  } as unknown as LimrunIosSession;
+  return { interactor: createLimrunIosInteractor(session), client };
+}
+
+test('the JPEG Limrun serves is written to the PNG path as a PNG of the same size', async () => {
+  const outPath = path.join(mkdtempForTestSync('agent-device-limrun-screenshot-'), 'shot.png');
+  const { interactor } = sessionWithScreenshot(JPEG_8X6_BASE64);
+
+  await interactor.screenshot(outPath);
+
+  const bytes = fs.readFileSync(outPath);
+  expect(bytes.subarray(0, 8)).toEqual(
+    Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+  );
+  await expect(readPngSize(outPath)).resolves.toEqual({ width: 8, height: 6 });
+});
+
+test('a capture that is neither PNG nor JPEG is refused instead of written', async () => {
+  const outPath = path.join(mkdtempForTestSync('agent-device-limrun-screenshot-'), 'shot.png');
+  const { interactor } = sessionWithScreenshot(Buffer.from('not an image').toString('base64'));
+
+  await expect(interactor.screenshot(outPath)).rejects.toMatchObject({
+    code: 'COMMAND_FAILED',
+    message: 'Limrun iOS screenshot is neither PNG nor JPEG',
+  });
+  expect(fs.existsSync(outPath)).toBe(false);
+});
+
+test('a JPEG header over a body that does not decode is refused with the typed decode error', async () => {
+  const outPath = path.join(mkdtempForTestSync('agent-device-limrun-screenshot-'), 'shot.png');
+  const garbage = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]);
+  const { interactor } = sessionWithScreenshot(garbage.toString('base64'));
+
+  await expect(interactor.screenshot(outPath)).rejects.toMatchObject({
+    code: 'COMMAND_FAILED',
+    message: 'Failed to decode Limrun iOS screenshot as JPEG',
+  });
+  expect(fs.existsSync(outPath)).toBe(false);
+});