From 8033598a32d1125d543d03d7b7e66a6ec734089e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Wed, 16 Sep 2026 12:49:17 +0200 Subject: [PATCH 1/3] fix(limrun): write the iOS screenshot as PNG instead of Limrun's JPEG bytes Limrun serves its simulator capture as JPEG and the interactor wrote those bytes straight to the .png path, so every screenshot on a Limrun iOS direct session failed downstream with "Screenshot file is not a valid PNG". Sniff the container and transcode a JPEG to PNG before writing; a PNG passes through unchanged. The transcode runs as a PNG worker job with the usual in-process fallback so a full-resolution capture never blocks the daemon event loop, and a JPEG whose body does not decode fails with the labeled COMMAND_FAILED the PNG decoder uses. jpeg-js is bundled like pngjs and loaded only on the screenshot path. --- CHANGELOG.md | 4 + package.json | 3 +- packages/capture-kit/package.json | 5 ++ .../capture-kit/src/png-transcode.test.ts | 76 +++++++++++++++++++ packages/capture-kit/src/png-transcode.ts | 48 ++++++++++++ .../capture-kit/src/png-worker-client.test.ts | 30 ++++++++ packages/capture-kit/src/png-worker-client.ts | 18 +++++ .../capture-kit/src/png-worker-contract.ts | 9 ++- packages/capture-kit/src/png-worker.ts | 9 +++ .../src/ios-interactor-screenshot.test.ts | 68 +++++++++++++++++ packages/provider-limrun/src/ios.ts | 13 +++- packages/provider-limrun/src/snapshot.ts | 7 -- .../provider-limrun/src/tmp-dir.fixtures.ts | 21 +++++ pnpm-lock.yaml | 11 +++ tsdown.config.ts | 1 + 15 files changed, 310 insertions(+), 13 deletions(-) create mode 100644 packages/capture-kit/src/png-transcode.test.ts create mode 100644 packages/capture-kit/src/png-transcode.ts create mode 100644 packages/provider-limrun/src/ios-interactor-screenshot.test.ts create mode 100644 packages/provider-limrun/src/tmp-dir.fixtures.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 02443941e6..457659a222 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Fixed (limrun): `screenshot` on Limrun iOS direct sessions writes a PNG. Limrun serves its capture + as JPEG and the interactor wrote those bytes straight to the `.png` path, so every capture failed + downstream with "Screenshot file is not a valid PNG". The bytes are now sniffed and a JPEG is + transcoded to PNG before it is written; a PNG passes through unchanged. - Changed (ios): a regular `snapshot --depth N` on the XCTest runner is a presentation cut over a full acquisition, not a bound on the walk. Acquisition publishes the frames the platform reported, one normalization pass turns geometry into the app's orientation space and recomputes `hittable`, diff --git a/package.json b/package.json index 34ef18047d..abe1e106e1 100644 --- a/package.json +++ b/package.json @@ -284,8 +284,8 @@ "@agent-device/device-selection": "workspace:*", "@agent-device/host-kit": "workspace:*", "@agent-device/kernel": "workspace:*", - "@agent-device/managed-allocation": "workspace:*", "@agent-device/maestro": "workspace:*", + "@agent-device/managed-allocation": "workspace:*", "@agent-device/platform-android": "workspace:*", "@agent-device/platform-apple": "workspace:*", "@agent-device/platform-harmonyos": "workspace:*", @@ -318,6 +318,7 @@ "fallow": "^2.104.0", "fast-check": "^4.9.0", "ipaddr.js": "^2.5.0", + "jpeg-js": "^0.4.4", "oxc-parser": "^0.146.0", "oxfmt": "^0.64.0", "oxlint": "^1.79.0", diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index de728bbb4b..3f3bb6eca8 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -126,6 +126,10 @@ "types": "./src/png-crop.ts", "default": "./src/png-crop.ts" }, + "./png-transcode": { + "types": "./src/png-transcode.ts", + "default": "./src/png-transcode.ts" + }, "./png-resize": { "types": "./src/png-resize.ts", "default": "./src/png-resize.ts" @@ -290,6 +294,7 @@ "devDependencies": { "@types/pngjs": "^6.0.5", "fast-check": "^4.9.0", + "jpeg-js": "^0.4.4", "pngjs": "^7.0.0" } } diff --git a/packages/capture-kit/src/png-transcode.test.ts b/packages/capture-kit/src/png-transcode.test.ts new file mode 100644 index 0000000000..91872ce16d --- /dev/null +++ b/packages/capture-kit/src/png-transcode.test.ts @@ -0,0 +1,76 @@ +import { expect, test } from 'vitest'; +import { encode as encodeJpeg } from 'jpeg-js'; +import { PNG } from './png.ts'; +import { detectScreenshotImageFormat, transcodeScreenshotToPng } from './png-transcode.ts'; + +function solidRgba(width: number, height: number, rgba: readonly [number, number, number, number]) { + const data = Buffer.alloc(width * height * 4); + for (let offset = 0; offset < data.length; offset += 4) data.set(rgba, offset); + return data; +} + +/** The value a synchronous call throws, so one assertion can check its code and message together. */ +function thrownBy(run: () => unknown): unknown { + try { + run(); + } catch (error) { + return error; + } + throw new Error('expected the call to throw'); +} + +test('a JPEG screenshot becomes a PNG of the same size with the same picture', () => { + const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [200, 30, 30, 255]) }, 100); + expect(detectScreenshotImageFormat(jpeg.data)).toBe('jpeg'); + + const png = transcodeScreenshotToPng(jpeg.data, 'test screenshot'); + + expect(detectScreenshotImageFormat(png)).toBe('png'); + const decoded = PNG.sync.read(png); + expect([decoded.width, decoded.height]).toEqual([6, 4]); + const [r = -1, g = -1, b = -1, a = -1] = decoded.data.subarray(0, 4); + // JPEG is lossy; a flat field survives within a few levels per channel. + expect(Math.abs(r - 200)).toBeLessThanOrEqual(4); + expect(Math.abs(g - 30)).toBeLessThanOrEqual(4); + expect(Math.abs(b - 30)).toBeLessThanOrEqual(4); + expect(a).toBe(255); +}); + +test('a PNG screenshot passes through byte for byte', () => { + const png = PNG.sync.write(new PNG({ width: 3, height: 2 })); + + expect(transcodeScreenshotToPng(png, 'test screenshot')).toBe(png); +}); + +test('bytes that are neither container are refused with a typed error carrying the label', () => { + expect( + thrownBy(() => transcodeScreenshotToPng(Buffer.from('GIF89a'), 'Limrun iOS screenshot')), + ).toMatchObject({ + code: 'COMMAND_FAILED', + message: 'Limrun iOS screenshot is neither PNG nor JPEG', + details: { label: 'Limrun iOS screenshot', leadingBytes: '47494638' }, + }); +}); + +test('a valid SOI marker followed by garbage is refused with the typed decode error, not the decoder exception', () => { + const garbage = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]); + expect(detectScreenshotImageFormat(garbage)).toBe('jpeg'); + + expect(thrownBy(() => transcodeScreenshotToPng(garbage, 'Limrun iOS screenshot'))).toMatchObject({ + code: 'COMMAND_FAILED', + message: 'Failed to decode Limrun iOS screenshot as JPEG', + details: { label: 'Limrun iOS screenshot', reason: expect.any(String) }, + }); +}); + +test('a truncated JPEG body is refused with the same typed decode error', () => { + const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [10, 200, 30, 255]) }, 100); + const truncated = jpeg.data.subarray(0, 24); + + expect( + thrownBy(() => transcodeScreenshotToPng(truncated, 'Limrun iOS screenshot')), + ).toMatchObject({ + code: 'COMMAND_FAILED', + message: 'Failed to decode Limrun iOS screenshot as JPEG', + }); +}); diff --git a/packages/capture-kit/src/png-transcode.ts b/packages/capture-kit/src/png-transcode.ts new file mode 100644 index 0000000000..d9c318fc43 --- /dev/null +++ b/packages/capture-kit/src/png-transcode.ts @@ -0,0 +1,48 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { decode as decodeJpeg } from 'jpeg-js'; +import { PNG } from 'pngjs'; + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const JPEG_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]); + +export type ScreenshotImageFormat = 'png' | 'jpeg'; + +/** Names the container a screenshot arrived in from its magic bytes, or `undefined` for neither. */ +export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageFormat | undefined { + if (bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)) return 'png'; + if (bytes.subarray(0, JPEG_SIGNATURE.length).equals(JPEG_SIGNATURE)) return 'jpeg'; + return undefined; +} + +/** + * Returns PNG bytes for a screenshot a provider handed back in whatever container it prefers. PNG + * passes through untouched; JPEG is decoded and re-encoded losslessly from the decoded pixels, so + * every PNG-only reader downstream (size, crop, overlay, diff) sees the format the path promises. + * + * Synchronous and CPU-bound: daemon request paths reach it through the PNG worker + * (`transcodeScreenshotToPngAsync` in `png-worker-client.ts`), like every other codec job. + */ +export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer { + const format = detectScreenshotImageFormat(bytes); + if (format === 'png') return bytes; + if (format === 'jpeg') return jpegToPng(bytes, label); + throw new AppError('COMMAND_FAILED', `${label} is neither PNG nor JPEG`, { + label, + leadingBytes: bytes.subarray(0, 4).toString('hex'), + }); +} + +/** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */ +function jpegToPng(bytes: Buffer, label: string): Buffer { + try { + const decoded = decodeJpeg(bytes, { useTArray: true, formatAsRGBA: true }); + const png = new PNG({ width: decoded.width, height: decoded.height }); + png.data = Buffer.from(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength); + return PNG.sync.write(png); + } catch (error) { + throw new AppError('COMMAND_FAILED', `Failed to decode ${label} as JPEG`, { + label, + reason: error instanceof Error ? error.message : String(error), + }); + } +} diff --git a/packages/capture-kit/src/png-worker-client.test.ts b/packages/capture-kit/src/png-worker-client.test.ts index 52a00cccb5..57b1afe0de 100644 --- a/packages/capture-kit/src/png-worker-client.test.ts +++ b/packages/capture-kit/src/png-worker-client.test.ts @@ -8,6 +8,7 @@ import { decodePngAsync, encodePngAsync, terminatePngWorker, + transcodeScreenshotToPngAsync, } from './png-worker-client.ts'; import { computeScreenshotDiffPixels } from './screenshot-diff-pixels.ts'; @@ -106,3 +107,32 @@ test('decodePngAsync rejects invalid PNG data with the canonical decode AppError }, ); }); + +test('transcodeScreenshotToPngAsync matches the synchronous transcoder byte for byte', async () => { + const { encode } = await import('jpeg-js'); + const { transcodeScreenshotToPng } = await import('./png-transcode.ts'); + const rgba = Buffer.alloc(9 * 7 * 4, 0x66); + const jpeg = encode({ width: 9, height: 7, data: rgba }, 90).data; + + const fromWorker = await transcodeScreenshotToPngAsync(jpeg, 'fixture'); + + assert.deepEqual(fromWorker, transcodeScreenshotToPng(jpeg, 'fixture')); + assert.equal(PNG.sync.read(fromWorker).width, 9); +}); + +test('transcodeScreenshotToPngAsync returns a PNG untouched without a worker round trip', async () => { + const png = PNG.sync.write(new PNG({ width: 2, height: 2 })); + + assert.equal(await transcodeScreenshotToPngAsync(png, 'fixture'), png); +}); + +test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical decode AppError', async () => { + const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(16, 0)]); + + await assert.rejects(transcodeScreenshotToPngAsync(corrupt, 'fixture'), (error) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'COMMAND_FAILED'); + assert.match((error as AppError).message, /Failed to decode fixture as JPEG/); + return true; + }); +}); diff --git a/packages/capture-kit/src/png-worker-client.ts b/packages/capture-kit/src/png-worker-client.ts index fba83ff102..d805990367 100644 --- a/packages/capture-kit/src/png-worker-client.ts +++ b/packages/capture-kit/src/png-worker-client.ts @@ -28,6 +28,8 @@ import { */ const PNG_WORKER_ENTRYPOINT = 'png-worker'; +// Kept local so the JPEG decoder module stays out of every entry that only needs PNG jobs. +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); /** Worker-infrastructure failure: the generic runner falls back to the sync path. */ class PngWorkerUnavailableError extends Error {} @@ -256,3 +258,19 @@ export async function computeScreenshotDiffPixelsAsync( })); return { ...result, diffData: toBuffer(result.diffData) }; } + +/** + * PNG bytes for a provider screenshot in whatever container it arrived in. A PNG returns as is + * without a worker round trip; a JPEG is decoded and re-encoded on the worker so a full-resolution + * capture never blocks the daemon event loop. Decode failures carry the canonical `AppError`. + */ +export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string): Promise { + if (bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)) return bytes; + const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => { + // Read on demand so the JPEG decoder stays out of the import closure of every entry that only + // needs the worker's other jobs. + const { transcodeScreenshotToPng } = await import('./png-transcode.ts'); + return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(bytes, label) }; + }); + return toBuffer(result.png); +} diff --git a/packages/capture-kit/src/png-worker-contract.ts b/packages/capture-kit/src/png-worker-contract.ts index 67dfd0b8c8..0511b11541 100644 --- a/packages/capture-kit/src/png-worker-contract.ts +++ b/packages/capture-kit/src/png-worker-contract.ts @@ -9,7 +9,7 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts'; /** * Message contract between the daemon-side PNG worker client * (`png-worker-client.ts`) and the worker thread entry (`png-worker.ts`). - * One message = one decode, encode, crop, or diff job. Binary payloads cross the + * One message = one decode, encode, crop, diff, or transcode job. Binary payloads cross the * thread boundary via structured clone (or transfer), so `Buffer` fields * arrive as plain `Uint8Array` views on the receiving side. */ @@ -19,7 +19,9 @@ export type PngWorkerJob = | { kind: 'encode'; width: number; height: number; data: Uint8Array } | { kind: 'crop'; png: Uint8Array; label: string; box: Rect } | { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string } - | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob); + | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob) + // A provider screenshot in whatever container it arrived in; answers PNG bytes. + | { kind: 'jpeg-to-png'; image: Uint8Array; label: string }; export type PngWorkerJobResult = | { kind: 'decode'; width: number; height: number; data: Uint8Array } @@ -27,7 +29,8 @@ export type PngWorkerJobResult = // A crop answers `null` when the box already covers the image, so the caller keeps the file. | { kind: 'crop'; png: Uint8Array | null } | ({ kind: 'rgb-difference' } & PngRgbDifferenceResult) - | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult); + | ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult) + | { kind: 'jpeg-to-png'; png: Uint8Array }; export type PngWorkerJobKind = PngWorkerJob['kind']; diff --git a/packages/capture-kit/src/png-worker.ts b/packages/capture-kit/src/png-worker.ts index 3d980393fb..1c255ded57 100644 --- a/packages/capture-kit/src/png-worker.ts +++ b/packages/capture-kit/src/png-worker.ts @@ -4,6 +4,7 @@ import { cropPngBytes } from './png-crop-bytes.ts'; import { decodePng, PNG } from './png.ts'; import { computeScreenshotDiffPixels } from './screenshot-diff-pixels.ts'; import { computePngRgbDifference } from './png-rgb-difference.ts'; +import { transcodeScreenshotToPng } from './png-transcode.ts'; import { toBuffer, type PngWorkerJobResult, @@ -42,6 +43,12 @@ function runJob(request: PngWorkerRequest): PngWorkerJobResult { case 'diff-pixels': { return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) }; } + case 'jpeg-to-png': { + return { + kind: 'jpeg-to-png', + png: transcodeScreenshotToPng(toBuffer(request.image), request.label), + }; + } } } @@ -81,6 +88,8 @@ function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] { return []; case 'diff-pixels': return [result.diffData, result.diffMask]; + case 'jpeg-to-png': + return [result.png]; } } diff --git a/packages/provider-limrun/src/ios-interactor-screenshot.test.ts b/packages/provider-limrun/src/ios-interactor-screenshot.test.ts new file mode 100644 index 0000000000..5f9fb9db0f --- /dev/null +++ b/packages/provider-limrun/src/ios-interactor-screenshot.test.ts @@ -0,0 +1,68 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test, vi } from 'vitest'; +import { readPngSize } from '@agent-device/capture-kit/png-size'; +import { createLimrunIosInteractor, type LimrunIosSession } from './ios.ts'; +import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; + +/** An 8x6 mid-gray JPEG at quality 90: the container Limrun serves its captures in. */ +const JPEG_8X6_BASE64 = [ + '/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4R', + 'DgsLEBYQERMUFRUVDA8XGBYUGBIUFRQBAwQEBQQFCQUFCRQNCw0UFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU', + 'FBQUFBQUFBQUFBQUFBQUFBQUFBQUFP/AABEIAAYACAMBEQACEQEDEQH/xAGiAAABBQEBAQEBAQAAAAAAAAAAAQID', + 'BAUGBwgJCgsQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYX', + 'GBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqi', + 'o6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+gEAAwEBAQEBAQEB', + 'AQAAAAAAAAECAwQFBgcICQoLEQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS', + '8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeI', + 'iYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/', + '2gAMAwEAAhEDEQA/ACgD/9k=', +].join(''); + +function sessionWithScreenshot(base64: string) { + const client = { + screenshot: vi.fn(async () => ({ base64, width: 402, height: 874 })), + }; + const session = { + platform: 'ios', + instanceId: 'limrun-screenshot-instance', + client, + } as unknown as LimrunIosSession; + return { interactor: createLimrunIosInteractor(session), client }; +} + +test('the JPEG Limrun serves is written to the PNG path as a PNG of the same size', async () => { + const outPath = path.join(mkdtempForTestSync('agent-device-limrun-screenshot-'), 'shot.png'); + const { interactor } = sessionWithScreenshot(JPEG_8X6_BASE64); + + await interactor.screenshot(outPath); + + const bytes = fs.readFileSync(outPath); + expect(bytes.subarray(0, 8)).toEqual( + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + ); + await expect(readPngSize(outPath)).resolves.toEqual({ width: 8, height: 6 }); +}); + +test('a capture that is neither PNG nor JPEG is refused instead of written', async () => { + const outPath = path.join(mkdtempForTestSync('agent-device-limrun-screenshot-'), 'shot.png'); + const { interactor } = sessionWithScreenshot(Buffer.from('not an image').toString('base64')); + + await expect(interactor.screenshot(outPath)).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + message: 'Limrun iOS screenshot is neither PNG nor JPEG', + }); + expect(fs.existsSync(outPath)).toBe(false); +}); + +test('a JPEG header over a body that does not decode is refused with the typed decode error', async () => { + const outPath = path.join(mkdtempForTestSync('agent-device-limrun-screenshot-'), 'shot.png'); + const garbage = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]); + const { interactor } = sessionWithScreenshot(garbage.toString('base64')); + + await expect(interactor.screenshot(outPath)).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + message: 'Failed to decode Limrun iOS screenshot as JPEG', + }); + expect(fs.existsSync(outPath)).toBe(false); +}); diff --git a/packages/provider-limrun/src/ios.ts b/packages/provider-limrun/src/ios.ts index 63deb428af..b0631088dd 100644 --- a/packages/provider-limrun/src/ios.ts +++ b/packages/provider-limrun/src/ios.ts @@ -17,7 +17,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; -import { toIosSelector, writeBase64File } from './snapshot.ts'; +import { toIosSelector } from './snapshot.ts'; import { normalizeOptionalString } from './strings.ts'; import { awaitLimrunDeploymentOperation, @@ -296,9 +296,18 @@ class LimrunIosInteractor implements Interactor { await this.session.client.scroll(direction, options?.pixels ?? 300); } + /** Limrun serves its capture as JPEG; the PNG-only readers behind `outPath` get a PNG. */ async screenshot(outPath: string): Promise { + // Loaded on the screenshot path to keep this provider's declared import-time closure budget. + const { transcodeScreenshotToPngAsync } = + await import('@agent-device/capture-kit/png-worker-client'); const screenshot = await this.session.client.screenshot(); - await writeBase64File(outPath, screenshot.base64); + const png = await transcodeScreenshotToPngAsync( + Buffer.from(screenshot.base64, 'base64'), + 'Limrun iOS screenshot', + ); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + await fs.promises.writeFile(outPath, png); } async snapshot() { diff --git a/packages/provider-limrun/src/snapshot.ts b/packages/provider-limrun/src/snapshot.ts index 47931445df..0c87e6d05c 100644 --- a/packages/provider-limrun/src/snapshot.ts +++ b/packages/provider-limrun/src/snapshot.ts @@ -1,5 +1,3 @@ -import fs from 'node:fs'; -import path from 'node:path'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; type LimrunSelector = { key: 'id' | 'label' | 'text' | 'value'; value: string }; @@ -111,8 +109,3 @@ export function toIosSelector(selector: LimrunSelector) { // agent-device label and text selectors target the provider's label field. return { AXLabel: selector.value }; } - -export async function writeBase64File(filePath: string, base64: string): Promise { - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - await fs.promises.writeFile(filePath, Buffer.from(base64, 'base64')); -} diff --git a/packages/provider-limrun/src/tmp-dir.fixtures.ts b/packages/provider-limrun/src/tmp-dir.fixtures.ts new file mode 100644 index 0000000000..dbc5c2724c --- /dev/null +++ b/packages/provider-limrun/src/tmp-dir.fixtures.ts @@ -0,0 +1,21 @@ +import fs from 'node:fs'; +import fsPromises from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +/** + * Creates a fresh scratch directory for one test. Cleanup is automatic: the + * unit suite redirects TMPDIR to a per-run directory (scripts/vitest-tmpdir-global-setup.ts) + * that gets removed in one recursive rm after every worker finishes, so + * individual tests never need their own afterEach/afterAll for this. + */ +// fallow-ignore-next-line code-duplication +export async function mkdtempForTest(prefix: string): Promise { + return fsPromises.mkdtemp(path.join(os.tmpdir(), prefix)); +} + +/** Sync counterpart of {@link mkdtempForTest}, for setup code that can't await. */ +// fallow-ignore-next-line code-duplication +export function mkdtempForTestSync(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d9736ae90..c5b7d73020 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -143,6 +143,9 @@ importers: ipaddr.js: specifier: ^2.5.0 version: 2.5.0 + jpeg-js: + specifier: ^0.4.4 + version: 0.4.4 oxc-parser: specifier: ^0.146.0 version: 0.146.0 @@ -225,6 +228,9 @@ importers: fast-check: specifier: ^4.9.0 version: 4.9.0 + jpeg-js: + specifier: ^0.4.4 + version: 0.4.4 pngjs: specifier: ^7.0.0 version: 7.0.0 @@ -2906,6 +2912,9 @@ packages: resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} engines: {node: '>=8'} + jpeg-js@0.4.4: + resolution: {integrity: sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==} + js-md4@0.3.2: resolution: {integrity: sha512-/GDnfQYsltsjRswQhN9fhv3EMw2sCpUdrdxyWDOUK7eyD++r3gRhzgiQgc/x4MAv2i1iuQ4lxO5mvqM3vj4bwA==} @@ -6283,6 +6292,8 @@ snapshots: html-escaper: 2.0.2 istanbul-lib-report: 3.0.1 + jpeg-js@0.4.4: {} + js-md4@0.3.2: {} js-tokens@10.0.0: {} diff --git a/tsdown.config.ts b/tsdown.config.ts index c312ea994a..d3b4ea27a3 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -127,6 +127,7 @@ export default defineConfig({ 'https-proxy-agent', 'ignore', 'ipaddr.js', + 'jpeg-js', 'ms', 'pend', 'pngjs', From ef2f42216a0d2d98b5f08ff38cbdb17c2a75a05c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 19 Sep 2026 14:01:03 +0200 Subject: [PATCH 2/3] refactor(capture-kit): keep the JPEG transcode behind the PNG worker The Limrun screenshot path reaches the transcoder only through png-worker-client, so the ./png-transcode subpath and its layering-guard entry are dropped. The PNG signature check lives in png.ts, which both the worker client and the transcoder already load, so the two sniffs cannot drift and no entry's eager closure grows. The corrupt-JPEG worker test now pins details.label and details.reason across the worker boundary. Also restores the root package.json dependency order main has. --- package.json | 2 +- packages/capture-kit/package.json | 4 ---- packages/capture-kit/src/png-transcode.ts | 5 ++--- packages/capture-kit/src/png-worker-client.test.ts | 2 ++ packages/capture-kit/src/png-worker-client.ts | 6 ++---- packages/capture-kit/src/png.ts | 7 +++++++ 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index abe1e106e1..df107cefdf 100644 --- a/package.json +++ b/package.json @@ -284,8 +284,8 @@ "@agent-device/device-selection": "workspace:*", "@agent-device/host-kit": "workspace:*", "@agent-device/kernel": "workspace:*", - "@agent-device/maestro": "workspace:*", "@agent-device/managed-allocation": "workspace:*", + "@agent-device/maestro": "workspace:*", "@agent-device/platform-android": "workspace:*", "@agent-device/platform-apple": "workspace:*", "@agent-device/platform-harmonyos": "workspace:*", diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 3f3bb6eca8..154cae9d3e 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -126,10 +126,6 @@ "types": "./src/png-crop.ts", "default": "./src/png-crop.ts" }, - "./png-transcode": { - "types": "./src/png-transcode.ts", - "default": "./src/png-transcode.ts" - }, "./png-resize": { "types": "./src/png-resize.ts", "default": "./src/png-resize.ts" diff --git a/packages/capture-kit/src/png-transcode.ts b/packages/capture-kit/src/png-transcode.ts index d9c318fc43..121360b95a 100644 --- a/packages/capture-kit/src/png-transcode.ts +++ b/packages/capture-kit/src/png-transcode.ts @@ -1,15 +1,14 @@ import { AppError } from '@agent-device/kernel/errors'; import { decode as decodeJpeg } from 'jpeg-js'; -import { PNG } from 'pngjs'; +import { hasPngSignature, PNG } from './png.ts'; -const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const JPEG_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]); export type ScreenshotImageFormat = 'png' | 'jpeg'; /** Names the container a screenshot arrived in from its magic bytes, or `undefined` for neither. */ export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageFormat | undefined { - if (bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)) return 'png'; + if (hasPngSignature(bytes)) return 'png'; if (bytes.subarray(0, JPEG_SIGNATURE.length).equals(JPEG_SIGNATURE)) return 'jpeg'; return undefined; } diff --git a/packages/capture-kit/src/png-worker-client.test.ts b/packages/capture-kit/src/png-worker-client.test.ts index 57b1afe0de..6a09d8efe5 100644 --- a/packages/capture-kit/src/png-worker-client.test.ts +++ b/packages/capture-kit/src/png-worker-client.test.ts @@ -133,6 +133,8 @@ test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical de assert.equal(error instanceof AppError, true); assert.equal((error as AppError).code, 'COMMAND_FAILED'); assert.match((error as AppError).message, /Failed to decode fixture as JPEG/); + assert.equal((error as AppError).details?.label, 'fixture'); + assert.ok(String((error as AppError).details?.reason).length > 0); return true; }); }); diff --git a/packages/capture-kit/src/png-worker-client.ts b/packages/capture-kit/src/png-worker-client.ts index d805990367..1111259af7 100644 --- a/packages/capture-kit/src/png-worker-client.ts +++ b/packages/capture-kit/src/png-worker-client.ts @@ -3,7 +3,7 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { AppError, toAppErrorCode } from '@agent-device/kernel/errors'; import type { Rect } from '@agent-device/kernel/snapshot'; import { resolveInternalEntryModulePath } from './internal-entry.ts'; -import { decodePng, PNG } from './png.ts'; +import { decodePng, hasPngSignature, PNG } from './png.ts'; import { computeScreenshotDiffPixels, type ScreenshotDiffPixelsJob, @@ -28,8 +28,6 @@ import { */ const PNG_WORKER_ENTRYPOINT = 'png-worker'; -// Kept local so the JPEG decoder module stays out of every entry that only needs PNG jobs. -const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); /** Worker-infrastructure failure: the generic runner falls back to the sync path. */ class PngWorkerUnavailableError extends Error {} @@ -265,7 +263,7 @@ export async function computeScreenshotDiffPixelsAsync( * capture never blocks the daemon event loop. Decode failures carry the canonical `AppError`. */ export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string): Promise { - if (bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)) return bytes; + if (hasPngSignature(bytes)) return bytes; const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => { // Read on demand so the JPEG decoder stays out of the import closure of every entry that only // needs the worker's other jobs. diff --git a/packages/capture-kit/src/png.ts b/packages/capture-kit/src/png.ts index a485e68184..4fc90b23d7 100644 --- a/packages/capture-kit/src/png.ts +++ b/packages/capture-kit/src/png.ts @@ -3,6 +3,13 @@ import { PNG } from 'pngjs'; export { PNG }; +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +/** Whether the bytes open with the PNG signature, before any decode is attempted. */ +export function hasPngSignature(bytes: Buffer): boolean { + return bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE); +} + /** * Decodes a PNG, wrapping failures in the canonical decode `AppError`. Shared * by the in-process sync path and the PNG worker thread (`png-worker.ts`), so From ea5f5f2e31659a2657945975e49aef9a107e4b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 19 Sep 2026 14:44:04 +0200 Subject: [PATCH 3/3] perf(capture-kit): load the JPEG decoder in the PNG worker only for a transcode job The daemon prewarms the PNG worker at startup, and its static import of png-transcode evaluated jpeg-js in every daemon's worker thread whether or not a Limrun screenshot was ever taken. The jpeg-to-png case now imports the transcoder on first use. --- packages/capture-kit/src/png-worker.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/capture-kit/src/png-worker.ts b/packages/capture-kit/src/png-worker.ts index 1c255ded57..68a66bf06d 100644 --- a/packages/capture-kit/src/png-worker.ts +++ b/packages/capture-kit/src/png-worker.ts @@ -4,7 +4,6 @@ import { cropPngBytes } from './png-crop-bytes.ts'; import { decodePng, PNG } from './png.ts'; import { computeScreenshotDiffPixels } from './screenshot-diff-pixels.ts'; import { computePngRgbDifference } from './png-rgb-difference.ts'; -import { transcodeScreenshotToPng } from './png-transcode.ts'; import { toBuffer, type PngWorkerJobResult, @@ -18,7 +17,8 @@ import { * `png-worker-client.ts`; published as the `internal/png-worker` build entry. */ -function runJob(request: PngWorkerRequest): PngWorkerJobResult { +// The daemon prewarms this worker at startup, so the JPEG decoder loads only for a transcode job. +async function runJob(request: PngWorkerRequest): Promise { switch (request.kind) { case 'decode': { const png = decodePng(toBuffer(request.png), request.label); @@ -44,6 +44,7 @@ function runJob(request: PngWorkerRequest): PngWorkerJobResult { return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) }; } case 'jpeg-to-png': { + const { transcodeScreenshotToPng } = await import('./png-transcode.ts'); return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(toBuffer(request.image), request.label), @@ -95,10 +96,10 @@ function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] { const port = parentPort; if (port) { - port.on('message', (request: PngWorkerRequest) => { + port.on('message', async (request: PngWorkerRequest) => { let response: PngWorkerResponse; try { - response = { id: request.id, ok: true, result: runJob(request) }; + response = { id: request.id, ok: true, result: await runJob(request) }; } catch (error) { response = { id: request.id, ok: false, error: normalizeError(error) }; }