From e057b79e743429046d20bceadc52b5a8bf485f8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 7 Oct 2026 22:33:00 +0200 Subject: [PATCH] chore(scripts): delete the iOS AX bridge spike and the PNG crop benchmark Both harnesses answered a one-time question that has since shipped: the AX bridge viability decision (#2237) and the region-crop comparison (#2505). Nothing in src, packages, docs, CI, or the gate catalog references them, and their raw evidence stays on the evidence tags. Co-Authored-By: Claude Fable 5.1 --- package.json | 2 - scripts/ios-ax-bridge-spike/README.md | 28 -- scripts/ios-ax-bridge-spike/adapter.test.ts | 30 -- scripts/ios-ax-bridge-spike/adapter.ts | 16 - scripts/ios-ax-bridge-spike/config.ts | 87 ---- .../ios-ax-bridge-spike/corrected-markdown.ts | 126 ----- .../corrected-report.test.ts | 263 ---------- .../ios-ax-bridge-spike/corrected-report.ts | 453 ------------------ .../ios-ax-bridge-spike/corrected-types.ts | 144 ------ scripts/ios-ax-bridge-spike/guest-adapter.ts | 237 --------- .../ios-ax-bridge-spike/guest-binary.test.ts | 32 -- scripts/ios-ax-bridge-spike/guest-binary.ts | 33 -- .../ios-ax-bridge-spike/guest-connection.ts | 290 ----------- .../guest-process-metrics.test.ts | 17 - .../guest-process-metrics.ts | 55 --- .../ios-ax-bridge-spike/guest-wire.test.ts | 206 -------- scripts/ios-ax-bridge-spike/guest-wire.ts | 408 ---------------- scripts/ios-ax-bridge-spike/limits.test.ts | 70 --- scripts/ios-ax-bridge-spike/limits.ts | 230 --------- scripts/ios-ax-bridge-spike/protocol.ts | 24 - .../ios-ax-bridge-spike/targeted-evidence.ts | 247 ---------- .../ios-ax-bridge-spike/targeted-readiness.ts | 71 --- .../ios-ax-bridge-spike/targeted-relaunch.ts | 151 ------ .../ios-ax-bridge-spike/targeted-request.ts | 68 --- scripts/ios-ax-bridge-spike/targeted-run.ts | 81 ---- scripts/ios-ax-bridge-spike/types.ts | 149 ------ scripts/png-crop-benchmark/README.md | 51 -- scripts/png-crop-benchmark/args.test.ts | 41 -- scripts/png-crop-benchmark/args.ts | 36 -- scripts/png-crop-benchmark/corpus.test.ts | 68 --- scripts/png-crop-benchmark/corpus.ts | 155 ------ scripts/png-crop-benchmark/pipelines.ts | 74 --- scripts/png-crop-benchmark/report.ts | 49 -- scripts/png-crop-benchmark/run.ts | 51 -- scripts/png-crop-benchmark/statistics.test.ts | 20 - scripts/png-crop-benchmark/statistics.ts | 34 -- vitest.config.ts | 2 - 37 files changed, 4099 deletions(-) delete mode 100644 scripts/ios-ax-bridge-spike/README.md delete mode 100644 scripts/ios-ax-bridge-spike/adapter.test.ts delete mode 100644 scripts/ios-ax-bridge-spike/adapter.ts delete mode 100644 scripts/ios-ax-bridge-spike/config.ts delete mode 100644 scripts/ios-ax-bridge-spike/corrected-markdown.ts delete mode 100644 scripts/ios-ax-bridge-spike/corrected-report.test.ts delete mode 100644 scripts/ios-ax-bridge-spike/corrected-report.ts delete mode 100644 scripts/ios-ax-bridge-spike/corrected-types.ts delete mode 100644 scripts/ios-ax-bridge-spike/guest-adapter.ts delete mode 100644 scripts/ios-ax-bridge-spike/guest-binary.test.ts delete mode 100644 scripts/ios-ax-bridge-spike/guest-binary.ts delete mode 100644 scripts/ios-ax-bridge-spike/guest-connection.ts delete mode 100644 scripts/ios-ax-bridge-spike/guest-process-metrics.test.ts delete mode 100644 scripts/ios-ax-bridge-spike/guest-process-metrics.ts delete mode 100644 scripts/ios-ax-bridge-spike/guest-wire.test.ts delete mode 100644 scripts/ios-ax-bridge-spike/guest-wire.ts delete mode 100644 scripts/ios-ax-bridge-spike/limits.test.ts delete mode 100644 scripts/ios-ax-bridge-spike/limits.ts delete mode 100644 scripts/ios-ax-bridge-spike/protocol.ts delete mode 100644 scripts/ios-ax-bridge-spike/targeted-evidence.ts delete mode 100644 scripts/ios-ax-bridge-spike/targeted-readiness.ts delete mode 100644 scripts/ios-ax-bridge-spike/targeted-relaunch.ts delete mode 100644 scripts/ios-ax-bridge-spike/targeted-request.ts delete mode 100644 scripts/ios-ax-bridge-spike/targeted-run.ts delete mode 100644 scripts/ios-ax-bridge-spike/types.ts delete mode 100644 scripts/png-crop-benchmark/README.md delete mode 100644 scripts/png-crop-benchmark/args.test.ts delete mode 100644 scripts/png-crop-benchmark/args.ts delete mode 100644 scripts/png-crop-benchmark/corpus.test.ts delete mode 100644 scripts/png-crop-benchmark/corpus.ts delete mode 100644 scripts/png-crop-benchmark/pipelines.ts delete mode 100644 scripts/png-crop-benchmark/report.ts delete mode 100644 scripts/png-crop-benchmark/run.ts delete mode 100644 scripts/png-crop-benchmark/statistics.test.ts delete mode 100644 scripts/png-crop-benchmark/statistics.ts diff --git a/package.json b/package.json index 040fd12169..f9db0f3d73 100644 --- a/package.json +++ b/package.json @@ -134,8 +134,6 @@ "bench:ios-snapshot": "node --experimental-strip-types scripts/ios-snapshot-benchmark/run.ts", "bench:ios-snapshot:deep-button": "node --experimental-strip-types scripts/ios-snapshot-benchmark/deep-button.ts", "bench:ios-snapshot:evidence": "node --experimental-strip-types scripts/ios-snapshot-benchmark/evidence.ts", - "bench:png-crop": "node --experimental-strip-types scripts/png-crop-benchmark/run.ts", - "bench:ios-ax-bridge:targeted": "node --experimental-strip-types scripts/ios-ax-bridge-spike/targeted-run.ts", "mutation:run": "node --experimental-strip-types scripts/mutation/run.ts", "mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run", "mutation:affected": "node --experimental-strip-types scripts/mutation/run.ts --affected", diff --git a/scripts/ios-ax-bridge-spike/README.md b/scripts/ios-ax-bridge-spike/README.md deleted file mode 100644 index 3a83436aaf..0000000000 --- a/scripts/ios-ax-bridge-spike/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# iOS Simulator AX bridge decision verifier - -This narrow harness supplies the decisive live evidence for [#2192](https://github.com/callstack/agent-device/issues/2192). It drives idb v1.5.2's in-Simulator `Resources/SimulatorFrameworkBridge` directly from Node over a private UNIX socket. It does not use `idb_companion`, gRPC, or Python, and it does not change production routing. - -The September 1 broad corpus is retained for its warm measurements and diagnostics. Its one-off Python runner and generated NO-GO reports were removed after the corrected contract made them obsolete. The corrected relaunch corpus is Node-direct. Raw artifacts are kept off-tree at immutable tag `evidence/ios-snapshot/44995806ea` (commit `f8b2fab28b8604f20094785c16e16a79fdc651a3`); their SHA-256 hashes are recorded in the evidence branch README. - -Obtain the guest executable from the official arm64 idb v1.5.2 release. The archive SHA-256 is `f17b718a513931705542a7fbfa9cfc11895ee191562c9ffd2343cf7f8254bc08`; `Resources/SimulatorFrameworkBridge` is `3545621d2dc98de32879ebac55e8b0c33dc8eb7cc2bfbc2d0d2d21a002c8de58`. The verifier hashes the supplied `--guest-bridge` before launching it and fails if it is not that binary. - -Fetch the broad input before rerunning: - -```sh -git fetch origin refs/tags/evidence/ios-snapshot/44995806ea -git show evidence/ios-snapshot/44995806ea:ios-simulator-ax-bridge-broad-268a90275.json.gz \ - > docs/evidence/ios-simulator-ax-bridge-2026-09-01-final.json.gz -shasum -a 256 docs/evidence/ios-simulator-ax-bridge-2026-09-01-final.json.gz -``` - -The expected broad artifact hash is `309f974b1dcb90768548a189f6af58b493b5d7b9d56a5bfad060d4335139eb7b`. - -Run the verifier from a clean commit using the dedicated Simulator with the fixture app installed: - -```sh -pnpm bench:ios-ax-bridge:targeted -- \ - --udid SIMULATOR_UDID \ - --guest-bridge /path/to/Resources/SimulatorFrameworkBridge -``` - -It captures five nonresident bootstrap samples after independently observing application readiness. It then captures 20 relaunches on each of the six representative screens through the Node-direct route; every timed read is paired with a separate readiness probe for the exact relaunched PID and expected screen anchor. Finally it exercises crash, timeout, cancellation, and stale-generation recovery. Successful reads record guest CPU time and resident memory, and the corrected report fails closed if those metrics are missing or exceed the declared bounds. diff --git a/scripts/ios-ax-bridge-spike/adapter.test.ts b/scripts/ios-ax-bridge-spike/adapter.test.ts deleted file mode 100644 index da5a8186ed..0000000000 --- a/scripts/ios-ax-bridge-spike/adapter.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { createGuestSimulatorFrameworkBridgeAdapter } from './guest-adapter.ts'; - -test('guest adapter fails closed when the guest bridge executable is not configured', async () => { - const adapter = createGuestSimulatorFrameworkBridgeAdapter({ repoRoot: '/repo' }); - const result = await adapter.acquireBatch([ - { - version: 1, - id: 'guest-unavailable', - candidate: 'guest-simulator-framework-bridge', - simulatorUdid: 'simulator', - state: 'warm', - screen: 'quiet', - limits: { - maxRequestBytes: 64 * 1024, - maxResponseBytes: 4 * 1024 * 1024, - maxNodes: 1500, - maxTraversalDepth: 12, - maxCpuMs: 2000, - maxMemoryBytes: 256 * 1024 * 1024, - maxDurationMs: 5000, - }, - }, - ]); - assert.deepEqual(result.responses[0]?.failure, { - kind: 'unsupported-mechanism', - code: 'guest-tool-unavailable', - }); -}); diff --git a/scripts/ios-ax-bridge-spike/adapter.ts b/scripts/ios-ax-bridge-spike/adapter.ts deleted file mode 100644 index 78a739d452..0000000000 --- a/scripts/ios-ax-bridge-spike/adapter.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { CandidateId, ResourceLimits, SpikeRequest, SpikeResponse } from './types.ts'; - -export type AcquisitionAdapter = Readonly<{ - candidate: CandidateId; - acquireBatch( - requests: readonly SpikeRequest[], - options?: Readonly<{ signal?: AbortSignal }>, - ): Promise>; - close?: () => Promise; - evidence?: Readonly<{ terminateReaderOnNextBatch?: () => void }>; -}>; - -export type AdapterOptions = Readonly<{ - limits?: ResourceLimits; - guestBridge?: string; -}>; diff --git a/scripts/ios-ax-bridge-spike/config.ts b/scripts/ios-ax-bridge-spike/config.ts deleted file mode 100644 index 4712101e87..0000000000 --- a/scripts/ios-ax-bridge-spike/config.ts +++ /dev/null @@ -1,87 +0,0 @@ -import path from 'node:path'; -import { resolveRepoRoot } from '../ios-snapshot-benchmark/host.ts'; -import { createBenchmarkStateRoot } from '../ios-snapshot-benchmark/state-ownership.ts'; -import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; -import type { ResourceLimits } from './types.ts'; - -class SpikeConfigurationError extends Error { - constructor(message: string) { - super(message); - this.name = 'SpikeConfigurationError'; - } -} - -export type SpikeConfig = Readonly<{ - repoRoot: string; - udid: string; - guestBridge: string; - stateDir: string; - derivedPath: string; - limits: ResourceLimits; - keepDevice: boolean; -}>; - -export function parseConfig(argv: readonly string[]): SpikeConfig { - const args = argv[0] === '--' ? argv.slice(1) : argv; - if (args.some((argument) => argument === '--help' || argument === '-h')) { - printHelp(); - process.exit(0); - } - const parsed = parseArguments(args); - const udid = required(parsed.values, '--udid'); - const guestBridge = required(parsed.values, '--guest-bridge'); - const stateDir = createBenchmarkStateRoot(); - return { - repoRoot: resolveRepoRoot(), - udid, - guestBridge, - stateDir, - derivedPath: path.join(stateDir, 'derived-data'), - limits: DEFAULT_SPIKE_LIMITS, - keepDevice: parsed.keepDevice, - }; -} - -function parseArguments(args: readonly string[]): { - values: ReadonlyMap; - keepDevice: boolean; -} { - const values = new Map(); - let keepDevice = false; - for (let index = 0; index < args.length; index += 1) { - const flag = args[index]; - if (flag === '--keep-device') { - keepDevice = true; - continue; - } - assertValueFlag(flag); - values.set(flag, readValue(args[index + 1], flag)); - index += 1; - } - return { values, keepDevice }; -} - -function assertValueFlag(flag: string | undefined): asserts flag is '--udid' | '--guest-bridge' { - if (flag !== '--udid' && flag !== '--guest-bridge') { - throw new SpikeConfigurationError(`Unknown option: ${String(flag)}`); - } -} - -function readValue(value: string | undefined, flag: string): string { - if (!value || value.startsWith('--')) { - throw new SpikeConfigurationError(`${flag} requires a value.`); - } - return value; -} - -function required(values: ReadonlyMap, flag: string): string { - const value = values.get(flag); - if (!value) throw new SpikeConfigurationError(`${flag} is required.`); - return value; -} - -function printHelp(): void { - process.stdout.write( - 'Usage: pnpm bench:ios-ax-bridge:targeted -- --udid --guest-bridge [--keep-device]\n', - ); -} diff --git a/scripts/ios-ax-bridge-spike/corrected-markdown.ts b/scripts/ios-ax-bridge-spike/corrected-markdown.ts deleted file mode 100644 index 03878e1683..0000000000 --- a/scripts/ios-ax-bridge-spike/corrected-markdown.ts +++ /dev/null @@ -1,126 +0,0 @@ -import type { CorrectedReport, GateResult, LatencySummary } from './corrected-types.ts'; - -export function renderCorrectedMarkdown(report: CorrectedReport): string { - const lines = [ - '# iOS Simulator AX bridge corrected evidence', - '', - `- Decision: **${report.decision}**`, - '- Interpretation: **maintainer-corrected**', - `- Revision: ${report.revision.commit} (${report.revision.branch})`, - `- Target: ${report.target.name} (${report.target.udid}, ${report.target.runtime})`, - `- Generated: ${report.generatedAt}`, - `- Immutable broad raw artifact: \`${report.sourceArtifact.path}\` (original ${report.sourceArtifact.originalDecision}; interpretation superseded to stretch-only; host client ${report.sourceArtifact.hostClient})`, - `- Narrow targeted raw artifact: \`${report.targetedArtifact.path}\` (host client ${report.guestMechanism.client})`, - `- Host at generation: load average ${report.host.loadAverage1m} on ${report.host.cpuCores} cores`, - '', - 'The broad raw corpus is preserved unchanged. Its old NO-GO used readiness-inclusive first-look and stretch thresholds; this report evaluates the corrected hard contract. The broad warm cells remain conservative upper bounds around the same in-Simulator reader. Relaunch uses the new Node-direct corpus below and does not rely on the legacy relaunch samples.', - '', - '## Evaluated guest mechanism', - '', - `- Guest reader: ${report.guestMechanism.implementation} ${report.guestMechanism.release} \`${report.guestMechanism.guestBinary}\` (observed SHA-256 \`${report.guestMechanism.guestBinarySha256}\`; required SHA-256 \`${report.guestMechanism.guestBinaryExpectedSha256 ?? 'not recorded'}\`) from \`${report.guestMechanism.companionArchive}\` (SHA-256 \`${report.guestMechanism.companionSha256}\`).`, - `- Transport: ${report.guestMechanism.transport}.`, - `- Traversal: ${report.guestMechanism.traversal}.`, - '', - '## Hard gates', - '', - '| Gate | Status | Target | Evidence |', - '|---|---|---|---|', - ...Object.entries(report.hardGates).map(([name, gate]) => gateLine(name, gate)), - '', - '## Readiness boundary and candidate-owned latency', - '', - 'Warm and relaunch timing starts at bridge acquisition after fixture/app readiness admission. Every relaunch row comes from the Node-direct route and is paired with a separate probe that observed the exact relaunched process generation and expected screen anchor. The old first-look value includes Simulator, app, daemon, and runner costs.', - '', - '| State | Screen | Samples | Readable | Ready generation | Candidate p50/p95 ms | Readiness p95 ms | Old first-look p95 ms | Generations |', - '|---|---|---:|---:|---:|---:|---:|---:|---:|', - ...report.readiness.map(readinessLine), - '', - '## Cold diagnostics', - '', - 'Cold and cold-cold first-look measurements remain visible for diagnosis, but are excluded from the candidate-owned hard verdict because they combine environment and readiness boundaries with bridge work.', - '', - '| State | Screen | Preparation p95 ms | First-look p95 ms | Interpretation |', - '|---|---|---:|---:|---|', - ...report.coldDiagnostics.map(coldDiagnosticLine), - '', - '## Nonresident bootstrap', - '', - `- ${report.hardGates.nonresidentBootstrap.evidence}.`, - `- ${report.hardGates.boundedResources.evidence}.`, - '- The timed boundary begins with no resident bridge and ends at the first usable guest tree. Before each timer the fixture app was relaunched and a throwaway probe bridge polled until the new generation answered with a tree (readiness), then exited.', - '', - '| Sample | Duration ms | CPU ms | RSS MiB | Usable tree | Nodes | Depth | Generation | Readiness ms | Attempts | Host load |', - '|---:|---:|---:|---:|---|---:|---:|---|---:|---:|---:|', - ...report.bootstrap.map(bootstrapLine), - '', - '## Live candidate recovery', - '', - `- ${report.hardGates.liveRecovery.evidence}.`, - '', - '| Operation | Observed failure | Recovery response | Recovered tree |', - '|---|---|---|---|', - ...report.liveRecovery.map(recoveryLine), - '', - '## Hierarchy', - '', - `- ${report.hardGates.hierarchy.evidence}.`, - `- Observed traversal depth: ${report.hierarchy.observedTraversalDepth}; depth complete: **${report.hierarchy.depthComplete}**; interpretation: ${report.hierarchy.interpretation}.`, - '', - '## Simulator preference control', - '', - `- ${report.hardGates.preferenceControl.evidence}.`, - '- The broad capture applied its accessibility preference changes only to the disposable benchmark Simulator before boot, verified fixture launch compatibility, and restored the prior preference files and Simulator state afterward.', - '', - '## Private-interface compatibility risk', - '', - `- ${report.compatibilityRisk.assessment}`, - `- Control: ${report.compatibilityRisk.control}`, - '', - '## Stretch findings', - '', - ...report.stretchFindings.map((finding) => `- ${finding}`), - '', - '## Production boundary', - '', - '- No production backend selection, fallback, runner-demand, open/relaunch, proxy, XCTest interaction, or public CLI changes were made.', - '- The corrected result is evidence for the #2192 decision boundary only; it does not start production routing.', - ]; - return `${lines.join('\n')}\n`; -} - -function gateLine(name: string, gate: GateResult): string { - return `| ${name} | **${gate.status}** | ${gate.target} | ${gate.evidence} |`; -} - -function readinessLine(summary: LatencySummary): string { - return `| ${summary.state} | ${summary.screen} | ${summary.samples} | ${summary.readableSamples} | ${summary.readinessObservedSamples} | ${formatMs(summary.candidateP50Ms)}/${formatMs(summary.candidateP95Ms)} | ${formatMs(summary.preparationP95Ms)} | ${formatMs(summary.firstLookP95Ms)} | ${summary.generationCount} |`; -} - -function coldDiagnosticLine(diagnostic: CorrectedReport['coldDiagnostics'][number]): string { - return `| ${diagnostic.state} | ${diagnostic.screen} | ${formatMs(diagnostic.preparationP95Ms)} | ${formatMs(diagnostic.firstLookP95Ms)} | excluded runner/app readiness costs |`; -} - -function bootstrapLine(sample: CorrectedReport['bootstrap'][number]): string { - const response = sample.response; - return `| ${sample.index} | ${sample.durationMs.toFixed(1)} | ${formatMs(response.metrics.cpuMs)} | ${formatMib(response.metrics.memoryBytes)} | ${sample.usableTree} | ${response.metrics.nodeCount} | ${response.metrics.maxTraversalDepth} | ${response.acquisition?.targetGeneration ?? '–'} | ${sample.readinessMs.toFixed(0)} | ${sample.readinessAttempts} | ${sample.host.loadAverage1m} |`; -} - -function recoveryLine(probe: CorrectedReport['liveRecovery'][number]): string { - const recovery = probe.recoveredResponse; - const status = recovery.ok ? 'ok' : 'failed'; - const nodes = recovery.acquisition?.nodes.length ?? 0; - return `| ${probe.operation} | ${failureText(probe.response.failure)} | ${status} | ${nodes} nodes |`; -} - -function failureText(failure: CorrectedReport['bootstrap'][number]['response']['failure']): string { - if (!failure) return 'none/none'; - return `${failure.kind}/${failure.code ?? 'none'}`; -} - -function formatMs(value: number | null): string { - return value === null ? '–' : value.toFixed(1); -} - -function formatMib(value: number | null): string { - return value === null ? '–' : (value / 1024 / 1024).toFixed(1); -} diff --git a/scripts/ios-ax-bridge-spike/corrected-report.test.ts b/scripts/ios-ax-bridge-spike/corrected-report.test.ts deleted file mode 100644 index b38291dffe..0000000000 --- a/scripts/ios-ax-bridge-spike/corrected-report.test.ts +++ /dev/null @@ -1,263 +0,0 @@ -import { describe, expect, test } from 'vitest'; -import { buildCorrectedReport } from './corrected-report.ts'; -import { renderCorrectedMarkdown } from './corrected-markdown.ts'; -import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; -import type { TargetedRawArtifact, TargetedRecoveryProbe } from './corrected-types.ts'; -import type { SpikeReport, SpikeResponse } from './types.ts'; - -const revision = { commit: 'abc123', branch: 'test', dirty: false } as const; -const mechanism = { - implementation: 'idb', - release: 'v1.5.2', - companionArchive: 'idb-companion.macos-arm64.tar.gz', - companionSha256: 'archive', - guestBinary: 'Resources/SimulatorFrameworkBridge', - guestBinaryExpectedSha256: 'guest', - guestBinarySha256: 'guest', - transport: 'socket', - traversal: 'tree', - client: 'node-direct-socket', -} as const; - -describe('corrected Simulator AX bridge report', () => { - test('counts retained generation evidence honestly and uses targeted readiness and resources', () => { - const source = broadReport(); - const targeted = targetedArtifact(); - const report = buildCorrectedReport({ - sourcePath: 'broad.json.gz', - source, - targetedPath: 'targeted.json.gz', - targeted, - }); - - expect( - report.readiness.every( - (cell) => cell.readinessObservedSamples === (cell.state === 'warm' ? 1 : 2), - ), - ).toBe(true); - expect(report.hardGates.relaunch.status).toBe('PASS'); - expect(report.hardGates.boundedResources.status).toBe('PASS'); - expect(report.hardGates.liveRecovery.status).toBe('PASS'); - expect(report.hardGates.hierarchy.status).toBe('PASS'); - expect(report.decision).toBe('GO'); - expect(renderCorrectedMarkdown(report)).toContain('Decision: **GO**'); - }); - - test('fails closed when a successful Node-direct read lacks resource metrics', () => { - const targeted = targetedArtifact(); - const first = targeted.bootstrap[0]!; - const report = buildCorrectedReport({ - sourcePath: 'broad.json.gz', - source: broadReport(), - targetedPath: 'targeted.json.gz', - targeted: { - ...targeted, - bootstrap: [ - { - ...first, - response: { - ...first.response, - metrics: { ...first.response.metrics, cpuMs: null, memoryBytes: null }, - }, - }, - ...targeted.bootstrap.slice(1), - ], - }, - }); - - expect(report.hardGates.boundedResources.status).toBe('FAIL'); - expect(report.decision).toBe('NO-GO'); - }); - - test('fails relaunch when a timed read is not paired to its expected generation', () => { - const targeted = targetedArtifact(); - const first = targeted.relaunch[0]!; - const report = buildCorrectedReport({ - sourcePath: 'broad.json.gz', - source: broadReport(), - targetedPath: 'targeted.json.gz', - targeted: { - ...targeted, - relaunch: [ - { - ...first, - response: successfulResponse('pid:999'), - }, - ...targeted.relaunch.slice(1), - ], - }, - }); - - expect(report.hardGates.relaunch.status).toBe('FAIL'); - expect(report.decision).toBe('NO-GO'); - }); - - test('fails relaunch when the configured corpus is incomplete', () => { - const targeted = targetedArtifact(); - const report = buildCorrectedReport({ - sourcePath: 'broad.json.gz', - source: broadReport(), - targetedPath: 'targeted.json.gz', - targeted: { ...targeted, relaunch: targeted.relaunch.slice(1) }, - }); - - expect(report.hardGates.relaunch.status).toBe('FAIL'); - expect(report.hardGates.relaunch.evidence).toContain('1/2 Node-direct samples'); - }); -}); - -function broadReport(): SpikeReport { - const samples = [ - { ok: true, firstTree: 'readable', wallClockMs: 40, target: 'pid:1' }, - { ok: true, firstTree: 'readable', wallClockMs: 60 }, - ] as const; - return { - revision, - guestMechanism: mechanism, - target: { udid: 'sim', name: 'simulator', runtime: 'iOS' }, - toolchain: { - node: 'node', - pnpm: 'pnpm', - xcode: 'xcode', - simctl: 'simctl', - os: 'macOS', - arch: 'arm64', - }, - cells: (['warm', 'relaunch'] as const).map((state) => ({ - candidate: 'guest-simulator-framework-bridge', - state, - screen: 'list', - acquisitionSamples: samples.map((sample) => ({ - ok: sample.ok, - firstTree: sample.firstTree, - wallClockMs: sample.wallClockMs, - ...(sample.target - ? { acquisition: { ...acquisition(), targetGeneration: sample.target } } - : {}), - })), - })), - decisionReasons: [], - preferenceEvidence: { - applied: true, - restored: true, - fixtureLaunchCompatible: true, - simulatorStateBefore: 'Shutdown', - diffs: [ - { - changes: [ - { key: 'AutomationEnabled', before: 0, after: true }, - { key: 'IgnoreAXServerEntitlements', after: true }, - ], - }, - ], - }, - }; -} - -function targetedArtifact(): TargetedRawArtifact { - const bootstrap = Array.from({ length: 5 }, (_, offset) => { - const appPid = 100 + offset; - return { - index: offset + 1, - durationMs: 100, - usableTree: true, - response: successfulResponse(`pid:${appPid}`), - stderr: '', - appPid, - readinessMs: 50, - readinessAttempts: 1, - host: { loadAverage1m: 1, cpuCores: 12 }, - }; - }); - const operations = ['process-crash', 'timeout', 'cancelled', 'stale-generation'] as const; - const recovery: TargetedRecoveryProbe[] = operations.map((operation) => ({ - operation, - request: request(`request-${operation}`), - response: { - ...successfulResponse('pid:100'), - ok: false, - acquisition: undefined, - failure: { kind: operation, code: 'probe' }, - }, - recoveredResponse: successfulResponse('pid:104'), - })); - const relaunch = Array.from({ length: 2 }, (_, offset) => { - const appPid = 200 + offset; - return { - index: offset + 1, - screen: 'list' as const, - expectedAnchor: 'Item 20', - appPid, - readinessMs: 40, - readinessAttempts: 1, - durationMs: 80, - response: successfulResponse(`pid:${appPid}`), - stderr: '', - }; - }); - return { - schemaVersion: 'ios-simulator-ax-bridge-targeted.v3', - generatedAt: '2026-09-03T00:00:00.000Z', - revision, - command: 'targeted', - sourceArtifact: { path: 'broad.json.gz', revision, hostClient: 'legacy' }, - target: { udid: 'sim', name: 'simulator', runtime: 'iOS' }, - toolchain: { - node: 'node', - pnpm: 'pnpm', - xcode: 'xcode', - simctl: 'simctl', - os: 'macOS', - arch: 'arm64', - }, - host: { loadAverage1m: 1, cpuCores: 12 }, - guestMechanism: mechanism, - limits: DEFAULT_SPIKE_LIMITS, - config: { states: ['warm', 'relaunch'], screens: ['list'], samples: 2, bootstrapSamples: 5 }, - bootstrap, - relaunch, - recovery, - }; -} - -function successfulResponse(generation: string): SpikeResponse { - return { - version: 1, - id: 'response', - candidate: 'guest-simulator-framework-bridge', - ok: true, - acquisition: { ...acquisition(), targetGeneration: generation }, - metrics: { - requestBytes: 100, - responseBytes: 1_000, - nodeCount: 2, - maxTraversalDepth: 1, - cpuMs: 20, - memoryBytes: 10_000, - durationMs: 100, - }, - }; -} - -function acquisition() { - return { - targetId: 'simulator:sim', - targetGeneration: 'pid:100', - nodes: [{ id: 'root' }, { id: 'child', parentId: 'root' }], - viewport: { kind: 'reported', rect: { x: 0, y: 0, width: 100, height: 100 } }, - truncated: false, - residue: [], - } as const; -} - -function request(id: string) { - return { - version: 1, - id, - candidate: 'guest-simulator-framework-bridge', - simulatorUdid: 'sim', - state: 'warm', - screen: 'list', - limits: DEFAULT_SPIKE_LIMITS, - } as const; -} diff --git a/scripts/ios-ax-bridge-spike/corrected-report.ts b/scripts/ios-ax-bridge-spike/corrected-report.ts deleted file mode 100644 index 561e8f2046..0000000000 --- a/scripts/ios-ax-bridge-spike/corrected-report.ts +++ /dev/null @@ -1,453 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import { gunzipSync, gzipSync } from 'node:zlib'; -import type { - CorrectedReport, - GateResult, - LatencySummary, - TargetedRawArtifact, -} from './corrected-types.ts'; -import { renderCorrectedMarkdown } from './corrected-markdown.ts'; -import type { SpikeCell, SpikeReport } from './types.ts'; - -export function readSpikeReport(filePath: string): SpikeReport { - return JSON.parse(gunzipSync(fs.readFileSync(filePath)).toString('utf8')) as SpikeReport; -} - -export function readTargetedArtifact(filePath: string): TargetedRawArtifact { - return JSON.parse(gunzipSync(fs.readFileSync(filePath)).toString('utf8')) as TargetedRawArtifact; -} - -export function buildCorrectedReport(options: { - sourcePath: string; - source: SpikeReport; - targetedPath: string; - targeted: TargetedRawArtifact; -}): CorrectedReport { - const readiness = [ - ...options.source.cells - .filter( - (cell) => cell.candidate === 'guest-simulator-framework-bridge' && cell.state === 'warm', - ) - .map(summarizeLatency), - ...summarizeTargetedRelaunch(options.targeted), - ]; - const coldDiagnostics = options.source.cells - .filter( - (cell) => - cell.candidate === 'guest-simulator-framework-bridge' && - (cell.state === 'cold' || cell.state === 'cold-cold'), - ) - .map(summarizeColdDiagnostic); - const hierarchy = hierarchyEvidence(options.targeted); - const hardGates = { - warm: latencyGate(readiness, 'warm', 'p50 <300 ms and p95 <500 ms per screen', false), - relaunch: relaunchGate(readiness, options.targeted), - nonresidentBootstrap: bootstrapGate(options.targeted), - boundedResources: resourceGate(options.targeted), - liveRecovery: recoveryGate(options.targeted), - hierarchy: hierarchy.gate, - preferenceControl: preferenceGate(options.source), - } as const; - const failedGates = Object.entries(hardGates).filter(([, gate]) => gate.status === 'FAIL'); - return { - schemaVersion: 'ios-simulator-ax-bridge-corrected.v3', - interpretation: 'maintainer-corrected', - generatedAt: new Date().toISOString(), - revision: options.targeted.revision, - sourceArtifact: { - path: options.sourcePath, - revision: options.source.revision, - originalDecision: 'NO-GO', - interpretation: 'superseded-stretch-only', - hostClient: options.targeted.sourceArtifact.hostClient, - }, - targetedArtifact: { - path: options.targetedPath, - revision: options.targeted.revision, - }, - target: options.targeted.target, - toolchain: options.targeted.toolchain, - host: options.targeted.host, - guestMechanism: options.targeted.guestMechanism, - readiness, - hardGates, - coldDiagnostics, - stretchFindings: [ - ...options.source.decisionReasons.map((reason) => `Original broad-run finding: ${reason}`), - 'Cold and cold-cold first-look measurements include Simulator, app, daemon, and runner readiness costs; they are diagnostics, not candidate-owned hard gates.', - 'The former warm 75/150 ms and relaunch 250 ms thresholds are stretch findings under the corrected contract.', - `Nonresident bootstrap samples were taken on a host with 1-minute load average ${options.targeted.host.loadAverage1m} on ${options.targeted.host.cpuCores} cores; per-sample load is recorded with each sample.`, - ], - decision: failedGates.length === 0 ? 'GO' : 'NO-GO', - decisionReasons: failedGates.map( - ([name, gate]) => `${name} hard gate failed: ${gate.evidence}.`, - ), - liveRecovery: options.targeted.recovery, - bootstrap: options.targeted.bootstrap, - hierarchy: hierarchy.value, - compatibilityRisk: { - interface: 'private-idb-simulator-guest', - assessment: - 'SimulatorFrameworkBridge and its accessibility wire protocol are private idb/Apple implementation details with no compatibility guarantee.', - control: - 'Pin the official idb release and observed guest SHA-256, keep this route Simulator-only behind the acquisition adapter, and re-run this verifier for every idb, Xcode, or Simulator runtime change before production adoption.', - }, - productionBoundary: 'no-production-routing-changes', - }; -} - -export function writeCorrectedReport(outputPath: string, report: CorrectedReport): void { - fs.mkdirSync(path.dirname(outputPath), { recursive: true }); - fs.writeFileSync(outputPath, gzipSync(`${JSON.stringify(report)}\n`, { level: 9 })); - fs.writeFileSync(markdownPath(outputPath), renderCorrectedMarkdown(report)); -} - -function markdownPath(outputPath: string): string { - const replaced = outputPath.replace(/\.json(?:\.gz)?$/u, '.md'); - return replaced === outputPath ? `${outputPath}.md` : replaced; -} - -function summarizeLatency(cell: SpikeCell): LatencySummary { - const samples = cell.acquisitionSamples; - const readable = samples.filter((sample) => sample.ok && sample.firstTree === 'readable'); - const observedGenerations = readable.flatMap((sample) => - observedGeneration(sample.acquisition?.targetGeneration), - ); - return { - state: cell.state as 'warm' | 'relaunch', - screen: cell.screen, - samples: samples.length, - readableSamples: readable.length, - readinessObservedSamples: observedGenerations.length, - generationCount: new Set(observedGenerations).size, - candidateP50Ms: optionalPercentile( - readable.map((sample) => sample.wallClockMs), - 50, - ), - candidateP95Ms: optionalPercentile( - readable.map((sample) => sample.wallClockMs), - 95, - ), - preparationP95Ms: optionalPercentile( - readable.map((sample) => sample.preparationMs), - 95, - ), - firstLookP95Ms: optionalPercentile( - readable.map((sample) => sample.firstLookMs), - 95, - ), - }; -} - -function summarizeColdDiagnostic(cell: SpikeCell): CorrectedReport['coldDiagnostics'][number] { - const readable = cell.acquisitionSamples.filter( - (sample) => sample.ok && sample.firstTree === 'readable', - ); - return { - state: cell.state as 'cold' | 'cold-cold', - screen: cell.screen, - preparationP95Ms: optionalPercentile( - readable.map((sample) => sample.preparationMs), - 95, - ), - firstLookP95Ms: optionalPercentile( - readable.map((sample) => sample.firstLookMs), - 95, - ), - interpretation: 'excluded-runner-and-app-readiness-costs', - }; -} - -function summarizeTargetedRelaunch(targeted: TargetedRawArtifact): LatencySummary[] { - return targeted.config.screens.map((screen) => { - const samples = targeted.relaunch.filter((sample) => sample.screen === screen); - const readable = samples.filter( - (sample) => - sample.response.ok && - sample.response.acquisition !== undefined && - sample.response.acquisition.nodes.length > 0 && - sample.response.acquisition.targetGeneration === `pid:${sample.appPid}`, - ); - const observedGenerations = readable.flatMap((sample) => - observedGeneration(sample.response.acquisition?.targetGeneration), - ); - return { - state: 'relaunch', - screen, - samples: samples.length, - readableSamples: readable.length, - readinessObservedSamples: readable.filter((sample) => sample.readinessAttempts > 0).length, - generationCount: new Set(observedGenerations).size, - candidateP50Ms: optionalPercentile( - readable.map((sample) => sample.durationMs), - 50, - ), - candidateP95Ms: optionalPercentile( - readable.map((sample) => sample.durationMs), - 95, - ), - preparationP95Ms: optionalPercentile( - readable.map((sample) => sample.readinessMs), - 95, - ), - firstLookP95Ms: optionalPercentile( - readable.map((sample) => sample.readinessMs + sample.durationMs), - 95, - ), - }; - }); -} - -function latencyGate( - readiness: readonly LatencySummary[], - state: 'warm' | 'relaunch', - target: string, - requireReadiness: boolean, -): GateResult { - const cells = readiness.filter((summary) => summary.state === state); - const passed = cells.filter((summary) => latencyPassed(summary, requireReadiness)); - return { - status: cells.length > 0 && passed.length === cells.length ? 'PASS' : 'FAIL', - target, - evidence: `${passed.length}/${cells.length} ${state} screen cells passed; ${cells.map(formatLatency).join('; ') || 'no cells'}`, - }; -} - -function relaunchGate( - readiness: readonly LatencySummary[], - targeted: TargetedRawArtifact, -): GateResult { - const latency = latencyGate( - readiness, - 'relaunch', - 'p95 <500 ms per representative screen with every read paired to its expected generation', - true, - ); - const complete = readiness - .filter((summary) => summary.state === 'relaunch') - .every((summary) => summary.samples === targeted.config.samples); - const expectedSampleCount = targeted.config.screens.length * targeted.config.samples; - const representativeScreens = new Set(targeted.relaunch.map((sample) => sample.screen)); - const corpusComplete = - targeted.relaunch.length === expectedSampleCount && - targeted.config.screens.every((screen) => representativeScreens.has(screen)); - return { - status: latency.status === 'PASS' && complete && corpusComplete ? 'PASS' : 'FAIL', - target: 'p95 <500 ms per screen after independently observed new-generation readiness', - evidence: `${latency.evidence}; ${targeted.relaunch.length}/${expectedSampleCount} Node-direct samples across ${representativeScreens.size}/${targeted.config.screens.length} screens`, - }; -} - -function preferenceGate(source: SpikeReport): GateResult { - const evidence = source.preferenceEvidence ?? missingPreferenceEvidence(); - const changes = evidence.diffs.flatMap((diff) => diff.changes); - const required = ['AutomationEnabled', 'IgnoreAXServerEntitlements']; - const applied = required.filter((key) => hasEnabledPreference(changes, key)); - const passed = [evidence.applied, evidence.restored, applied.length === required.length].every( - Boolean, - ); - return { - status: gateStatus(passed), - target: 'task-owned Simulator accessibility preferences applied preboot and restored', - evidence: `applied=${String(evidence.applied)}; restored=${String(evidence.restored)}; enabled keys=${applied.join(', ')}; fixture launch compatible=${String(evidence.fixtureLaunchCompatible)}`, - }; -} - -function missingPreferenceEvidence(): NonNullable { - return { - applied: false, - restored: false, - fixtureLaunchCompatible: null, - simulatorStateBefore: 'unknown', - diffs: [], - }; -} - -function gateStatus(passed: boolean): GateResult['status'] { - return passed ? 'PASS' : 'FAIL'; -} - -function hasEnabledPreference( - changes: NonNullable['diffs'][number]['changes'], - key: string, -): boolean { - return changes.some((change) => change.key === key && change.after === true); -} - -function bootstrapGate(targeted: TargetedRawArtifact): GateResult { - const usable = targeted.bootstrap.filter((sample) => sample.usableTree); - const p95 = optionalPercentile( - targeted.bootstrap.map((sample) => sample.durationMs), - 95, - ); - const passed = [ - targeted.bootstrap.length === 5, - usable.length === targeted.bootstrap.length, - p95 !== null, - p95 !== null && p95 < 2_000, - ].every(Boolean); - return { - status: passed ? 'PASS' : 'FAIL', - target: 'nonresident companion + reader bootstrap and first usable tree p95 <2,000 ms', - evidence: `${usable.length}/${targeted.bootstrap.length} usable trees; p95=${formatMs(p95)}; the timer covered guest spawn, socket connect, and the first tree after a throwaway probe observed the relaunched app's readiness (readiness p95=${formatMs( - optionalPercentile( - targeted.bootstrap.map((sample) => sample.readinessMs), - 95, - ), - )}), with no resident bridge, xcodebuild, XCTest, or agent-device runner in the timed path`, - }; -} - -function recoveryGate(targeted: TargetedRawArtifact): GateResult { - const passed = targeted.recovery.filter( - (probe) => - recoveryOutcomeMatches(probe) && - probe.recoveredResponse.ok === true && - probe.recoveredResponse.acquisition?.nodes.length !== 0, - ); - return { - status: targeted.recovery.length === 4 && passed.length === 4 ? 'PASS' : 'FAIL', - target: 'live crash, timeout, cancellation, and honest target-generation handling', - evidence: `${passed.length}/${targeted.recovery.length} probes returned a typed failure or typed unavailable-generation residue and a usable recovered response`, - }; -} - -function resourceGate(targeted: TargetedRawArtifact): GateResult { - const responses = [ - ...targeted.bootstrap.map((sample) => sample.response), - ...targeted.relaunch.map((sample) => sample.response), - ...targeted.recovery.map((probe) => probe.recoveredResponse), - ].filter((response) => response.ok); - const measured = responses.filter( - (response) => response.metrics.cpuMs !== null && response.metrics.memoryBytes !== null, - ); - const withinBounds = measured.filter( - (response) => - response.metrics.cpuMs! <= targeted.limits.maxCpuMs && - response.metrics.memoryBytes! <= targeted.limits.maxMemoryBytes, - ); - const maxCpuMs = Math.max(0, ...measured.map((response) => response.metrics.cpuMs!)); - const maxMemoryBytes = Math.max(0, ...measured.map((response) => response.metrics.memoryBytes!)); - return { - status: responses.length > 0 && withinBounds.length === responses.length ? 'PASS' : 'FAIL', - target: `guest CPU <=${targeted.limits.maxCpuMs} ms and RSS <=${targeted.limits.maxMemoryBytes} bytes per successful read`, - evidence: `${withinBounds.length}/${responses.length} successful reads measured within bounds; max CPU=${maxCpuMs.toFixed(1)} ms; max RSS=${String(maxMemoryBytes)} bytes`, - }; -} - -function recoveryOutcomeMatches(probe: TargetedRawArtifact['recovery'][number]): boolean { - const failure = probe.response.failure; - if (failure) return failure.kind === probe.operation; - if (probe.operation !== 'stale-generation') return false; - return hasUnavailableGeneration(probe.response); -} - -function hasUnavailableGeneration( - response: TargetedRawArtifact['recovery'][number]['response'], -): boolean { - const acquisition = response.acquisition; - if (!acquisition) return false; - return [ - response.ok, - acquisition.targetGeneration === null, - acquisition.residue.some(isUnavailableGeneration), - ].every(Boolean); -} - -function isUnavailableGeneration( - residue: NonNullable< - TargetedRawArtifact['bootstrap'][number]['response']['acquisition'] - >['residue'][number], -): boolean { - return residue.kind === 'unavailable-fact' && residue.fact === 'generation'; -} - -/** - * Hierarchy is a hard fact, not a presentation: the guest must either return structural depth with - * an honest truncation flag, or type its absence as provider-pruned residue. A flat tree claiming - * completeness fails. - */ -function hierarchyEvidence(targeted: TargetedRawArtifact): { - gate: GateResult; - value: CorrectedReport['hierarchy']; -} { - const usable = targeted.bootstrap.filter((sample) => sample.usableTree); - const depth = Math.max(0, ...usable.map((sample) => sample.response.metrics.maxTraversalDepth)); - const truncated = usable.some((sample) => sample.response.acquisition?.truncated === true); - const typedFlat = usable.some((sample) => - (sample.response.acquisition?.residue ?? []).some( - (residue) => residue.kind === 'provider-pruned' && residue.fields.includes('depth'), - ), - ); - if (usable.length > 0 && depth > 0) { - return { - gate: { - status: 'PASS', - target: - 'structural hierarchy acquired with typed truncation, or its absence typed as residue', - evidence: `nested tree with traversal depth ${depth} in ${usable.length}/${targeted.bootstrap.length} samples; truncated=${truncated}`, - }, - value: { - observedTraversalDepth: depth, - depthComplete: !truncated, - interpretation: 'nested-tree', - }, - }; - } - return { - gate: { - status: typedFlat ? 'PASS' : 'FAIL', - target: - 'structural hierarchy acquired with typed truncation, or its absence typed as residue', - evidence: typedFlat - ? 'flat response with typed provider-pruned/depth residue; depth is not treated as complete' - : 'no hierarchy and no typed residue observed', - }, - value: { - observedTraversalDepth: 0, - depthComplete: false, - interpretation: usable.length === 0 ? 'not-observed' : 'flat-provider-response', - }, - }; -} - -function formatLatency(summary: LatencySummary): string { - return `${summary.screen} p50/p95=${formatMs(summary.candidateP50Ms)}/${formatMs(summary.candidateP95Ms)} ready=${summary.readinessObservedSamples}/${summary.samples}`; -} - -function optionalPercentile( - values: readonly (number | undefined)[], - percentage: number, -): number | null { - const finite = values.filter( - (value): value is number => typeof value === 'number' && Number.isFinite(value), - ); - if (finite.length === 0) return null; - return percentile(finite, percentage); -} - -function observedGeneration(value: string | null | undefined): readonly string[] { - return typeof value === 'string' ? [value] : []; -} - -function latencyPassed(summary: LatencySummary, requireReadiness: boolean): boolean { - return [ - summary.readableSamples === summary.samples, - !requireReadiness || summary.readinessObservedSamples === summary.samples, - summary.candidateP50Ms !== null, - summary.candidateP50Ms !== null && summary.candidateP50Ms < 300, - summary.candidateP95Ms !== null, - summary.candidateP95Ms !== null && summary.candidateP95Ms < 500, - ].every(Boolean); -} - -function formatMs(value: number | null): string { - return value === null ? '–' : `${value.toFixed(1)} ms`; -} - -function percentile(values: readonly number[], percentage: number): number { - const sorted = [...values].sort((left, right) => left - right); - const index = Math.ceil((percentage / 100) * sorted.length) - 1; - return sorted[Math.max(0, index)]!; -} diff --git a/scripts/ios-ax-bridge-spike/corrected-types.ts b/scripts/ios-ax-bridge-spike/corrected-types.ts deleted file mode 100644 index 8c7bb6c32b..0000000000 --- a/scripts/ios-ax-bridge-spike/corrected-types.ts +++ /dev/null @@ -1,144 +0,0 @@ -import type { - ResourceLimits, - SpikeCell, - SpikeReport, - SpikeRequest, - SpikeResponse, -} from './types.ts'; - -const CORRECTED_SCHEMA_VERSION = 'ios-simulator-ax-bridge-corrected.v3' as const; -export const TARGETED_SCHEMA_VERSION = 'ios-simulator-ax-bridge-targeted.v3' as const; - -export type TargetedRevision = SpikeReport['revision']; - -export type HostLoad = Readonly<{ loadAverage1m: number; cpuCores: number }>; - -export type TargetedBootstrapSample = Readonly<{ - index: number; - /** Candidate-owned: fresh guest spawn + connect + first usable tree, after readiness. */ - durationMs: number; - usableTree: boolean; - response: SpikeResponse; - stderr: string; - /** Fixture-owned: app relaunch until a throwaway probe first read a tree; not charged. */ - appPid: number; - readinessMs: number; - readinessAttempts: number; - host: HostLoad; -}>; - -export type TargetedRecoveryProbe = Readonly<{ - operation: 'process-crash' | 'timeout' | 'cancelled' | 'stale-generation'; - request: SpikeRequest; - response: SpikeResponse; - recoveredResponse: SpikeResponse; -}>; - -export type TargetedRelaunchSample = Readonly<{ - index: number; - screen: SpikeCell['screen']; - expectedAnchor: string; - appPid: number; - readinessMs: number; - readinessAttempts: number; - durationMs: number; - response: SpikeResponse; - stderr: string; -}>; - -export type TargetedRawArtifact = Readonly<{ - schemaVersion: typeof TARGETED_SCHEMA_VERSION; - generatedAt: string; - revision: TargetedRevision; - command: string; - sourceArtifact: Readonly<{ - path: string; - revision: TargetedRevision; - hostClient: string; - }>; - target: SpikeReport['target']; - toolchain: SpikeReport['toolchain']; - host: HostLoad; - guestMechanism: SpikeReport['guestMechanism']; - limits: ResourceLimits; - config: Readonly<{ - states: readonly SpikeCell['state'][]; - screens: readonly SpikeCell['screen'][]; - samples: number; - bootstrapSamples: number; - }>; - bootstrap: readonly TargetedBootstrapSample[]; - relaunch: readonly TargetedRelaunchSample[]; - recovery: readonly TargetedRecoveryProbe[]; -}>; - -export type LatencySummary = Readonly<{ - state: 'warm' | 'relaunch'; - screen: SpikeCell['screen']; - samples: number; - readableSamples: number; - readinessObservedSamples: number; - generationCount: number; - candidateP50Ms: number | null; - candidateP95Ms: number | null; - preparationP95Ms: number | null; - firstLookP95Ms: number | null; -}>; - -export type GateResult = Readonly<{ - status: 'PASS' | 'FAIL'; - target: string; - evidence: string; -}>; - -export type CorrectedReport = Readonly<{ - schemaVersion: typeof CORRECTED_SCHEMA_VERSION; - interpretation: 'maintainer-corrected'; - generatedAt: string; - revision: TargetedRevision; - sourceArtifact: Readonly<{ - path: string; - revision: TargetedRevision; - originalDecision: 'NO-GO'; - interpretation: 'superseded-stretch-only'; - hostClient: string; - }>; - targetedArtifact: Readonly<{ path: string; revision: TargetedRevision }>; - target: SpikeReport['target']; - toolchain: SpikeReport['toolchain']; - host: HostLoad; - guestMechanism: SpikeReport['guestMechanism']; - readiness: readonly LatencySummary[]; - hardGates: Readonly<{ - warm: GateResult; - relaunch: GateResult; - nonresidentBootstrap: GateResult; - boundedResources: GateResult; - liveRecovery: GateResult; - hierarchy: GateResult; - preferenceControl: GateResult; - }>; - coldDiagnostics: readonly Readonly<{ - state: 'cold-cold' | 'cold'; - screen: SpikeCell['screen']; - preparationP95Ms: number | null; - firstLookP95Ms: number | null; - interpretation: 'excluded-runner-and-app-readiness-costs'; - }>[]; - stretchFindings: readonly string[]; - decision: 'GO' | 'NO-GO'; - decisionReasons: readonly string[]; - liveRecovery: readonly TargetedRecoveryProbe[]; - bootstrap: readonly TargetedBootstrapSample[]; - hierarchy: Readonly<{ - observedTraversalDepth: number; - depthComplete: boolean; - interpretation: 'nested-tree' | 'flat-provider-response' | 'not-observed'; - }>; - compatibilityRisk: Readonly<{ - interface: 'private-idb-simulator-guest'; - assessment: string; - control: string; - }>; - productionBoundary: 'no-production-routing-changes'; -}>; diff --git a/scripts/ios-ax-bridge-spike/guest-adapter.ts b/scripts/ios-ax-bridge-spike/guest-adapter.ts deleted file mode 100644 index b5da0ec2c0..0000000000 --- a/scripts/ios-ax-bridge-spike/guest-adapter.ts +++ /dev/null @@ -1,237 +0,0 @@ -import fs from 'node:fs'; -import { performance } from 'node:perf_hooks'; -import { GuestConnection, GuestConnectionError, processAlive } from './guest-connection.ts'; -import { processUsageDelta } from './guest-process-metrics.ts'; -import { - acquisitionFromEnvelope, - encodeGuestFrame, - failureFromEnvelope, - guestDescribeRequest, - GuestWireError, - isTargetNotReady, - type GuestEnvelope, -} from './guest-wire.ts'; -import { DEFAULT_SPIKE_LIMITS, validateRawAcquisition } from './limits.ts'; -import { failureResponse } from './protocol.ts'; -import type { AcquisitionAdapter, AdapterOptions } from './adapter.ts'; -import type { ResourceLimits, SpikeFailure, SpikeRequest, SpikeResponse } from './types.ts'; - -const CANDIDATE = 'guest-simulator-framework-bridge' as const; -const TARGET_NOT_READY_RETRY_MS = 150; -const TARGET_NOT_READY_RETRIES = 2; - -export function createGuestSimulatorFrameworkBridgeAdapter( - options: AdapterOptions, -): AcquisitionAdapter { - const limits = options.limits ?? DEFAULT_SPIKE_LIMITS; - if (!options.guestBridge || !fs.existsSync(options.guestBridge)) { - return unavailableAdapter('guest-tool-unavailable'); - } - const session = new GuestSession(options.guestBridge, limits); - return { - candidate: CANDIDATE, - acquireBatch: (requests, acquireOptions) => session.acquireBatch(requests, acquireOptions), - close: () => session.close(), - evidence: { - terminateReaderOnNextBatch: () => session.killGuestOnNextRequestForEvidence(), - }, - }; -} - -function unavailableAdapter(code: string): AcquisitionAdapter { - return { - candidate: CANDIDATE, - async acquireBatch(requests) { - return { - responses: requests.map((request) => - failureResponse(request, { kind: 'unsupported-mechanism', code }), - ), - stderr: '', - }; - }, - }; -} - -class GuestSession { - private readonly connection: GuestConnection; - private readonly limits: ResourceLimits; - private closed = false; - private serial: Promise = Promise.resolve(); - - constructor(bridgePath: string, limits: ResourceLimits) { - this.limits = limits; - this.connection = new GuestConnection(bridgePath, limits.maxResponseBytes); - } - - acquireBatch( - requests: readonly SpikeRequest[], - options: Readonly<{ signal?: AbortSignal }> = {}, - ): Promise<{ responses: readonly SpikeResponse[]; stderr: string }> { - const operation = this.serial.then(() => this.execute(requests, options.signal)); - this.serial = operation.catch(() => undefined); - return operation; - } - - async close(): Promise { - this.closed = true; - await this.connection.close(); - } - - killGuestOnNextRequestForEvidence(): void { - this.connection.killOnNextRequest(); - } - - private async execute( - requests: readonly SpikeRequest[], - signal: AbortSignal | undefined, - ): Promise<{ responses: readonly SpikeResponse[]; stderr: string }> { - const responses: SpikeResponse[] = []; - for (const request of requests) { - const deadline = performance.now() + request.limits.maxDurationMs; - responses.push(await this.acquire(request, deadline, signal)); - } - return { responses, stderr: this.connection.takeLog() }; - } - - private async acquire( - request: SpikeRequest, - deadline: number, - signal: AbortSignal | undefined, - ): Promise { - const started = performance.now(); - const frame = encodeGuestFrame(guestDescribeRequest(request, request.limits)); - if (frame.length > request.limits.maxRequestBytes) { - return failureResponse(request, { - kind: 'transport-failure', - code: 'request-limit-exceeded', - }); - } - try { - return await this.acquireConnected(request, frame, deadline, started, signal); - } catch (error) { - return failedGuestRequest(request, frame.length, started, error); - } - } - - private async acquireConnected( - request: SpikeRequest, - frame: Buffer, - deadline: number, - started: number, - signal: AbortSignal | undefined, - ): Promise { - assertRequestActive(signal, this.closed); - const wasConnected = await this.connection.ensureConnected( - request.simulatorUdid, - deadline, - signal, - ); - const before = wasConnected ? this.connection.processSample() : undefined; - const { envelope, responseBytes } = await this.readWithReadinessRetry(frame, deadline, signal); - const resources = processUsageDelta(before, this.connection.processSample()); - return this.responseFor(request, envelope, { - requestBytes: frame.length, - responseBytes, - durationMs: performance.now() - started, - cpuMs: resources?.cpuMs ?? null, - memoryBytes: resources?.memoryBytes ?? null, - }); - } - - private async readWithReadinessRetry( - frame: Buffer, - deadline: number, - signal: AbortSignal | undefined, - ): Promise<{ envelope: GuestEnvelope; responseBytes: number }> { - let attempt = await this.connection.roundTrip(frame, deadline, signal); - for (let retry = 0; retry < TARGET_NOT_READY_RETRIES; retry += 1) { - if (attempt.envelope.ok === true || !isTargetNotReady(attempt.envelope)) break; - if (performance.now() + TARGET_NOT_READY_RETRY_MS * 2 > deadline) break; - await sleep(TARGET_NOT_READY_RETRY_MS); - attempt = await this.connection.roundTrip(frame, deadline, signal); - } - return attempt; - } - - private responseFor( - request: SpikeRequest, - envelope: GuestEnvelope, - metrics: { - requestBytes: number; - responseBytes: number; - durationMs: number; - cpuMs: number | null; - memoryBytes: number | null; - }, - ): SpikeResponse { - if (envelope.ok !== true) { - return failureResponse( - request, - failureFromEnvelope(envelope, request, processAlive), - metrics, - ); - } - const parsed = acquisitionFromEnvelope(envelope, request, request.limits); - if ('kind' in parsed) return failureResponse(request, parsed, metrics); - const validated = validateRawAcquisition(parsed.acquisition, request.limits); - if (!validated.ok) { - return failureResponse(request, { kind: 'malformed-tree', code: validated.code }, metrics); - } - const resourceFailure = resourceLimitFailure(metrics, request.limits); - if (resourceFailure) return failureResponse(request, resourceFailure, metrics); - return { - version: 1, - id: request.id, - candidate: request.candidate, - ok: true, - acquisition: parsed.acquisition, - metrics: { - ...metrics, - nodeCount: parsed.acquisition.nodes.length, - maxTraversalDepth: validated.maxTraversalDepth, - }, - }; - } -} - -function asGuestError(error: unknown): GuestConnectionError { - if (error instanceof GuestConnectionError) return error; - if (error instanceof GuestWireError) return new GuestConnectionError(error.kind, error.code); - return new GuestConnectionError('transport-failure', 'guest-unexpected-error'); -} - -function assertRequestActive(signal: AbortSignal | undefined, closed: boolean): void { - if (signal?.aborted) throw new GuestConnectionError('cancelled', 'abort-signal'); - if (closed) throw new GuestConnectionError('transport-failure', 'guest-adapter-closed'); -} - -function failedGuestRequest( - request: SpikeRequest, - requestBytes: number, - started: number, - error: unknown, -): SpikeResponse { - const guestError = asGuestError(error); - return failureResponse( - request, - { kind: guestError.kind, code: guestError.code }, - { requestBytes, durationMs: performance.now() - started }, - ); -} - -function resourceLimitFailure( - metrics: { cpuMs: number | null; memoryBytes: number | null }, - limits: ResourceLimits, -): SpikeFailure | undefined { - if (metrics.cpuMs !== null && metrics.cpuMs > limits.maxCpuMs) { - return { kind: 'transport-failure', code: 'cpu-limit-exceeded' }; - } - if (metrics.memoryBytes !== null && metrics.memoryBytes > limits.maxMemoryBytes) { - return { kind: 'transport-failure', code: 'memory-limit-exceeded' }; - } - return undefined; -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/scripts/ios-ax-bridge-spike/guest-binary.test.ts b/scripts/ios-ax-bridge-spike/guest-binary.test.ts deleted file mode 100644 index 74982e99ab..0000000000 --- a/scripts/ios-ax-bridge-spike/guest-binary.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { describe, expect, test } from 'vitest'; -import { - EXPECTED_GUEST_BINARY_SHA256, - readVerifiedGuestMechanism, - sha256File, -} from './guest-binary.ts'; - -describe('guest binary provenance', () => { - test('hashes the supplied file bytes', () => { - const file = temporaryFile('verified bytes'); - expect(sha256File(file)).toBe( - '186287b2d987891f027b4bc8baaf621a3e5a4a73ec78e04b0f65dc309b1ccc03', - ); - }); - - test('rejects a supplied binary that is not the pinned idb guest', () => { - const file = temporaryFile('arbitrary bridge'); - expect(() => readVerifiedGuestMechanism(file)).toThrow( - `expected ${EXPECTED_GUEST_BINARY_SHA256}`, - ); - }); -}); - -function temporaryFile(contents: string): string { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'guest-binary-test-')); - const file = path.join(directory, 'SimulatorFrameworkBridge'); - fs.writeFileSync(file, contents); - return file; -} diff --git a/scripts/ios-ax-bridge-spike/guest-binary.ts b/scripts/ios-ax-bridge-spike/guest-binary.ts deleted file mode 100644 index 6a53b22ce4..0000000000 --- a/scripts/ios-ax-bridge-spike/guest-binary.ts +++ /dev/null @@ -1,33 +0,0 @@ -import crypto from 'node:crypto'; -import fs from 'node:fs'; -import type { GuestMechanismEvidence } from './types.ts'; - -export const EXPECTED_GUEST_BINARY_SHA256 = - '3545621d2dc98de32879ebac55e8b0c33dc8eb7cc2bfbc2d0d2d21a002c8de58'; - -export function readVerifiedGuestMechanism(guestBridge: string): GuestMechanismEvidence { - const observed = sha256File(guestBridge); - if (observed !== EXPECTED_GUEST_BINARY_SHA256) { - throw new Error( - `Guest bridge SHA-256 mismatch: expected ${EXPECTED_GUEST_BINARY_SHA256}, observed ${observed}.`, - ); - } - return { - implementation: 'idb', - release: 'v1.5.2', - companionArchive: 'idb-companion.macos-arm64.tar.gz', - companionSha256: 'f17b718a513931705542a7fbfa9cfc11895ee191562c9ffd2343cf7f8254bc08', - guestBinary: 'Resources/SimulatorFrameworkBridge', - guestBinaryExpectedSha256: EXPECTED_GUEST_BINARY_SHA256, - guestBinarySha256: observed, - transport: - 'xcrun simctl spawn SimulatorFrameworkBridge accessibility serve --idle-timeout 300 --exit-on-disconnect true; UNIX socket frames are a 4-byte big-endian length + JSON', - traversal: - 'describe with snapshotTree=true (one XCTest snapshot fetch per read) and automationMode=true asserted per request; no idb_companion, gRPC, or Python client', - client: 'node-direct-socket', - }; -} - -export function sha256File(filePath: string): string { - return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); -} diff --git a/scripts/ios-ax-bridge-spike/guest-connection.ts b/scripts/ios-ax-bridge-spike/guest-connection.ts deleted file mode 100644 index f9af137c6e..0000000000 --- a/scripts/ios-ax-bridge-spike/guest-connection.ts +++ /dev/null @@ -1,290 +0,0 @@ -import fs from 'node:fs'; -import net from 'node:net'; -import os from 'node:os'; -import path from 'node:path'; -import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; -import { performance } from 'node:perf_hooks'; -import { readGuestProcessSample, type GuestProcessSample } from './guest-process-metrics.ts'; -import { GuestFrameDecoder, GuestWireError, type GuestEnvelope } from './guest-wire.ts'; -import type { SpikeFailure } from './types.ts'; - -const GUEST_IDLE_TIMEOUT_SECONDS = 300; -const CONNECT_POLL_MS = 15; - -type Pending = Readonly<{ - resolve: (frame: Buffer) => void; - reject: (error: GuestConnectionError) => void; -}>; - -export class GuestConnectionError extends Error { - readonly kind: SpikeFailure['kind']; - readonly code: string; - - constructor(kind: SpikeFailure['kind'], code: string) { - super(`${kind}/${code}`); - this.name = 'GuestConnectionError'; - this.kind = kind; - this.code = code; - } -} - -export class GuestConnection { - private readonly bridgePath: string; - private readonly maxResponseBytes: number; - private readonly socketPath = path.join( - socketDirectory(), - `${process.pid.toString(36)}-${Math.random().toString(36).slice(2, 8)}.sock`, - ); - private child?: ChildProcess; - private socket?: net.Socket; - private decoder: GuestFrameDecoder; - private pending?: Pending; - private udid?: string; - private log = ''; - private killNext = false; - - constructor(bridgePath: string, maxResponseBytes: number) { - this.bridgePath = bridgePath; - this.maxResponseBytes = maxResponseBytes; - this.decoder = new GuestFrameDecoder(maxResponseBytes); - } - - async ensureConnected( - udid: string, - deadline: number, - signal: AbortSignal | undefined, - ): Promise { - if (this.socket && !this.socket.destroyed) { - if (this.udid !== udid) { - throw new GuestConnectionError('transport-failure', 'guest-udid-changed'); - } - return true; - } - this.udid = udid; - this.spawnGuest(udid); - this.socket = await this.connect(deadline, signal); - this.decoder = new GuestFrameDecoder(this.maxResponseBytes); - const socket = this.socket; - socket.on('data', (chunk: Buffer) => this.consume(chunk)); - socket.on('close', () => { - if (this.socket === socket) this.socket = undefined; - this.failPending(new GuestConnectionError('process-crash', 'guest-exited')); - }); - socket.on('error', () => { - this.failPending(new GuestConnectionError('transport-failure', 'guest-socket-error')); - }); - return false; - } - - processSample(): GuestProcessSample | undefined { - return readGuestProcessSample(this.socketPath); - } - - roundTrip( - frame: Buffer, - deadline: number, - signal: AbortSignal | undefined, - ): Promise<{ envelope: GuestEnvelope; responseBytes: number }> { - return new Promise((resolve, reject) => { - const socket = this.socket; - if (!socket) { - reject(new GuestConnectionError('transport-failure', 'guest-not-connected')); - return; - } - const timer = setTimeout( - () => this.drop(new GuestConnectionError('timeout', 'batch-duration-limit')), - Math.max(0, deadline - performance.now()), - ); - const onAbort = (): void => this.drop(new GuestConnectionError('cancelled', 'abort-signal')); - signal?.addEventListener('abort', onAbort, { once: true }); - const settle = (): void => { - clearTimeout(timer); - signal?.removeEventListener('abort', onAbort); - }; - this.pending = { - resolve: (body) => { - settle(); - try { - resolve({ - envelope: JSON.parse(body.toString('utf8')) as GuestEnvelope, - responseBytes: body.length + 4, - }); - } catch { - reject(new GuestConnectionError('malformed-tree', 'invalid-json')); - } - }, - reject: (error) => { - settle(); - reject(error); - }, - }; - socket.write(frame); - if (this.killNext) { - this.killNext = false; - killGuestProcesses(this.socketPath); - } - }); - } - - killOnNextRequest(): void { - this.killNext = true; - } - - takeLog(): string { - const log = this.log; - this.log = ''; - return log; - } - - async close(): Promise { - this.drop(new GuestConnectionError('cancelled', 'process-closed')); - await this.reapChild(); - fs.rmSync(this.socketPath, { force: true }); - } - - private spawnGuest(udid: string): void { - fs.rmSync(this.socketPath, { force: true }); - const child = spawn( - 'xcrun', - [ - 'simctl', - 'spawn', - udid, - this.bridgePath, - 'accessibility', - 'serve', - this.socketPath, - '--idle-timeout', - String(GUEST_IDLE_TIMEOUT_SECONDS), - '--exit-on-disconnect', - 'true', - ], - { stdio: ['ignore', 'pipe', 'pipe'] }, - ); - child.stdout?.on('data', (chunk: Buffer) => this.appendLog(chunk)); - child.stderr?.on('data', (chunk: Buffer) => this.appendLog(chunk)); - child.on('error', (error) => this.appendLog(Buffer.from(`${error.message}\n`))); - child.on('exit', () => { - if (this.child === child) this.child = undefined; - }); - this.child = child; - } - - private connect(deadline: number, signal: AbortSignal | undefined): Promise { - return new Promise((resolve, reject) => { - const attempt = (): void => { - if (signal?.aborted) { - void this.reapChild(); - reject(new GuestConnectionError('cancelled', 'abort-signal')); - return; - } - if (performance.now() > deadline) { - void this.reapChild(); - reject(new GuestConnectionError('timeout', 'guest-connect-timeout')); - return; - } - if (!this.child) { - reject(new GuestConnectionError('transport-failure', 'guest-exited-before-ready')); - return; - } - const socket = net.createConnection(this.socketPath); - socket.once('connect', () => { - socket.removeAllListeners('error'); - resolve(socket); - }); - socket.once('error', () => { - socket.destroy(); - setTimeout(attempt, CONNECT_POLL_MS); - }); - }; - attempt(); - }); - } - - private consume(chunk: Buffer): void { - let frames: Buffer[]; - try { - frames = this.decoder.push(chunk); - } catch (error) { - const wire = error instanceof GuestWireError ? error : undefined; - this.drop( - new GuestConnectionError( - wire?.kind ?? 'malformed-tree', - wire?.code ?? 'frame-limit-exceeded', - ), - ); - return; - } - for (const frame of frames) { - const pending = this.pending; - this.pending = undefined; - pending?.resolve(frame); - } - } - - private failPending(error: GuestConnectionError): void { - const pending = this.pending; - this.pending = undefined; - pending?.reject(error); - } - - private drop(error: GuestConnectionError): void { - this.failPending(error); - const socket = this.socket; - this.socket = undefined; - socket?.destroy(); - killGuestProcesses(this.socketPath); - } - - private async reapChild(): Promise { - const child = this.child; - if (!child || child.exitCode !== null) return; - const exited = new Promise((resolve) => child.once('exit', () => resolve())); - child.kill('SIGTERM'); - await Promise.race([exited, sleep(1_000)]); - if (child.exitCode === null) child.kill('SIGKILL'); - } - - private appendLog(chunk: Buffer): void { - if (this.log.length >= 64 * 1024) return; - this.log += chunk.toString('utf8').slice(0, 64 * 1024 - this.log.length); - } -} - -export function processAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} - -function socketDirectory(): string { - const directory = path.join(os.tmpdir(), 'agent-device-ax'); - fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); - fs.chmodSync(directory, 0o700); - if (directory.length + 24 >= 104) { - throw new Error(`Socket directory path is too long for a UNIX socket: ${directory}`); - } - return directory; -} - -function killGuestProcesses(socketPath: string): void { - const found = spawnSync('pgrep', ['-f', `accessibility serve ${socketPath}`], { - encoding: 'utf8', - }); - for (const line of (found.stdout ?? '').split('\n')) { - const pid = Number(line.trim()); - if (!Number.isSafeInteger(pid) || pid <= 0) continue; - try { - process.kill(pid, 'SIGKILL'); - } catch { - // already gone - } - } -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/scripts/ios-ax-bridge-spike/guest-process-metrics.test.ts b/scripts/ios-ax-bridge-spike/guest-process-metrics.test.ts deleted file mode 100644 index 85ad5946af..0000000000 --- a/scripts/ios-ax-bridge-spike/guest-process-metrics.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { parseProcessTime, processUsageDelta } from './guest-process-metrics.ts'; - -test('parses macOS process CPU time and derives one request resource sample', () => { - assert.equal(parseProcessTime('0:00.37'), 370); - assert.equal(parseProcessTime('1:02:03.50'), 3_723_500); - assert.equal(parseProcessTime('2-01:02:03.50'), 176_523_500); - assert.deepEqual( - processUsageDelta({ cpuMs: 120, memoryBytes: 10 }, { cpuMs: 350, memoryBytes: 42 }), - { cpuMs: 230, memoryBytes: 42 }, - ); - assert.deepEqual(processUsageDelta(undefined, { cpuMs: 350, memoryBytes: 42 }), { - cpuMs: 350, - memoryBytes: 42, - }); -}); diff --git a/scripts/ios-ax-bridge-spike/guest-process-metrics.ts b/scripts/ios-ax-bridge-spike/guest-process-metrics.ts deleted file mode 100644 index fa5219bc73..0000000000 --- a/scripts/ios-ax-bridge-spike/guest-process-metrics.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { spawnSync } from 'node:child_process'; - -export type GuestProcessSample = Readonly<{ cpuMs: number; memoryBytes: number }>; - -export function readGuestProcessSample(socketPath: string): GuestProcessSample | undefined { - const found = spawnSync( - 'pgrep', - ['-f', `SimulatorFrameworkBridge accessibility serve ${socketPath}`], - { encoding: 'utf8' }, - ); - const samples = (found.stdout ?? '') - .split('\n') - .map((value) => Number(value.trim())) - .filter((pid) => Number.isSafeInteger(pid) && pid > 0) - .flatMap(readProcessSample); - if (samples.length === 0) return undefined; - return { - cpuMs: samples.reduce((total, sample) => total + sample.cpuMs, 0), - memoryBytes: samples.reduce((maximum, sample) => Math.max(maximum, sample.memoryBytes), 0), - }; -} - -export function processUsageDelta( - before: GuestProcessSample | undefined, - after: GuestProcessSample | undefined, -): GuestProcessSample | undefined { - if (!after) return undefined; - return { - cpuMs: Math.max(0, after.cpuMs - (before?.cpuMs ?? 0)), - memoryBytes: after.memoryBytes, - }; -} - -export function parseProcessTime(value: string): number | undefined { - const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)$/u.exec(value.trim()); - if (!match) return undefined; - const days = Number(match[1] ?? 0); - const hours = Number(match[2] ?? 0); - const minutes = Number(match[3]); - const seconds = Number(match[4]); - return (((days * 24 + hours) * 60 + minutes) * 60 + seconds) * 1_000; -} - -function readProcessSample(pid: number): GuestProcessSample[] { - const result = spawnSync('ps', ['-o', 'time=', '-o', 'rss=', '-o', 'args=', '-p', String(pid)], { - encoding: 'utf8', - }); - const fields = (result.stdout ?? '').trim().split(/\s+/u); - if (!fields[2]?.endsWith('SimulatorFrameworkBridge')) return []; - const cpuMs = parseProcessTime(fields[0] ?? ''); - const memoryKb = Number(fields[1]); - return cpuMs === undefined || !Number.isFinite(memoryKb) || memoryKb < 0 - ? [] - : [{ cpuMs, memoryBytes: memoryKb * 1_024 }]; -} diff --git a/scripts/ios-ax-bridge-spike/guest-wire.test.ts b/scripts/ios-ax-bridge-spike/guest-wire.test.ts deleted file mode 100644 index c4b09e720c..0000000000 --- a/scripts/ios-ax-bridge-spike/guest-wire.test.ts +++ /dev/null @@ -1,206 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; -import { - acquisitionFromEnvelope, - elementTypeName, - encodeGuestFrame, - failureFromEnvelope, - flattenGuestTree, - GuestFrameDecoder, - guestDescribeRequest, - isTargetNotReady, - parseExpectedPid, -} from './guest-wire.ts'; -import type { SpikeRequest } from './types.ts'; - -function request(overrides: Partial = {}): SpikeRequest { - return { - version: 1, - id: 'one', - candidate: 'guest-simulator-framework-bridge', - simulatorUdid: '00000000-0000-0000-0000-000000000000', - state: 'warm', - screen: 'list', - limits: DEFAULT_SPIKE_LIMITS, - ...overrides, - }; -} - -const tree = { - XC_kAXXCAttributeElementType: 'UIApplication', - XC_kAXXCAttributeElementBaseType: 'UIApplication', - XC_kAXXCAttributeAutomationType: 1, - XC_kAXXCAttributeLabel: 'Agent Device Tester', - XC_kAXXCAttributeFrame: { X: 0, Y: 0, Width: 402, Height: 874 }, - XC_kAXXCAttributeChildren: [ - { - XC_kAXXCAttributeElementType: 'UIWindow', - XC_kAXXCAttributeElementBaseType: 'UIWindow', - XC_kAXXCAttributeAutomationType: 2, - XC_kAXXCAttributeFrame: { X: 0, Y: 0, Width: 402, Height: 874 }, - XC_kAXXCAttributeChildren: [ - { - XC_kAXXCAttributeElementType: '_UITabButton', - XC_kAXXCAttributeElementBaseType: 'UIControl', - XC_kAXXCAttributeAutomationType: 9, - XC_kAXXCAttributeLabel: 'Catalog', - XC_kAXXCAttributeIdentifier: 'tab-catalog', - XC_kAXXCAttributeValue: 1, - XC_kAXXCAttributeFrame: { X: 10, Y: 800, Width: 60, Height: 50 }, - XC_kAXXCAttributeChildren: [], - }, - ], - }, - ], -}; - -test('frames round-trip through the 4-byte big-endian length prefix, even when split', () => { - const frame = encodeGuestFrame({ verb: 'describe', pid: 7 }); - const decoder = new GuestFrameDecoder(); - assert.deepEqual(decoder.push(frame.subarray(0, 3)), []); - const frames = decoder.push(Buffer.concat([frame.subarray(3), frame])); - assert.equal(frames.length, 2); - assert.deepEqual(JSON.parse(frames[1]!.toString('utf8')), { - verb: 'describe', - pid: 7, - }); - assert.throws(() => new GuestFrameDecoder(8).push(encodeGuestFrame({ padding: 'x'.repeat(32) }))); -}); - -test('a known generation reads by pid and an unknown one resolves the frontmost app in-guest', () => { - assert.equal(parseExpectedPid('pid:4242'), 4242); - assert.equal(parseExpectedPid('gen-1'), undefined); - const byPid = guestDescribeRequest( - request({ expectedTargetGeneration: 'pid:4242' }), - DEFAULT_SPIKE_LIMITS, - ); - assert.equal(byPid.pid, 4242); - assert.equal(byPid.snapshotTree, true); - assert.equal(byPid.automationMode, true); - assert.equal(byPid.maxDepth, DEFAULT_SPIKE_LIMITS.maxTraversalDepth); - const frontmost = guestDescribeRequest(request(), DEFAULT_SPIKE_LIMITS); - assert.equal(frontmost.pid, undefined); - assert.equal(frontmost.method, 'runningboard'); -}); - -test('nested XC_kAXXC trees flatten to parent-linked raw nodes with XCTest type names', () => { - const nodes = flattenGuestTree([tree]); - assert.deepEqual( - nodes.map((node) => [node.id, node.parentId, node.type, node.role]), - [ - ['n0', undefined, 'Application', 'UIApplication'], - ['n1', 'n0', 'Window', 'UIWindow'], - ['n2', 'n1', 'Button', '_UITabButton'], - ], - ); - assert.equal(nodes[2]?.subrole, 'UIControl'); - assert.equal(nodes[2]?.value, '1'); - assert.deepEqual(nodes[2]?.frame, { x: 10, y: 800, width: 60, height: 50 }); - assert.equal(elementTypeName(undefined, 48), 'StaticText'); - assert.equal(elementTypeName(undefined, 0), 'Other'); - assert.equal(elementTypeName(undefined, 999), 'Other'); -}); - -test('a successful envelope becomes an acquisition with generation, viewport, and typed truncation', () => { - const parsed = acquisitionFromEnvelope( - { ok: true, tree: [tree], pid: 4242, truncated: true }, - request(), - DEFAULT_SPIKE_LIMITS, - ); - assert.ok('acquisition' in parsed); - if (!('acquisition' in parsed)) return; - assert.equal(parsed.acquisition.targetGeneration, 'pid:4242'); - assert.deepEqual(parsed.acquisition.viewport, { - kind: 'reported', - rect: { x: 0, y: 0, width: 402, height: 874 }, - }); - assert.equal(parsed.acquisition.truncated, true); - assert.deepEqual(parsed.acquisition.residue, [ - { - kind: 'truncated', - dimension: 'depth', - limit: DEFAULT_SPIKE_LIMITS.maxTraversalDepth, - }, - ]); -}); - -test('an observed pid that differs from the expected generation is a typed stale generation', () => { - const parsed = acquisitionFromEnvelope( - { ok: true, tree: [tree], pid: 4243 }, - request({ expectedTargetGeneration: 'pid:4242' }), - DEFAULT_SPIKE_LIMITS, - ); - assert.deepEqual(parsed, { - kind: 'stale-generation', - code: 'target-generation-mismatch', - expectedTargetGeneration: 'pid:4242', - observedTargetGeneration: 'pid:4243', - }); -}); - -test('guest errors map to typed failures without reading free text as truth', () => { - const notReady = { - ok: false, - error_kind: 'application_not_responding', - error: 'Error kAXErrorServerNotFound getting snapshot', - }; - assert.equal(isTargetNotReady(notReady), true); - assert.deepEqual( - failureFromEnvelope(notReady, request({ expectedTargetGeneration: 'pid:1' }), () => true), - { - kind: 'transport-failure', - code: 'target-application-unavailable', - }, - ); - assert.deepEqual( - failureFromEnvelope( - { - ok: false, - error_kind: 'application_unavailable', - error: 'pid 1 has no accessibility server', - }, - request({ expectedTargetGeneration: 'pid:1' }), - () => false, - ), - { - kind: 'stale-generation', - code: 'target-generation-mismatch', - expectedTargetGeneration: 'pid:1', - }, - ); - assert.deepEqual( - failureFromEnvelope( - { ok: false, error_kind: 'application_not_responding', error: 'pid 9 did not answer' }, - request({ expectedTargetGeneration: 'pid:9' }), - () => false, - ), - { - kind: 'stale-generation', - code: 'target-generation-mismatch', - expectedTargetGeneration: 'pid:9', - }, - ); - assert.deepEqual( - failureFromEnvelope( - { ok: false, error_kind: 'application_not_responding', error: 'pid 9 did not answer' }, - request({ expectedTargetGeneration: 'pid:9' }), - () => true, - ), - { kind: 'timeout', code: 'application-not-responding' }, - ); - assert.deepEqual( - failureFromEnvelope({ ok: false, error_kind: 'reader_unavailable' }, request(), () => true), - { - kind: 'unsupported-mechanism', - code: 'reader-unavailable', - }, - ); - assert.deepEqual( - failureFromEnvelope({ ok: false, error_kind: 'bad_request' }, request(), () => true), - { - kind: 'transport-failure', - code: 'bad-request', - }, - ); -}); diff --git a/scripts/ios-ax-bridge-spike/guest-wire.ts b/scripts/ios-ax-bridge-spike/guest-wire.ts deleted file mode 100644 index a3e467b5ac..0000000000 --- a/scripts/ios-ax-bridge-spike/guest-wire.ts +++ /dev/null @@ -1,408 +0,0 @@ -import type { - RawAcquiredNode, - RawAcquisition, - ResourceLimits, - SpikeFailure, - SpikeRequest, -} from './types.ts'; - -/** - * Host side of the guest `accessibility serve` wire contract, byte-matching - * `SimulatorFrameworkBridge/AccessibilityService.m` in idb v1.5.2: frames are a 4-byte big-endian - * length prefix followed by one JSON object; the guest answers `{ ok, tree | error, error_kind, pid, - * truncated, phases, automation }`. - */ -const GUEST_FRAME_HEADER_BYTES = 4; -const GUEST_MAX_FRAME_BYTES = 16 * 1024 * 1024; - -const ATTRIBUTE = { - elementType: 'XC_kAXXCAttributeElementType', - elementBaseType: 'XC_kAXXCAttributeElementBaseType', - label: 'XC_kAXXCAttributeLabel', - value: 'XC_kAXXCAttributeValue', - identifier: 'XC_kAXXCAttributeIdentifier', - frame: 'XC_kAXXCAttributeFrame', - automationType: 'XC_kAXXCAttributeAutomationType', - children: 'XC_kAXXCAttributeChildren', -} as const; - -/** `XCUIElementType` raw values, from Xcode's `XCUIElementTypes.h`; `Any` reads as `Other`. */ -const ELEMENT_TYPE_NAMES: readonly string[] = [ - 'Other', - 'Other', - 'Application', - 'Group', - 'Window', - 'Sheet', - 'Drawer', - 'Alert', - 'Dialog', - 'Button', - 'RadioButton', - 'RadioGroup', - 'CheckBox', - 'DisclosureTriangle', - 'PopUpButton', - 'ComboBox', - 'MenuButton', - 'ToolbarButton', - 'Popover', - 'Keyboard', - 'Key', - 'NavigationBar', - 'TabBar', - 'TabGroup', - 'Toolbar', - 'StatusBar', - 'Table', - 'TableRow', - 'TableColumn', - 'Outline', - 'OutlineRow', - 'Browser', - 'CollectionView', - 'Slider', - 'PageIndicator', - 'ProgressIndicator', - 'ActivityIndicator', - 'SegmentedControl', - 'Picker', - 'PickerWheel', - 'Switch', - 'Toggle', - 'Link', - 'Image', - 'Icon', - 'SearchField', - 'ScrollView', - 'ScrollBar', - 'StaticText', - 'TextField', - 'SecureTextField', - 'DatePicker', - 'TextView', - 'Menu', - 'MenuItem', - 'MenuBar', - 'MenuBarItem', - 'Map', - 'WebView', - 'IncrementArrow', - 'DecrementArrow', - 'Timeline', - 'RatingIndicator', - 'ValueIndicator', - 'SplitGroup', - 'Splitter', - 'RelevanceIndicator', - 'ColorWell', - 'HelpTag', - 'Matte', - 'DockItem', - 'Ruler', - 'RulerMarker', - 'Grid', - 'LevelIndicator', - 'Cell', - 'LayoutArea', - 'LayoutItem', - 'Handle', - 'Stepper', - 'Tab', - 'TouchBar', - 'StatusItem', -]; - -/** - * XCTest names the application and window elements by class rather than by automation type; the - * XCTest control tree reports `Application`/`Window` where the guest attribute says 1/2. Observed - * node-for-node against the control on the catalog fixture (279/279 aligned). - */ -const CLASS_PROMOTED_TYPES: Readonly> = { - UIApplication: 'Application', - UIWindow: 'Window', -}; - -export type GuestEnvelope = Readonly>; - -export type GuestErrorKind = - | 'application_unavailable' - | 'application_not_responding' - | 'frontmost_unresolved' - | 'reader_unavailable' - | 'bad_request' - | 'assertion_failed'; - -export function encodeGuestFrame(value: unknown): Buffer { - const body = Buffer.from(JSON.stringify(value), 'utf8'); - const header = Buffer.alloc(GUEST_FRAME_HEADER_BYTES); - header.writeUInt32BE(body.length, 0); - return Buffer.concat([header, body]); -} - -/** Reassembles length-prefixed frames from a byte stream; oversize frames throw. */ -export class GuestFrameDecoder { - private buffer = Buffer.alloc(0); - private readonly maxFrameBytes: number; - - constructor(maxFrameBytes = GUEST_MAX_FRAME_BYTES) { - this.maxFrameBytes = maxFrameBytes; - } - - push(chunk: Buffer): Buffer[] { - this.buffer = Buffer.concat([this.buffer, chunk]); - const frames: Buffer[] = []; - while (this.buffer.length >= GUEST_FRAME_HEADER_BYTES) { - const length = this.buffer.readUInt32BE(0); - if (length === 0 || length > this.maxFrameBytes) { - throw new GuestWireError('malformed-tree', 'frame-limit-exceeded'); - } - if (this.buffer.length < GUEST_FRAME_HEADER_BYTES + length) break; - frames.push( - this.buffer.subarray(GUEST_FRAME_HEADER_BYTES, GUEST_FRAME_HEADER_BYTES + length), - ); - this.buffer = this.buffer.subarray(GUEST_FRAME_HEADER_BYTES + length); - } - return frames; - } -} - -export class GuestWireError extends Error { - readonly kind: SpikeFailure['kind']; - readonly code: string; - - constructor(kind: SpikeFailure['kind'], code: string) { - super(`${kind}/${code}`); - this.name = 'GuestWireError'; - this.kind = kind; - this.code = code; - } -} - -export function parseExpectedPid(generation: string | undefined): number | undefined { - if (!generation?.startsWith('pid:')) return undefined; - const pid = Number(generation.slice('pid:'.length).split(':', 1)[0]); - return Number.isSafeInteger(pid) && pid > 0 ? pid : undefined; -} - -/** - * The guest `describe` request for one spike request. A known target generation names the app by - * pid; otherwise the guest resolves the foreground app in-guest through RunningBoard (the anchor is - * required by the wire even when the method ignores it). Automation mode is asserted on every read so - * the target exposes its accessibility server without preboot preference edits; the single-fetch - * traversal keeps one Mach round trip per read. - */ -export function guestDescribeRequest( - request: SpikeRequest, - limits: ResourceLimits, -): Record { - const pid = parseExpectedPid(request.expectedTargetGeneration); - return { - verb: 'describe', - ...(pid === undefined ? { x: 1, y: 1, method: 'runningboard' } : { pid }), - snapshotTree: true, - automationMode: true, - maxDepth: limits.maxTraversalDepth, - maxNodes: limits.maxNodes, - }; -} - -function guestErrorKind(envelope: GuestEnvelope): GuestErrorKind | undefined { - const kind = envelope.error_kind; - return typeof kind === 'string' ? (kind as GuestErrorKind) : undefined; -} - -function guestErrorText(envelope: GuestEnvelope): string { - return typeof envelope.error === 'string' ? envelope.error : ''; -} - -/** - * Whether a failed read is the target's accessibility server not being reachable yet, which is what - * a freshly launched app (or one whose automation mode was just asserted) reports for a moment. - */ -export function isTargetNotReady(envelope: GuestEnvelope): boolean { - return ( - guestErrorKind(envelope) === 'application_unavailable' || - guestErrorText(envelope).includes('kAXErrorServerNotFound') - ); -} - -export function failureFromEnvelope( - envelope: GuestEnvelope, - request: SpikeRequest, - targetAlive: (pid: number) => boolean, -): SpikeFailure { - const expectedPid = parseExpectedPid(request.expectedTargetGeneration); - const kind = guestErrorKind(envelope); - const targetSymptom = - kind === 'application_unavailable' || - kind === 'application_not_responding' || - isTargetNotReady(envelope); - // The expected generation is provably gone: whatever symptom the guest saw while reaching for the - // dead pid, the honest answer is a stale generation, never a timeout the caller might retry. - if (targetSymptom && expectedPid !== undefined && !targetAlive(expectedPid)) { - return { - kind: 'stale-generation', - code: 'target-generation-mismatch', - expectedTargetGeneration: request.expectedTargetGeneration!, - }; - } - if (isTargetNotReady(envelope)) { - return { kind: 'transport-failure', code: 'target-application-unavailable' }; - } - switch (kind) { - case 'application_unavailable': - return { kind: 'transport-failure', code: 'target-application-unavailable' }; - case 'application_not_responding': - return { kind: 'timeout', code: 'application-not-responding' }; - case 'frontmost_unresolved': - return { kind: 'transport-failure', code: 'frontmost-unresolved' }; - case 'reader_unavailable': - return { kind: 'unsupported-mechanism', code: 'reader-unavailable' }; - case 'bad_request': - return { kind: 'transport-failure', code: 'bad-request' }; - default: - return { kind: 'transport-failure', code: 'guest-error' }; - } -} - -export type GuestAcquisition = Readonly<{ - acquisition: RawAcquisition; - observedPid: number | undefined; -}>; - -export function acquisitionFromEnvelope( - envelope: GuestEnvelope, - request: SpikeRequest, - limits: ResourceLimits, -): GuestAcquisition | SpikeFailure { - const expectedPid = parseExpectedPid(request.expectedTargetGeneration); - const observedPid = typeof envelope.pid === 'number' ? envelope.pid : undefined; - if (expectedPid !== undefined && observedPid !== undefined && observedPid !== expectedPid) { - return { - kind: 'stale-generation', - code: 'target-generation-mismatch', - expectedTargetGeneration: request.expectedTargetGeneration!, - observedTargetGeneration: `pid:${observedPid}`, - }; - } - const roots = guestRoots(envelope.tree); - if (!roots) return { kind: 'malformed-tree', code: 'guest-tree-shape' }; - const nodes = flattenGuestTree(roots); - const truncated = envelope.truncated === true; - const pid = expectedPid ?? observedPid; - const viewport = viewportFromRoot(roots[0]); - return { - observedPid, - acquisition: { - targetId: `simulator:${request.simulatorUdid}`, - targetGeneration: pid === undefined ? null : `pid:${pid}`, - nodes, - viewport, - truncated, - residue: [ - ...(truncated ? [truncationResidue(nodes.length, limits)] : []), - ...(pid === undefined ? [{ kind: 'unavailable-fact', fact: 'generation' } as const] : []), - ], - }, - }; -} - -function truncationResidue( - nodeCount: number, - limits: ResourceLimits, -): RawAcquisition['residue'][number] { - return nodeCount >= limits.maxNodes - ? { kind: 'truncated', dimension: 'nodes', limit: limits.maxNodes } - : { - kind: 'truncated', - dimension: 'depth', - limit: limits.maxTraversalDepth, - }; -} - -function guestRoots(tree: unknown): Record[] | undefined { - if (Array.isArray(tree)) return tree.every(isRecord) ? tree : undefined; - return isRecord(tree) ? [tree] : undefined; -} - -/** Depth-first flattening; ids follow traversal order and parents precede children. */ -export function flattenGuestTree(roots: readonly Record[]): RawAcquiredNode[] { - const nodes: RawAcquiredNode[] = []; - const visit = (element: Record, parentId: string | undefined): void => { - const id = `n${nodes.length}`; - nodes.push({ - id, - ...(parentId === undefined ? {} : { parentId }), - ...nodeFacts(element), - }); - const children = element[ATTRIBUTE.children]; - if (!Array.isArray(children)) return; - for (const child of children) if (isRecord(child)) visit(child, id); - }; - for (const root of roots) visit(root, undefined); - return nodes; -} - -function nodeFacts(element: Record): Omit { - const elementClass = optionalString(element[ATTRIBUTE.elementType]); - const baseClass = optionalString(element[ATTRIBUTE.elementBaseType]); - const type = elementTypeName(elementClass, element[ATTRIBUTE.automationType]); - const label = optionalString(element[ATTRIBUTE.label]); - const value = optionalScalar(element[ATTRIBUTE.value]); - const identifier = optionalString(element[ATTRIBUTE.identifier]); - const frame = frameFromGuest(element[ATTRIBUTE.frame]); - return { - ...(type === undefined ? {} : { type }), - ...(elementClass === undefined ? {} : { role: elementClass }), - ...(baseClass === undefined || baseClass === elementClass ? {} : { subrole: baseClass }), - ...(label === undefined ? {} : { label }), - ...(value === undefined ? {} : { value }), - ...(identifier === undefined ? {} : { identifier }), - ...(frame === undefined ? {} : { frame }), - }; -} - -export function elementTypeName( - elementClass: string | undefined, - automationType: unknown, -): string | undefined { - if (elementClass !== undefined && CLASS_PROMOTED_TYPES[elementClass]) { - return CLASS_PROMOTED_TYPES[elementClass]; - } - if (typeof automationType !== 'number' || !Number.isInteger(automationType)) return undefined; - return ELEMENT_TYPE_NAMES[automationType] ?? 'Other'; -} - -function frameFromGuest(value: unknown): RawAcquiredNode['frame'] | undefined { - if (!isRecord(value)) return undefined; - const numbers = ['X', 'Y', 'Width', 'Height'].map((key) => value[key]); - if (!numbers.every((number) => typeof number === 'number' && Number.isFinite(number))) { - return undefined; - } - const [x, y, width, height] = numbers as [number, number, number, number]; - return { x, y, width, height }; -} - -function viewportFromRoot(root: Record | undefined): RawAcquisition['viewport'] { - const frame = root === undefined ? undefined : frameFromGuest(root[ATTRIBUTE.frame]); - const isApplication = - root !== undefined && optionalString(root[ATTRIBUTE.elementType]) === 'UIApplication'; - return isApplication && frame - ? { kind: 'reported', rect: frame } - : { kind: 'missing', reason: 'not-provided' }; -} - -function optionalString(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function optionalScalar(value: unknown): string | undefined { - if (typeof value === 'string') return value.length > 0 ? value : undefined; - if (typeof value === 'number' || typeof value === 'boolean') return String(value); - return undefined; -} - -function isRecord(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value); -} diff --git a/scripts/ios-ax-bridge-spike/limits.test.ts b/scripts/ios-ax-bridge-spike/limits.test.ts deleted file mode 100644 index 85b1dc5b50..0000000000 --- a/scripts/ios-ax-bridge-spike/limits.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { DEFAULT_SPIKE_LIMITS, encodeFrame, validateRawAcquisition } from './limits.ts'; - -const acquisition = { - targetId: 'simulator:test', - targetGeneration: 'generation-1', - nodes: [ - { id: 'n0', type: 'XCUIElementTypeApplication', role: 'AXApplication' }, - { id: 'n1', parentId: 'n0', role: 'AXWindow', frame: { x: 0, y: 0, width: 100, height: 200 } }, - ], - viewport: { kind: 'reported', rect: { x: 0, y: 0, width: 100, height: 200 } }, - truncated: false, - residue: [], -} as const; - -test('accepts a raw tree and reports structural depth without publishing it on nodes', () => { - const result = validateRawAcquisition(acquisition, DEFAULT_SPIKE_LIMITS); - assert.equal(result.ok, true); - if (result.ok) { - assert.equal(result.maxTraversalDepth, 1); - assert.equal(result.acquisition.nodes[0]?.type, 'XCUIElementTypeApplication'); - } - assert.equal('depth' in acquisition.nodes[0], false); - assert.equal('hittable' in acquisition.nodes[0], false); -}); - -test('rejects presentation facts in the acquisition reader', () => { - const result = validateRawAcquisition( - { - ...acquisition, - nodes: [{ id: 'n0', visibleToUser: true }], - }, - DEFAULT_SPIKE_LIMITS, - ); - assert.deepEqual(result, { ok: false, code: 'node-contains-presentation-fact' }); -}); - -test('rejects cycles, missing parents, and resource-limit violations', () => { - assert.deepEqual( - validateRawAcquisition( - { ...acquisition, nodes: [{ id: 'n0', parentId: 'missing' }] }, - DEFAULT_SPIKE_LIMITS, - ), - { ok: false, code: 'parent-node-missing' }, - ); - assert.deepEqual( - validateRawAcquisition( - { ...acquisition, nodes: [{ id: 'n0', parentId: 'n0' }] }, - DEFAULT_SPIKE_LIMITS, - ), - { ok: false, code: 'traversal-depth-exceeded' }, - ); - assert.deepEqual( - validateRawAcquisition( - { ...acquisition, nodes: Array.from({ length: 3 }, (_, index) => ({ id: `n${index}` })) }, - { - ...DEFAULT_SPIKE_LIMITS, - maxNodes: 2, - }, - ), - { ok: false, code: 'node-limit-exceeded' }, - ); -}); - -test('frames are newline-delimited and byte bounded', () => { - const frame = encodeFrame({ id: 'one', text: 'ok' }); - assert.equal(frame.line.endsWith('\n'), true); - assert.equal(frame.bytes, Buffer.byteLength(frame.line)); -}); diff --git a/scripts/ios-ax-bridge-spike/limits.ts b/scripts/ios-ax-bridge-spike/limits.ts deleted file mode 100644 index 8a84ef303d..0000000000 --- a/scripts/ios-ax-bridge-spike/limits.ts +++ /dev/null @@ -1,230 +0,0 @@ -import type { RawAcquiredNode, RawAcquisition, ResourceLimits, SpikeRect } from './types.ts'; - -export const DEFAULT_SPIKE_LIMITS: ResourceLimits = Object.freeze({ - maxRequestBytes: 64 * 1024, - maxResponseBytes: 4 * 1024 * 1024, - maxNodes: 1500, - maxTraversalDepth: 64, - maxCpuMs: 2_000, - maxMemoryBytes: 256 * 1024 * 1024, - maxDurationMs: 5_000, -}); - -const NODE_KEYS = new Set([ - 'id', - 'type', - 'parentId', - 'role', - 'subrole', - 'label', - 'value', - 'identifier', - 'frame', - 'enabled', - 'selected', - 'focused', -]); - -export type TreeValidation = - | Readonly<{ ok: true; acquisition: RawAcquisition; maxTraversalDepth: number }> - | Readonly<{ ok: false; code: string }>; - -export function encodeFrame(value: unknown): { bytes: number; line: string } { - const line = `${JSON.stringify(value)}\n`; - return { bytes: Buffer.byteLength(line), line }; -} - -export function validateRawAcquisition(value: unknown, limits: ResourceLimits): TreeValidation { - const record = readAcquisitionRecord(value); - if (!record.ok) return record; - const envelope = validateAcquisitionEnvelope(record.value); - if (!envelope.ok) return envelope; - if (record.value.nodes.length > limits.maxNodes) - return { ok: false, code: 'node-limit-exceeded' }; - const nodes = validateNodes(record.value.nodes); - if (!nodes.ok) return nodes; - const maxTraversalDepth = treeDepth(nodes.nodes); - if (maxTraversalDepth > limits.maxTraversalDepth) { - return { ok: false, code: 'traversal-depth-exceeded' }; - } - return { - ok: true, - acquisition: record.value as unknown as RawAcquisition, - maxTraversalDepth, - }; -} - -function readAcquisitionRecord( - value: unknown, -): - | { ok: true; value: Record & { nodes: unknown[] } } - | { ok: false; code: string } { - if (!isRecord(value)) return { ok: false, code: 'acquisition-not-object' }; - if (typeof value.targetId !== 'string' || value.targetId.length === 0) { - return { ok: false, code: 'target-id-missing' }; - } - if (value.targetGeneration !== null && typeof value.targetGeneration !== 'string') { - return { ok: false, code: 'target-generation-invalid' }; - } - if (!Array.isArray(value.nodes)) return { ok: false, code: 'nodes-not-array' }; - return { ok: true, value: value as Record & { nodes: unknown[] } }; -} - -function validateAcquisitionEnvelope( - value: Record, -): { ok: true } | { ok: false; code: string } { - if (typeof value.truncated !== 'boolean') return { ok: false, code: 'truncated-invalid' }; - if (!validateViewport(value.viewport)) return { ok: false, code: 'viewport-invalid' }; - if (!validateResidue(value.residue)) return { ok: false, code: 'residue-invalid' }; - return { ok: true }; -} - -function validateNodes( - rawNodes: readonly unknown[], -): { ok: true; nodes: RawAcquiredNode[] } | { ok: false; code: string } { - const nodes: RawAcquiredNode[] = []; - const ids = new Set(); - for (const rawNode of rawNodes) { - const node = validateNode(rawNode); - if (!node.ok) return node; - if (ids.has(node.node.id)) return { ok: false, code: 'duplicate-node-id' }; - ids.add(node.node.id); - nodes.push(node.node); - } - return validateParents(nodes, ids); -} - -function validateParents( - nodes: readonly RawAcquiredNode[], - ids: ReadonlySet, -): { ok: true; nodes: RawAcquiredNode[] } | { ok: false; code: string } { - for (const node of nodes) { - if (node.parentId !== undefined && !ids.has(node.parentId)) { - return { ok: false, code: 'parent-node-missing' }; - } - } - return { ok: true, nodes: [...nodes] }; -} - -function validateNode( - value: unknown, -): { ok: true; node: RawAcquiredNode } | { ok: false; code: string } { - const node = asNodeRecord(value); - if (!node) return { ok: false, code: 'node-not-object' }; - const code = nodeValidationCode(node); - return code === undefined ? { ok: true, node: node as RawAcquiredNode } : { ok: false, code }; -} - -function asNodeRecord(value: unknown): Record | undefined { - return isRecord(value) ? value : undefined; -} - -function nodeValidationCode(value: Record): string | undefined { - const identityCode = nodeIdentityCode(value); - if (identityCode) return identityCode; - return nodeFactCode(value); -} - -function nodeIdentityCode(value: Record): string | undefined { - if (Object.keys(value).some((key) => !NODE_KEYS.has(key))) { - return 'node-contains-presentation-fact'; - } - if (typeof value.id !== 'string' || value.id.length === 0) return 'node-id-missing'; - if (value.parentId !== undefined && typeof value.parentId !== 'string') { - return 'parent-id-invalid'; - } - return undefined; -} - -function nodeFactCode(value: Record): string | undefined { - const textCode = optionalStringCode(value); - if (textCode) return textCode; - const booleanCode = optionalBooleanCode(value); - if (booleanCode) return booleanCode; - return value.frame !== undefined && !validateRect(value.frame) ? 'frame-invalid' : undefined; -} - -function optionalStringCode(value: Record): string | undefined { - for (const key of ['type', 'role', 'subrole', 'label', 'value', 'identifier'] as const) { - if (value[key] !== undefined && typeof value[key] !== 'string') return `${key}-invalid`; - } - return undefined; -} - -function optionalBooleanCode(value: Record): string | undefined { - for (const key of ['enabled', 'selected', 'focused'] as const) { - if (value[key] !== undefined && typeof value[key] !== 'boolean') return `${key}-invalid`; - } - return undefined; -} - -function validateViewport(value: unknown): boolean { - if (!isRecord(value) || typeof value.kind !== 'string') return false; - if (value.kind === 'missing') { - return ( - value.reason === 'not-provided' || - value.reason === 'not-supported' || - value.reason === 'invalid' - ); - } - return (value.kind === 'reported' || value.kind === 'derived') && validateRect(value.rect); -} - -function validateRect(value: unknown): value is SpikeRect { - if (!isRecord(value)) return false; - for (const key of ['x', 'y', 'width', 'height']) { - if (typeof value[key] !== 'number' || !Number.isFinite(value[key])) return false; - } - const width = value.width; - const height = value.height; - return typeof width === 'number' && typeof height === 'number' && width >= 0 && height >= 0; -} - -function validateResidue(value: unknown): boolean { - if (!Array.isArray(value)) return false; - for (const residue of value) { - if (!validateResidueItem(residue)) return false; - } - return true; -} - -const RESIDUE_VALIDATORS: Readonly) => boolean>> = { - 'provider-pruned': (value) => - Array.isArray(value.fields) && value.fields.every((field) => typeof field === 'string'), - 'missing-viewport': (value) => - ['not-provided', 'not-supported', 'invalid'].includes(String(value.reason)), - truncated: (value) => ['nodes', 'depth', 'payload'].includes(String(value.dimension)), - 'stale-generation': (value) => value.expected === undefined || typeof value.expected === 'string', - 'unavailable-fact': (value) => typeof value.fact === 'string', - 'fallback-source': (value) => typeof value.producer === 'string', -}; - -function validateResidueItem(value: unknown): boolean { - if (!isRecord(value) || typeof value.kind !== 'string') return false; - return RESIDUE_VALIDATORS[value.kind]?.(value) ?? false; -} - -function treeDepth(nodes: readonly RawAcquiredNode[]): number { - const byId = new Map(nodes.map((node) => [node.id, node])); - const memo = new Map(); - const visiting = new Set(); - let maximum = 0; - for (const node of nodes) maximum = Math.max(maximum, depth(node.id)); - return maximum; - - function depth(id: string): number { - const cached = memo.get(id); - if (cached !== undefined) return cached; - if (visiting.has(id)) return Number.POSITIVE_INFINITY; - visiting.add(id); - const parentId = byId.get(id)?.parentId; - const result = parentId === undefined ? 0 : depth(parentId) + 1; - visiting.delete(id); - memo.set(id, result); - return result; - } -} - -function isRecord(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value); -} diff --git a/scripts/ios-ax-bridge-spike/protocol.ts b/scripts/ios-ax-bridge-spike/protocol.ts deleted file mode 100644 index 4ba7e560d5..0000000000 --- a/scripts/ios-ax-bridge-spike/protocol.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { ResourceMetrics, SpikeFailure, SpikeRequest, SpikeResponse } from './types.ts'; - -export function failureResponse( - request: SpikeRequest, - failure: SpikeFailure, - metrics: Partial = {}, -): SpikeResponse { - return { - version: 1, - id: request.id, - candidate: request.candidate, - ok: false, - failure, - metrics: { - requestBytes: metrics.requestBytes ?? 0, - responseBytes: metrics.responseBytes ?? 0, - nodeCount: metrics.nodeCount ?? 0, - maxTraversalDepth: metrics.maxTraversalDepth ?? 0, - cpuMs: metrics.cpuMs ?? null, - memoryBytes: metrics.memoryBytes ?? null, - durationMs: metrics.durationMs ?? 0, - }, - }; -} diff --git a/scripts/ios-ax-bridge-spike/targeted-evidence.ts b/scripts/ios-ax-bridge-spike/targeted-evidence.ts deleted file mode 100644 index 2d98c3c518..0000000000 --- a/scripts/ios-ax-bridge-spike/targeted-evidence.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { execFileSync } from 'node:child_process'; -import os from 'node:os'; -import { performance } from 'node:perf_hooks'; -import { createGuestSimulatorFrameworkBridgeAdapter } from './guest-adapter.ts'; -import type { SpikeConfig } from './config.ts'; -import { runTargetedRelaunch } from './targeted-relaunch.ts'; -import { awaitAppReadiness } from './targeted-readiness.ts'; -import { - adapterOptions, - missingResponse, - targetedRequest, - usableTree, -} from './targeted-request.ts'; -import { - bootSimulator, - readRunningAppPids, - shutdownSimulator, - terminateApp, -} from '../ios-snapshot-benchmark/lifecycle.ts'; -import type { SpikeRequest, SpikeResponse } from './types.ts'; -import type { - HostLoad, - TargetedBootstrapSample, - TargetedRelaunchSample, - TargetedRecoveryProbe, -} from './corrected-types.ts'; - -const APP_ID = 'com.callstack.agentdevicelab'; -const BOOTSTRAP_SAMPLES = 5; - -type GuestAdapter = ReturnType; - -export type TargetedRunResult = Readonly<{ - bootstrap: readonly TargetedBootstrapSample[]; - relaunch: readonly TargetedRelaunchSample[]; - recovery: readonly TargetedRecoveryProbe[]; - host: HostLoad; -}>; - -/** - * Live nonresident-bootstrap and recovery evidence for the guest bridge. - * - * Boundary: the Simulator is booted and the fixture app is relaunched for every bootstrap sample. - * App readiness is *observed*, not assumed from a pid: a throwaway probe bridge polls until the new - * app generation answers with a tree, then is torn down, so the timed sample starts with no resident - * bridge and a ready app. Automation mode is asserted per request by the guest. - */ -export async function runTargetedEvidence(config: SpikeConfig): Promise { - bootSimulator(config.udid); - try { - const bootstrap = await runNonresidentBootstrap(config); - return { - bootstrap, - relaunch: await runTargetedRelaunch(config), - recovery: await runLiveRecovery(config, bootstrap), - host: hostLoad(), - }; - } finally { - if (!config.keepDevice) shutdownSimulator(config.udid); - } -} - -async function runNonresidentBootstrap( - config: SpikeConfig, -): Promise { - const samples: TargetedBootstrapSample[] = []; - for (let index = 1; index <= BOOTSTRAP_SAMPLES; index += 1) { - samples.push(await captureBootstrap(config, index)); - } - return samples; -} - -async function captureBootstrap( - config: SpikeConfig, - index: number, -): Promise { - const appPid = await relaunchApp(config.udid); - const readiness = await awaitAppReadiness(config, appPid, 'list'); - await assertNoResidentGuest(); - const adapter = createGuestSimulatorFrameworkBridgeAdapter(adapterOptions(config)); - const started = performance.now(); - const result = await adapter.acquireBatch([ - targetedRequest(config, `bootstrap-${index}`, { - expectedTargetGeneration: `pid:${appPid}`, - }), - ]); - const durationMs = performance.now() - started; - await adapter.close?.(); - const response = result.responses[0] ?? missingResponse(`bootstrap-${index}`); - return { - index, - durationMs, - usableTree: usableTree(response), - response, - stderr: result.stderr, - appPid, - readinessMs: readiness.readinessMs, - readinessAttempts: readiness.attempts, - host: hostLoad(), - }; -} - -/** A killed guest can linger for a moment while launchd_sim reaps it; wait briefly, then fail closed. */ -async function assertNoResidentGuest(): Promise { - const deadline = Date.now() + 5_000; - let found = residentGuestPids(); - while (found.length > 0 && Date.now() < deadline) { - await sleep(100); - found = residentGuestPids(); - } - if (found.length > 0) { - throw new Error(`A guest bridge is still resident before a nonresident sample: pids ${found}`); - } -} - -function residentGuestPids(): number[] { - return execFileSync( - 'sh', - ['-c', 'pgrep -f "SimulatorFrameworkBridge accessibility serve" || true'], - { encoding: 'utf8' }, - ) - .split('\n') - .map((value) => Number(value.trim())) - .filter((value) => Number.isSafeInteger(value) && value > 0); -} - -async function runLiveRecovery( - config: SpikeConfig, - bootstrap: readonly TargetedBootstrapSample[], -): Promise { - const adapter = createGuestSimulatorFrameworkBridgeAdapter(adapterOptions(config)); - try { - const probes: TargetedRecoveryProbe[] = []; - await adapter.acquireBatch([targetedRequest(config, 'recovery-prime')]); - adapter.evidence?.terminateReaderOnNextBatch?.(); - probes.push( - await recoveryProbe( - config, - adapter, - 'process-crash', - targetedRequest(config, 'recovery-crash'), - ), - ); - probes.push( - await recoveryProbe( - config, - adapter, - 'timeout', - targetedRequest(config, 'recovery-timeout', { - limits: { ...config.limits, maxDurationMs: 1 }, - }), - ), - ); - probes.push(await cancellationProbe(config, adapter)); - probes.push( - await recoveryProbe( - config, - adapter, - 'stale-generation', - targetedRequest(config, 'recovery-stale-generation', { - expectedTargetGeneration: `pid:${deadGeneration(bootstrap)}`, - }), - ), - ); - return probes; - } finally { - await adapter.close?.(); - } -} - -/** A previous app generation's pid: the bootstrap relaunches guarantee it is dead. */ -function deadGeneration(bootstrap: readonly TargetedBootstrapSample[]): number { - const previous = bootstrap.at(-2)?.appPid; - if (previous === undefined) throw new Error('No previous app generation to test staleness.'); - return previous; -} - -async function recoveryProbe( - config: SpikeConfig, - adapter: GuestAdapter, - operation: TargetedRecoveryProbe['operation'], - probeRequest: SpikeRequest, -): Promise { - const result = await adapter.acquireBatch([probeRequest]); - return { - operation, - request: probeRequest, - response: result.responses[0] ?? missingResponse(probeRequest.id), - recoveredResponse: await healthyResponse(config, adapter, `${probeRequest.id}-recovered`), - }; -} - -async function cancellationProbe( - config: SpikeConfig, - adapter: GuestAdapter, -): Promise { - const probeRequest = targetedRequest(config, 'recovery-cancelled'); - const controller = new AbortController(); - const pending = adapter.acquireBatch([probeRequest], { - signal: controller.signal, - }); - setTimeout(() => controller.abort(), 1); - const result = await pending; - return { - operation: 'cancelled', - request: probeRequest, - response: result.responses[0] ?? missingResponse(probeRequest.id), - recoveredResponse: await healthyResponse(config, adapter, 'recovery-cancelled-recovered'), - }; -} - -async function healthyResponse( - config: SpikeConfig, - adapter: GuestAdapter, - id: string, -): Promise { - const result = await adapter.acquireBatch([targetedRequest(config, id)]); - return result.responses[0] ?? missingResponse(id); -} - -/** A fresh app generation: terminate, launch, and wait for exactly one running pid. */ -async function relaunchApp(udid: string): Promise { - terminateApp(udid, APP_ID); - execFileSync('xcrun', ['simctl', 'launch', udid, APP_ID], { - encoding: 'utf8', - timeout: 60_000, - stdio: ['ignore', 'pipe', 'pipe'], - }); - const deadline = Date.now() + 30_000; - while (Date.now() < deadline) { - const pids = readRunningAppPids(udid, APP_ID); - if (pids.length === 1) return pids[0]!; - await sleep(100); - } - throw new Error(`App ${APP_ID} did not start on ${udid}.`); -} - -function hostLoad(): HostLoad { - return { - loadAverage1m: Number(os.loadavg()[0]?.toFixed(2)), - cpuCores: os.cpus().length, - }; -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/scripts/ios-ax-bridge-spike/targeted-readiness.ts b/scripts/ios-ax-bridge-spike/targeted-readiness.ts deleted file mode 100644 index 832566400e..0000000000 --- a/scripts/ios-ax-bridge-spike/targeted-readiness.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { performance } from 'node:perf_hooks'; -import { createGuestSimulatorFrameworkBridgeAdapter } from './guest-adapter.ts'; -import { - adapterOptions, - missingResponse, - targetedRequest, - usableTree, -} from './targeted-request.ts'; -import type { SpikeConfig } from './config.ts'; -import type { ScreenId } from '../ios-snapshot-benchmark/types.ts'; - -const READINESS_POLL_MS = 200; -const READINESS_DEADLINE_MS = 90_000; -const READINESS_PROBE_REQUEST_MS = 60_000; - -export async function awaitAppReadiness( - config: SpikeConfig, - appPid: number, - screen: ScreenId, - expectedAnchor?: string, -): Promise<{ readinessMs: number; attempts: number }> { - const limits = { ...config.limits, maxDurationMs: READINESS_PROBE_REQUEST_MS }; - const probe = createGuestSimulatorFrameworkBridgeAdapter({ - ...adapterOptions(config), - limits, - }); - const started = performance.now(); - let attempts = 0; - try { - for (;;) { - assertReadinessDeadline(started, appPid, screen); - attempts += 1; - if (await probeReady(config, probe, limits, appPid, screen, expectedAnchor, attempts)) { - return { readinessMs: performance.now() - started, attempts }; - } - await sleep(READINESS_POLL_MS); - } - } finally { - await probe.close?.(); - } -} - -async function probeReady( - config: SpikeConfig, - probe: ReturnType, - limits: SpikeConfig['limits'], - appPid: number, - screen: ScreenId, - expectedAnchor: string | undefined, - attempt: number, -): Promise { - const id = `readiness-${screen}-${appPid}-${attempt}`; - const result = await probe.acquireBatch([ - targetedRequest(config, id, { - state: 'relaunch', - screen, - expectedTargetGeneration: `pid:${appPid}`, - limits, - }), - ]); - return usableTree(result.responses[0] ?? missingResponse(id), `pid:${appPid}`, expectedAnchor); -} - -function assertReadinessDeadline(started: number, appPid: number, screen: ScreenId): void { - if (performance.now() - started < READINESS_DEADLINE_MS) return; - throw new Error(`App generation pid:${appPid} did not expose ${screen} in time.`); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/scripts/ios-ax-bridge-spike/targeted-relaunch.ts b/scripts/ios-ax-bridge-spike/targeted-relaunch.ts deleted file mode 100644 index db8317d713..0000000000 --- a/scripts/ios-ax-bridge-spike/targeted-relaunch.ts +++ /dev/null @@ -1,151 +0,0 @@ -import path from 'node:path'; -import { performance } from 'node:perf_hooks'; -import { - openFixture, - pressFixtureTarget, - scrollFixtureSetup, - snapshotFixture, - type CliContext, -} from '../ios-snapshot-benchmark/command.ts'; -import { screenFixture } from '../ios-snapshot-benchmark/definitions.ts'; -import { - fixtureOperationFromCli, - prepareFixture, - requireFixtureOperationSuccess, -} from '../ios-snapshot-benchmark/fixture-admission.ts'; -import { readRunningAppPids, stopDaemon } from '../ios-snapshot-benchmark/lifecycle.ts'; -import { closeSession } from '../ios-snapshot-benchmark/local-runner.ts'; -import { createGuestSimulatorFrameworkBridgeAdapter } from './guest-adapter.ts'; -import { awaitAppReadiness } from './targeted-readiness.ts'; -import { adapterOptions, missingResponse, targetedRequest } from './targeted-request.ts'; -import type { SpikeConfig } from './config.ts'; -import type { TargetedRelaunchSample } from './corrected-types.ts'; -import type { ScreenFixture, ScreenId } from '../ios-snapshot-benchmark/types.ts'; - -const RELAUNCH_SAMPLES = 20; -const SCREENS: readonly ScreenId[] = [ - 'quiet', - 'list', - 'nested-scroll', - 'alert', - 'system-surface', - 'xctest-stress', -]; - -export async function runTargetedRelaunch( - config: SpikeConfig, -): Promise { - const samples: TargetedRelaunchSample[] = []; - for (const screen of SCREENS) { - samples.push(...(await runScreen(config, screen))); - } - return samples; -} - -async function runScreen( - config: SpikeConfig, - screen: ScreenId, -): Promise { - const fixture = screenFixture(screen); - const expectedAnchor = fixture.postSetupAnchorText ?? fixture.anchorText; - const context = contextFor(config, screen); - const adapter = createGuestSimulatorFrameworkBridgeAdapter(adapterOptions(config)); - const samples: TargetedRelaunchSample[] = []; - let previousPid: number | undefined; - try { - for (let index = 1; index <= RELAUNCH_SAMPLES; index += 1) { - await prepareRelaunch(context, fixture, screen); - const appPid = exactAppPid(config.udid, fixture.app); - assertNewPid(previousPid, appPid, screen); - previousPid = appPid; - samples.push(await captureRelaunch(config, adapter, screen, expectedAnchor, appPid, index)); - } - return samples; - } finally { - await adapter.close?.(); - closeSession(context); - stopDaemon(config.repoRoot, config.stateDir); - } -} - -async function prepareRelaunch( - context: CliContext, - fixture: ScreenFixture, - screen: ScreenId, -): Promise { - const opened = openFixture(context, fixture, { relaunch: true }); - requireFixtureOperationSuccess( - fixtureOperationFromCli(opened, `relaunch ${screen} setup`), - `relaunch ${screen} setup`, - 'cell-state', - ); - await prepareFixture(fixture, { - observe: () => fixtureOperationFromCli(snapshotFixture(context), 'fixture snapshot'), - scrollToBottom: () => - fixtureOperationFromCli(scrollFixtureSetup(context), 'fixture scroll bottom'), - openAlert: () => - fixtureOperationFromCli( - pressFixtureTarget(context, 'id="automation-open-alert"'), - 'fixture open alert', - ), - }); -} - -async function captureRelaunch( - config: SpikeConfig, - adapter: ReturnType, - screen: ScreenId, - expectedAnchor: string, - appPid: number, - index: number, -): Promise { - const readiness = await awaitAppReadiness(config, appPid, screen, expectedAnchor); - const id = `relaunch-${screen}-${index}`; - const started = performance.now(); - const result = await adapter.acquireBatch([ - targetedRequest(config, id, { - state: 'relaunch', - screen, - expectedTargetGeneration: `pid:${appPid}`, - }), - ]); - return { - index, - screen, - expectedAnchor, - appPid, - readinessMs: readiness.readinessMs, - readinessAttempts: readiness.attempts, - durationMs: performance.now() - started, - response: result.responses[0] ?? missingResponse(id), - stderr: result.stderr, - }; -} - -function assertNewPid(previousPid: number | undefined, appPid: number, screen: ScreenId): void { - if (previousPid !== appPid) return; - throw new Error(`Relaunch ${screen} reused app pid ${appPid}.`); -} - -function exactAppPid(udid: string, appId: string): number { - const pids = readRunningAppPids(udid, appId); - if (pids.length !== 1) { - throw new Error( - `Expected one ${appId} process on ${udid}, observed ${pids.join(', ') || 'none'}.`, - ); - } - return pids[0]!; -} - -function contextFor(config: SpikeConfig, screen: ScreenId): CliContext { - return { - repoRoot: config.repoRoot, - stateDir: config.stateDir, - session: `ax-bridge-relaunch-${screen}`, - udid: config.udid, - derivedPath: path.join(config.derivedPath, screen), - }; -} - -export const TARGETED_RELAUNCH_SCREENS = SCREENS; -export const TARGETED_RELAUNCH_SAMPLES = RELAUNCH_SAMPLES; diff --git a/scripts/ios-ax-bridge-spike/targeted-request.ts b/scripts/ios-ax-bridge-spike/targeted-request.ts deleted file mode 100644 index e0c0c8d7f9..0000000000 --- a/scripts/ios-ax-bridge-spike/targeted-request.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { SpikeConfig } from './config.ts'; -import type { SpikeRequest, SpikeResponse } from './types.ts'; - -export function targetedRequest( - config: SpikeConfig, - id: string, - overrides: Partial = {}, -): SpikeRequest { - return { - version: 1, - id, - candidate: 'guest-simulator-framework-bridge', - simulatorUdid: config.udid, - state: 'warm', - screen: 'list', - limits: config.limits, - ...overrides, - }; -} - -export function missingResponse(id: string): SpikeResponse { - return { - version: 1, - id, - candidate: 'guest-simulator-framework-bridge', - ok: false, - failure: { kind: 'transport-failure', code: 'missing-response' }, - metrics: { - requestBytes: 0, - responseBytes: 0, - nodeCount: 0, - maxTraversalDepth: 0, - cpuMs: null, - memoryBytes: null, - durationMs: 0, - }, - }; -} - -export function usableTree( - response: SpikeResponse, - expectedGeneration?: string, - expectedAnchor?: string, -): boolean { - const acquisition = response.acquisition; - if (!response.ok) return false; - if (!acquisition || acquisition.nodes.length === 0) return false; - return [ - matchesOptional(expectedGeneration, acquisition.targetGeneration), - containsOptionalAnchor(expectedAnchor, acquisition.nodes), - ].every(Boolean); -} - -export function adapterOptions(config: SpikeConfig) { - return { guestBridge: config.guestBridge, limits: config.limits }; -} - -function matchesOptional(expected: string | undefined, observed: string | null): boolean { - return expected === undefined || expected === observed; -} - -function containsOptionalAnchor( - expected: string | undefined, - nodes: NonNullable['nodes'], -): boolean { - if (expected === undefined) return true; - return nodes.some((node) => node.label === expected || node.value === expected); -} diff --git a/scripts/ios-ax-bridge-spike/targeted-run.ts b/scripts/ios-ax-bridge-spike/targeted-run.ts deleted file mode 100644 index 3a44abc160..0000000000 --- a/scripts/ios-ax-bridge-spike/targeted-run.ts +++ /dev/null @@ -1,81 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import { gzipSync } from 'node:zlib'; -import { fileURLToPath } from 'node:url'; -import { parseConfig } from './config.ts'; -import { - buildCorrectedReport, - readSpikeReport, - readTargetedArtifact, - writeCorrectedReport, -} from './corrected-report.ts'; -import { readVerifiedGuestMechanism } from './guest-binary.ts'; -import { runTargetedEvidence } from './targeted-evidence.ts'; -import { TARGETED_RELAUNCH_SAMPLES, TARGETED_RELAUNCH_SCREENS } from './targeted-relaunch.ts'; -import { TARGETED_SCHEMA_VERSION, type TargetedRawArtifact } from './corrected-types.ts'; -import { readGitRevision, readTarget, readToolchain } from '../ios-snapshot-benchmark/host.ts'; - -const SOURCE = 'docs/evidence/ios-simulator-ax-bridge-2026-09-01-final.json.gz'; -const TARGETED = 'docs/evidence/ios-simulator-ax-bridge-2026-09-02-targeted.json.gz'; -const CORRECTED = 'docs/evidence/ios-simulator-ax-bridge-2026-09-02-corrected.json.gz'; - -if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - await main(process.argv.slice(2)); -} - -async function main(argv: readonly string[]): Promise { - const config = parseConfig(argv); - try { - await capture(config); - } finally { - fs.rmSync(config.stateDir, { recursive: true, force: true }); - } -} - -async function capture(config: ReturnType): Promise { - const revision = readGitRevision(config.repoRoot); - if (revision.dirty) { - throw new Error('Targeted bridge evidence must be captured from a clean Git revision.'); - } - const guestMechanism = readVerifiedGuestMechanism(config.guestBridge); - const source = readSpikeReport(SOURCE); - const evidence = await runTargetedEvidence(config); - const target = readTarget(config.udid, 'com.callstack.agentdevicelab'); - const artifact: TargetedRawArtifact = { - schemaVersion: TARGETED_SCHEMA_VERSION, - generatedAt: new Date().toISOString(), - revision, - command: - 'pnpm bench:ios-ax-bridge:targeted -- --udid --guest-bridge /Resources/SimulatorFrameworkBridge', - sourceArtifact: { - path: SOURCE, - revision: source.revision, - hostClient: String((source.guestMechanism as { client?: unknown }).client ?? 'unknown'), - }, - target: { udid: target.udid, name: target.name, runtime: target.runtime }, - toolchain: readToolchain(), - host: evidence.host, - guestMechanism, - limits: config.limits, - config: { - states: ['warm', 'relaunch'], - screens: TARGETED_RELAUNCH_SCREENS, - samples: TARGETED_RELAUNCH_SAMPLES, - bootstrapSamples: evidence.bootstrap.length, - }, - bootstrap: evidence.bootstrap, - relaunch: evidence.relaunch, - recovery: evidence.recovery, - }; - fs.writeFileSync(TARGETED, gzipSync(`${JSON.stringify(artifact)}\n`, { level: 9 })); - writeCorrectedReport( - CORRECTED, - buildCorrectedReport({ - sourcePath: SOURCE, - source, - targetedPath: TARGETED, - targeted: readTargetedArtifact(TARGETED), - }), - ); - process.stdout.write(`${CORRECTED}\n`); -} diff --git a/scripts/ios-ax-bridge-spike/types.ts b/scripts/ios-ax-bridge-spike/types.ts deleted file mode 100644 index 826b99f72d..0000000000 --- a/scripts/ios-ax-bridge-spike/types.ts +++ /dev/null @@ -1,149 +0,0 @@ -import type { - IosAcquisitionResidue, - IosViewportEvidence, -} from '@agent-device/contracts/ios-snapshot'; -import type { LocalState, ScreenId } from '../ios-snapshot-benchmark/types.ts'; - -export type CandidateId = 'guest-simulator-framework-bridge' | 'xctest-control'; - -export type SpikeFailureKind = - | 'unsupported-mechanism' - | 'malformed-tree' - | 'stale-generation' - | 'timeout' - | 'cancelled' - | 'process-crash' - | 'transport-failure'; - -export type SpikeFailure = Readonly<{ - kind: SpikeFailureKind; - code?: string; - expectedTargetGeneration?: string; - observedTargetGeneration?: string; -}>; - -export type SpikeRect = Readonly<{ x: number; y: number; width: number; height: number }>; - -export type RawAcquiredNode = Readonly<{ - id: string; - type?: string; - parentId?: string; - role?: string; - subrole?: string; - label?: string; - value?: string; - identifier?: string; - frame?: SpikeRect; - enabled?: boolean; - selected?: boolean; - focused?: boolean; -}>; - -export type RawAcquisition = Readonly<{ - targetId: string; - targetGeneration: string | null; - nodes: readonly RawAcquiredNode[]; - viewport: IosViewportEvidence; - truncated: boolean; - residue: readonly IosAcquisitionResidue[]; -}>; - -export type ResourceLimits = Readonly<{ - maxRequestBytes: number; - maxResponseBytes: number; - maxNodes: number; - maxTraversalDepth: number; - maxCpuMs: number; - maxMemoryBytes: number; - maxDurationMs: number; -}>; - -export type ResourceMetrics = Readonly<{ - requestBytes: number; - responseBytes: number; - nodeCount: number; - maxTraversalDepth: number; - cpuMs: number | null; - memoryBytes: number | null; - durationMs: number; -}>; - -export type SpikeRequest = Readonly<{ - version: 1; - id: string; - candidate: CandidateId; - simulatorUdid: string; - state: LocalState; - screen: ScreenId | 'unprepared-surface'; - expectedTargetGeneration?: string; - limits: ResourceLimits; -}>; - -export type SpikeResponse = Readonly<{ - version: 1; - id: string; - candidate: CandidateId; - ok: boolean; - acquisition?: RawAcquisition; - failure?: SpikeFailure; - metrics: ResourceMetrics; -}>; - -export type SpikeSample = Readonly<{ - wallClockMs: number; - preparationMs?: number; - firstLookMs?: number; - firstTree: 'readable' | 'empty' | 'unreadable' | 'not-observed'; - ok: boolean; - acquisition?: RawAcquisition; -}>; - -export type SpikeCell = Readonly<{ - candidate: CandidateId; - state: LocalState; - screen: ScreenId; - acquisitionSamples: readonly SpikeSample[]; -}>; - -export type Revision = Readonly<{ commit: string; branch: string; dirty: boolean }>; -export type Toolchain = Readonly<{ - node: string; - pnpm: string; - xcode: string; - simctl: string; - os: string; - arch: string; -}>; -export type Target = Readonly<{ udid: string; name: string; runtime: string }>; -export type GuestMechanismEvidence = Readonly<{ - implementation: 'idb'; - release: 'v1.5.2'; - companionArchive: 'idb-companion.macos-arm64.tar.gz'; - companionSha256: string; - guestBinary: 'Resources/SimulatorFrameworkBridge'; - guestBinaryExpectedSha256?: string; - guestBinarySha256: string; - transport: string; - traversal: string; - client: 'node-direct-socket'; -}>; - -export type PreferenceEvidence = Readonly<{ - applied: boolean; - restored: boolean; - fixtureLaunchCompatible: boolean | null; - simulatorStateBefore: string; - diffs: readonly Readonly<{ - changes: readonly Readonly<{ key: string; before?: unknown; after?: unknown }>[]; - }>[]; -}>; - -export type SpikeReport = Readonly<{ - revision: Revision; - guestMechanism: GuestMechanismEvidence; - target: Target; - toolchain: Toolchain; - cells: readonly SpikeCell[]; - decisionReasons: readonly string[]; - preferenceEvidence?: PreferenceEvidence; -}>; diff --git a/scripts/png-crop-benchmark/README.md b/scripts/png-crop-benchmark/README.md deleted file mode 100644 index 26b6f6580c..0000000000 --- a/scripts/png-crop-benchmark/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# PNG crop benchmark - -```sh -pnpm bench:png-crop -- --rounds 5 -pnpm bench:png-crop -- --rounds 5 --file /path/to/real-capture.png -``` - -Compares the two ways this repository can crop a screenshot. Both run over the same bytes as PNG -worker jobs in one process, so what differs is the algorithm and not the thread it happens to land -on, or a file write on one side only: - -- `whole-image` — the previous crop: one job decodes the whole capture to RGBA, the box rows are - copied out of that bitmap, and a second job encodes the box. -- `region` — the shipped crop: one job reads the box's rows and encodes them, as RGB rather than - RGBA whenever the cropped pixels are all opaque. - -The corpus is generated, so a full run costs seconds and needs no device. Generated captures are -written to `.tmp/png-crop-benchmark/` and each one's compressed size is printed under the table: -a corpus that stops resembling a real capture becomes visible there instead of flattering the -result. Pass real captures with `--file` (repeatable) to put their numbers in the same table; the -real captures decide the verdict, since generated content cannot match a device's deflate stream. - -## What is actually saved - -Neither pipeline reads less of the file: a deflate stream has to be inflated to its end, so both -inflate the whole compressed image, and the region path inflates it into a buffer sized for every -filtered row in the capture. What the region path saves is the pixel work — reconstructing only down -to the box's last row and producing only the box's pixels, instead of a full RGBA bitmap for the -whole capture — plus one worker round trip, and the RGBA re-encode of the answer. - -## What the measurements have said - -Measured at that matched boundary over 7 rounds, on captures taken from an iOS Simulator and an -Android Emulator: the iOS captures go 2.5x to 5.6x faster and their crops come out 1.04x to 2.16x -smaller, mostly because an opaque crop is written as RGB instead of RGBA. A flat UI capture gains -the most, because the previous pipeline still expands the whole capture to an RGBA bitmap whatever -the filters look like, while the region path skips the pixels above the box. - -The noisiest capture in that set — a full-screen Android `screencap`, 1.4 MB compressed — is a wash -on time (1.0x to 1.6x) and its crop comes out up to 1.13x *larger*. Inflating and reconstructing -that much entropy dominates both pipelines, and the region writer's `None` filter cannot beat the -general writer's filter search on content that noisy. Check your own captures with `--file` before -reading a win or a loss into any number here. - -The encoder keeps the `None` filter on every scanline. Scoring the five PNG filters per row is -1.4x to 2.6x slower and produces a *larger* file on UI captures, where the smallest-sum-of-absolute- -differences heuristic prefers Sub or Up on text rows that deflate smaller unfiltered. On synthetic -low-frequency content — the `photo` captures here, which are smooth 8px blocks — `None` is still -faster but writes about 1.7x more bytes than a filtered encoding would. Real photo-filled captures -still come out smaller with `None` than with the general writer's own filter search, so the corpus -here overstates that case; check your own captures with `--file` before treating it as a limit. diff --git a/scripts/png-crop-benchmark/args.test.ts b/scripts/png-crop-benchmark/args.test.ts deleted file mode 100644 index d239a4a988..0000000000 --- a/scripts/png-crop-benchmark/args.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { test } from 'vitest'; -import assert from 'node:assert/strict'; -import { parseBenchmarkArgs } from './args.ts'; - -test('an empty command line keeps the default round count', () => { - assert.deepEqual(parseBenchmarkArgs([]), { - rounds: 5, - jsonPath: undefined, - captureFiles: [], - }); -}); - -test('rounds, json path, and every capture file are read from their flags', () => { - assert.deepEqual( - parseBenchmarkArgs([ - '--rounds', - '9', - '--json', - 'out/bench.json', - '--file', - 'a.png', - '--file', - 'b.png', - ]), - { rounds: 9, jsonPath: 'out/bench.json', captureFiles: ['a.png', 'b.png'] }, - ); -}); - -test('a round count that is not a positive number falls back to the default', () => { - assert.equal(parseBenchmarkArgs(['--rounds', '0']).rounds, 5); - assert.equal(parseBenchmarkArgs(['--rounds', 'many']).rounds, 5); - assert.equal(parseBenchmarkArgs(['--rounds', '2.4']).rounds, 2); -}); - -test('a flag with no value after it is ignored', () => { - assert.deepEqual(parseBenchmarkArgs(['--json']), { - rounds: 5, - jsonPath: undefined, - captureFiles: [], - }); -}); diff --git a/scripts/png-crop-benchmark/args.ts b/scripts/png-crop-benchmark/args.ts deleted file mode 100644 index b0f3c33630..0000000000 --- a/scripts/png-crop-benchmark/args.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** The command line `pnpm bench:png-crop` accepts. */ - -export type BenchmarkOptions = Readonly<{ - rounds: number; - jsonPath: string | undefined; - captureFiles: readonly string[]; -}>; - -const DEFAULT_ROUNDS = 5; - -export function parseBenchmarkArgs(argv: readonly string[]): BenchmarkOptions { - return { - rounds: readNumber(argv, '--rounds') ?? DEFAULT_ROUNDS, - jsonPath: readString(argv, '--json'), - captureFiles: readAll(argv, '--file'), - }; -} - -function readNumber(argv: readonly string[], flag: string): number | undefined { - const value = readString(argv, flag); - const parsed = value === undefined ? Number.NaN : Number(value); - return Number.isFinite(parsed) && parsed > 0 ? Math.round(parsed) : undefined; -} - -function readString(argv: readonly string[], flag: string): string | undefined { - const index = argv.indexOf(flag); - return index >= 0 ? argv[index + 1] : undefined; -} - -function readAll(argv: readonly string[], flag: string): string[] { - const values: string[] = []; - argv.forEach((entry, index) => { - if (entry === flag && argv[index + 1] !== undefined) values.push(argv[index + 1]!); - }); - return values; -} diff --git a/scripts/png-crop-benchmark/corpus.test.ts b/scripts/png-crop-benchmark/corpus.test.ts deleted file mode 100644 index 20c5528adf..0000000000 --- a/scripts/png-crop-benchmark/corpus.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { test } from 'vitest'; -import assert from 'node:assert/strict'; -import { mkdtempSync, rmSync } from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { PNG } from '@agent-device/capture-kit/png'; -import { buildCorpus, cropBoxOf, CROP_SCENARIOS } from './corpus.ts'; - -const SCENARIO_BY_NAME = new Map(CROP_SCENARIOS.map((scenario) => [scenario.name, scenario])); -const CAPTURE = { - name: 'phone', - label: 'phone', - width: 1200, - height: 2400, - bytes: Buffer.alloc(0), -}; -const SMALL = [{ name: 'tiny', label: 'tiny', width: 24, height: 40 }]; - -test('a card crop keeps the framed fraction of the capture', () => { - const box = cropBoxOf(CAPTURE, SCENARIO_BY_NAME.get('card')!); - - assert.deepEqual(box, { x: 96, y: 600, width: 960, height: 480 }); -}); - -test('a full-bleed header is clamped to the image, never one pixel past it', () => { - const box = cropBoxOf(CAPTURE, SCENARIO_BY_NAME.get('header')!); - - assert.equal(box.x, 0); - assert.equal(box.width, CAPTURE.width); - assert.equal(box.y + box.height, 288); -}); - -test('the same generated capture comes out byte for byte every run', () => { - const directory = mkdtempSync(path.join(os.tmpdir(), 'png-crop-corpus-')); - try { - const first = buildCorpus(path.join(directory, 'first'), SMALL); - const second = buildCorpus(path.join(directory, 'second'), SMALL); - - assert.equal(first.length, 2); - assert.deepEqual( - first.map((capture) => capture.bytes), - second.map((capture) => capture.bytes), - ); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); - -test('interface and photo captures differ, and both decode at their declared size', () => { - const directory = mkdtempSync(path.join(os.tmpdir(), 'png-crop-corpus-')); - try { - const [interfaceCapture, photoCapture] = buildCorpus(directory, SMALL); - - const decoded = [interfaceCapture, photoCapture].map((capture) => - PNG.sync.read(capture!.bytes), - ); - assert.deepEqual( - decoded.map((png) => [png.width, png.height]), - [ - [24, 40], - [24, 40], - ], - ); - assert.notDeepEqual(decoded[0]!.data, decoded[1]!.data); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); diff --git a/scripts/png-crop-benchmark/corpus.ts b/scripts/png-crop-benchmark/corpus.ts deleted file mode 100644 index a6dd43ba2e..0000000000 --- a/scripts/png-crop-benchmark/corpus.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import path from 'node:path'; -import { PNG } from '@agent-device/capture-kit/png'; - -/** - * The captures and crop boxes this benchmark measures. They are generated rather than recorded, - * so a run costs seconds and needs no device; each one reports its own compressed size, which is - * how a corpus that stopped resembling a real capture becomes visible instead of flattering. - * Pass real captures with `--file` to put their numbers in the same table. - */ - -export type CaptureResolution = Readonly<{ - name: string; - label: string; - width: number; - height: number; -}>; - -const RESOLUTIONS: readonly CaptureResolution[] = [ - { name: 'ios-phone', label: 'iPhone-class 1206x2622', width: 1206, height: 2622 }, - { name: 'android-phone', label: 'Android-class 1080x2400', width: 1080, height: 2400 }, - { name: 'ios-pad', label: 'iPad-class 2048x2732', width: 2048, height: 2732 }, -]; - -export type Capture = Readonly<{ - name: string; - label: string; - width: number; - height: number; - bytes: Buffer; -}>; - -export type CropScenario = Readonly<{ - name: string; - /** Fractions of the capture, so one scenario reads well at every resolution. */ - fraction: Readonly<{ x: number; y: number; width: number; height: number }>; -}>; - -/** The box shapes `--crop-on` resolves: a card, a full-bleed header, and a small control. */ -export const CROP_SCENARIOS: readonly CropScenario[] = [ - { name: 'card', fraction: { x: 0.08, y: 0.25, width: 0.8, height: 0.2 } }, - { name: 'header', fraction: { x: 0, y: 0, width: 1, height: 0.12 } }, - { name: 'control', fraction: { x: 0.3, y: 0.6, width: 0.3, height: 0.04 } }, -]; - -export function buildCorpus( - outDir: string, - resolutions: readonly CaptureResolution[] = RESOLUTIONS, -): Capture[] { - mkdirSync(outDir, { recursive: true }); - return resolutions.flatMap(({ name, label, width, height }) => - (['interface', 'photo'] as const).map((kind) => { - const bytes = PNG.sync.write(encodeCapture(width, height, kind)); - writeFileSync(path.join(outDir, `${name}-${kind}.png`), bytes); - return { name: `${name}-${kind}`, label: `${label} ${kind}`, width, height, bytes }; - }), - ); -} - -export function readCaptureFile(filePath: string, index: number): Capture { - const bytes = readFileSync(filePath); - const png = PNG.sync.read(bytes); - return { - name: `capture-${index}`, - label: path.basename(filePath), - width: png.width, - height: png.height, - bytes, - }; -} - -type Content = 'interface' | 'photo'; - -/** - * A capture a device could have produced. `interface` is flat panels with high-frequency text - * where the copy is drawn, which is what a settings or list screen looks like. `photo` is - * low-frequency detail in 8px blocks plus a gradient, which is what a photo, artwork, or a blurred - * background does to the deflate stream. - */ -function encodeCapture(width: number, height: number, content: Content): PNG { - const png = new PNG({ width, height }); - const tones = content === 'photo' ? photoToneField(width, height) : null; - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const offset = (y * width + x) * 4; - const [red, green, blue] = tones ? photoPixel(tones, x, y) : interfacePixel(x, y); - png.data[offset] = red; - png.data[offset + 1] = green; - png.data[offset + 2] = blue; - png.data[offset + 3] = 255; - } - } - return png; -} - -/** Flat panels with high-frequency text where the copy is drawn. */ -function interfacePixel(x: number, y: number): readonly [number, number, number] { - const ink = x % 320 < 2 || y % 96 < 2 || (y % 24 > 6 && y % 24 < 18 && x % 7 < 3); - if (ink) return [24, 26, 30]; - const band = Math.floor(y / 96); - return [(band * 11) % 240, (band * 17) % 244, (band * 23) % 248]; -} - -/** The 8px-block detail and soft gradient of a photo, scaled by the pixel position. */ -function photoPixel( - tones: readonly (readonly number[])[], - x: number, - y: number, -): readonly [number, number, number] { - const tone = tones[y >> 3]?.[x >> 3] ?? 0; - return [ - clampByte(tone + y / 6), - clampByte(tone * 0.8 + x / 9), - clampByte(tone * 0.6 + (x + y) / 24), - ]; -} - -/** Smooth 8px-block luminance in [40, 215], so a photo corpus compresses like a real photo. */ -function photoToneField(width: number, height: number): number[][] { - let state = 0x9e3779b9; - const next = () => { - state = (state * 1664525 + 1013904223) >>> 0; - return state / 0x100000000; - }; - return Array.from({ length: Math.ceil(height / 8) }, (_row, blockRow) => - Array.from({ length: Math.ceil(width / 8) }, (_column, blockColumn) => { - const drift = Math.sin(blockRow / 26) * 45 + Math.cos(blockColumn / 19) * 45; - return clampByte(128 + drift + (next() - 0.5) * 70); - }), - ); -} - -function clampByte(value: number): number { - return Math.max(0, Math.min(255, Math.round(value))); -} - -export function cropBoxOf( - capture: Capture, - scenario: CropScenario, -): { - x: number; - y: number; - width: number; - height: number; -} { - const { fraction } = scenario; - const x = Math.round(capture.width * fraction.x); - const y = Math.round(capture.height * fraction.y); - return { - x, - y, - width: Math.max(1, Math.min(capture.width - x, Math.round(capture.width * fraction.width))), - height: Math.max(1, Math.min(capture.height - y, Math.round(capture.height * fraction.height))), - }; -} diff --git a/scripts/png-crop-benchmark/pipelines.ts b/scripts/png-crop-benchmark/pipelines.ts deleted file mode 100644 index 58ec2dc4ec..0000000000 --- a/scripts/png-crop-benchmark/pipelines.ts +++ /dev/null @@ -1,74 +0,0 @@ -import type { Rect } from '@agent-device/kernel/snapshot'; -import { PNG } from '@agent-device/capture-kit/png'; -import { - cropPngBytesAsync, - decodePngAsync, - encodePngAsync, -} from '@agent-device/capture-kit/png-worker-client'; -import type { Capture, CropScenario } from './corpus.ts'; -import { cropBoxOf } from './corpus.ts'; -import { measureAsync } from './statistics.ts'; - -/** - * The two ways this repository can crop a capture, both run as PNG worker jobs over the same - * bytes, so what differs between them is the algorithm rather than the thread it lands on: - * - * - `wholeImage`: the previous crop. One job decodes the whole capture to RGBA, the box rows are - * copied out of that bitmap, and a second job encodes the box. - * - `region`: the shipped crop. One job reads the box's rows and encodes them. - * - * Neither writes a file. Publishing the artifact is the same work on both sides, so it would only - * dilute the ratio; the byte length of the encoded answer is reported instead. - */ - -export type PipelineSample = Readonly<{ medianMs: number; bestMs: number; outBytes: number }>; - -export type ScenarioSample = Readonly<{ - capture: string; - label: string; - scenario: string; - captureBytes: number; - box: Rect; - wholeImage: PipelineSample; - region: PipelineSample; -}>; - -export async function sampleScenario( - capture: Capture, - scenario: CropScenario, - rounds: number, -): Promise { - const box = cropBoxOf(capture, scenario); - const wholeImageBytes = await wholeImageCrop(capture.bytes, box); - const regionBytes = await regionCrop(capture.bytes, box); - const wholeImage = await measureAsync(rounds, async () => { - await wholeImageCrop(capture.bytes, box); - }); - const region = await measureAsync(rounds, async () => { - await regionCrop(capture.bytes, box); - }); - return { - capture: capture.name, - label: capture.label, - scenario: scenario.name, - captureBytes: capture.bytes.length, - box, - wholeImage: { ...wholeImage, outBytes: wholeImageBytes }, - region: { ...region, outBytes: regionBytes }, - }; -} - -async function wholeImageCrop(source: Buffer, box: Rect): Promise { - const decoded = await decodePngAsync(source, 'capture'); - const cropped = new PNG({ width: box.width, height: box.height }); - for (let row = 0; row < box.height; row += 1) { - const from = ((row + box.y) * decoded.width + box.x) * 4; - decoded.data.copy(cropped.data, row * cropped.width * 4, from, from + box.width * 4); - } - return (await encodePngAsync(cropped)).length; -} - -async function regionCrop(source: Buffer, box: Rect): Promise { - const cropped = await cropPngBytesAsync(source, box, 'capture'); - return cropped === null ? source.length : cropped.length; -} diff --git a/scripts/png-crop-benchmark/report.ts b/scripts/png-crop-benchmark/report.ts deleted file mode 100644 index 1c82d6f471..0000000000 --- a/scripts/png-crop-benchmark/report.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { ScenarioSample } from './pipelines.ts'; -import { speedup } from './statistics.ts'; - -/** The benchmark's report: one row per capture and crop box, plus the corpus honesty check. */ - -const COLUMN_WIDTHS: readonly number[] = [30, 11, 13, 11, 10, 22]; - -export function renderReport(samples: readonly ScenarioSample[]): string { - const rows = [ - renderRow(['capture / crop', 'box', 'whole-image', 'region', 'speed-up', 'artifact']), - renderRow(['-', '-', '-', '-', '-', '-']), - ...samples.map((sample) => - renderRow([ - `${sample.capture} ${sample.scenario}`, - `${sample.box.width}x${sample.box.height}`, - `${sample.wholeImage.medianMs.toFixed(1)}ms`, - `${sample.region.medianMs.toFixed(1)}ms`, - `${speedup(sample.wholeImage.medianMs, sample.region.medianMs).toFixed(2)}x`, - `${kilobytes(sample.wholeImage.outBytes)} -> ${kilobytes(sample.region.outBytes)}`, - ]), - ), - ]; - return [...rows, '', footnotes(samples)].join('\n'); -} - -function footnotes(samples: readonly ScenarioSample[]): string { - const timings = 'median of the measured rounds; artifact = encoded crop, whole-image -> region'; - const corpus = [...new Map(samples.map((sample) => [sample.capture, sample])).values()].map( - (sample) => ` ${sample.label}: ${(sample.captureBytes / 1024).toFixed(0)} kB compressed`, - ); - return [ - timings, - '', - 'Corpus compressed sizes, so an unrealistic corpus is visible. A real 1206x2622 simulator', - 'capture of a UI screen is ~245 kB, and the same device showing a photo screen is ~3 MB.', - ...corpus, - ].join('\n'); -} - -function renderRow(cells: readonly string[]): string { - return cells - .map((cell, index) => cell.padEnd(COLUMN_WIDTHS[index] ?? cell.length)) - .join(' ') - .trimEnd(); -} - -function kilobytes(bytes: number): string { - return `${(bytes / 1024).toFixed(0)}kB`; -} diff --git a/scripts/png-crop-benchmark/run.ts b/scripts/png-crop-benchmark/run.ts deleted file mode 100644 index c3e0008520..0000000000 --- a/scripts/png-crop-benchmark/run.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { mkdirSync, writeFileSync } from 'node:fs'; -import path from 'node:path'; -import { terminatePngWorker } from '@agent-device/capture-kit/png-worker-client'; -import { parseBenchmarkArgs } from './args.ts'; -import { buildCorpus, CROP_SCENARIOS, readCaptureFile, type Capture } from './corpus.ts'; -import { sampleScenario, type ScenarioSample } from './pipelines.ts'; -import { renderReport } from './report.ts'; - -/** - * `pnpm bench:png-crop` — a cheap, device-free comparison of the two crop pipelines over generated - * captures. Pass `--file ` (repeatable) to add real captures to the same table. - */ - -const CORPUS_DIR = path.join('.tmp', 'png-crop-benchmark'); - -async function main(): Promise { - const options = parseBenchmarkArgs(process.argv.slice(2)); - const captures: Capture[] = [ - ...buildCorpus(CORPUS_DIR), - ...options.captureFiles.map((filePath, index) => readCaptureFile(filePath, index + 1)), - ]; - process.stderr.write( - `[png-crop-bench] ${captures.length} captures x ${CROP_SCENARIOS.length} crop boxes, ${options.rounds} rounds each\n`, - ); - try { - const samples = await sampleAll(captures, options.rounds); - process.stdout.write(`${renderReport(samples)}\n`); - writeJsonReport(options.jsonPath, options.rounds, samples); - } finally { - await terminatePngWorker(); - } -} - -async function sampleAll(captures: readonly Capture[], rounds: number): Promise { - const samples: ScenarioSample[] = []; - for (const capture of captures) { - for (const scenario of CROP_SCENARIOS) { - samples.push(await sampleScenario(capture, scenario, rounds)); - } - } - return samples; -} - -function writeJsonReport(jsonPath: string | undefined, rounds: number, samples: unknown): void { - if (jsonPath === undefined) return; - mkdirSync(path.dirname(path.resolve(jsonPath)), { recursive: true }); - writeFileSync(jsonPath, `${JSON.stringify({ rounds, samples }, null, 2)}\n`); - process.stderr.write(`[png-crop-bench] json: ${jsonPath}\n`); -} - -await main(); diff --git a/scripts/png-crop-benchmark/statistics.test.ts b/scripts/png-crop-benchmark/statistics.test.ts deleted file mode 100644 index 9f837af48a..0000000000 --- a/scripts/png-crop-benchmark/statistics.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { test } from 'vitest'; -import assert from 'node:assert/strict'; -import { speedup, summarize } from './statistics.ts'; - -test('summarize reports the middle of an odd sample set', () => { - assert.deepEqual(summarize([9, 1, 5]), { medianMs: 5, bestMs: 1, worstMs: 9 }); -}); - -test('summarize averages the two middle samples of an even set', () => { - assert.equal(summarize([4, 8, 2, 6]).medianMs, 5); -}); - -test('summarize survives an empty sample set', () => { - assert.deepEqual(summarize([]), { medianMs: 0, bestMs: 0, worstMs: 0 }); -}); - -test('speedup expresses the old pipeline in multiples of the new one', () => { - assert.equal(speedup(120, 40), 3); - assert.equal(speedup(120, 0), 0); -}); diff --git a/scripts/png-crop-benchmark/statistics.ts b/scripts/png-crop-benchmark/statistics.ts deleted file mode 100644 index 6a8ff74222..0000000000 --- a/scripts/png-crop-benchmark/statistics.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { performance } from 'node:perf_hooks'; - -/** A timing summary cheap enough to compute on every scenario. */ - -export type Timing = Readonly<{ medianMs: number; bestMs: number; worstMs: number }>; - -export async function measureAsync(rounds: number, run: () => Promise): Promise { - const samples: number[] = []; - for (let round = 0; round < rounds; round += 1) { - const started = performance.now(); - await run(); - samples.push(performance.now() - started); - } - return summarize(samples); -} - -export function summarize(samples: readonly number[]): Timing { - if (samples.length === 0) return { medianMs: 0, bestMs: 0, worstMs: 0 }; - const sorted = [...samples].sort((left, right) => left - right); - const middle = Math.floor(sorted.length / 2); - const median = - sorted.length % 2 === 1 - ? sorted[middle]! - : ((sorted.at(middle - 1) ?? 0) + (sorted[middle] ?? 0)) / 2; - return { - medianMs: median, - bestMs: sorted[0] ?? 0, - worstMs: sorted.at(-1) ?? 0, - }; -} - -export function speedup(wholeImageMs: number, regionMs: number): number { - return regionMs > 0 ? wholeImageMs / regionMs : 0; -} diff --git a/vitest.config.ts b/vitest.config.ts index 1bcd155b27..95f8c54173 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -167,8 +167,6 @@ export default defineConfig({ // the fixed mcp subcommand, or registry-format launchers run the bare CLI. 'scripts/__tests__/mcp-metadata.test.ts', 'scripts/ios-snapshot-benchmark/*.test.ts', - 'scripts/png-crop-benchmark/*.test.ts', - 'scripts/ios-ax-bridge-spike/*.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', 'test/ci/upload-artifact-hidden-paths.test.ts',