From 7962f9639886c19ac5f84d0ae381c91be99ad800 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Tue, 27 Oct 2026 19:28:28 -0500 Subject: [PATCH 1/3] fix(browser-surface): persist refreshed browser connections Refresh newly idle surfaces after boot releases and keep asynchronous surface writes ordered before returning a connection for dispatch. Persist allocator metadata and retention across storage reloads. Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../browser-surface/run-coordinator.ts | 97 ++++-- reference-implementation/server/db.ts | 6 + .../server/postgres-storage.ts | 4 +- .../stores/browser-surface-lease-store.ts | 24 +- .../controller-browser-surface-leases.test.ts | 289 ++++++++++++++++++ 5 files changed, 389 insertions(+), 31 deletions(-) diff --git a/reference-implementation/runtime/browser-surface/run-coordinator.ts b/reference-implementation/runtime/browser-surface/run-coordinator.ts index 4347225d1..e4ada5884 100644 --- a/reference-implementation/runtime/browser-surface/run-coordinator.ts +++ b/reference-implementation/runtime/browser-surface/run-coordinator.ts @@ -157,6 +157,7 @@ interface ReclaimResolution { interface AllocatorSurfaceReconciliation { readonly downgraded: readonly BrowserSurface[]; readonly evicted: readonly BrowserSurface[]; + readonly refreshed: readonly BrowserSurface[]; } // ─── Deps object ───────────────────────────────────────────────────────────── @@ -323,6 +324,19 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br const { allocator: replacementAwareAllocator } = replacementHooks; const connectorInstanceIdByRunId = new Map(); let browserSurfaceSweepInFlight = false; + let surfacePersistenceTail: Promise = Promise.resolve(); + let allocatorReconciliationInFlight: Promise = Promise.resolve(); + + // Both stores allow asynchronous transactions. Keep this controller's + // full-row surface writes in order, including a refresh delayed by I/O. + function serializeSurfacePersistence(action: () => Promise): Promise { + const work = surfacePersistenceTail.then(action); + surfacePersistenceTail = work.then( + () => undefined, + () => undefined + ); + return work; + } const windowSettleReconciliation = createWindowSettleReconciliation({ invalidateDeferredLease: invalidateBrowserSurfaceAfterProbeFailure, invalidateIdleSurface: invalidateIdleSurfaceAfterProbeFailure, @@ -648,12 +662,15 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br if (!browserSurfaceLeaseStore) { return; } - await browserSurfaceLeaseStore.withLeaseTransaction(async (store) => { - if (surface) { - await store.upsertSurface(surface); - } - await store.upsertLease(lease); - }); + const leaseStore = browserSurfaceLeaseStore; + await serializeSurfacePersistence(() => + leaseStore.withLeaseTransaction(async (store) => { + if (surface) { + await store.upsertSurface(surface); + } + await store.upsertLease(lease); + }) + ); } async function markSurfaceUnhealthy( @@ -671,7 +688,10 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br return; } await runBestEffort( - () => browserSurfaceLeaseStore.withLeaseTransaction((store) => markSurfaceUnhealthy(store, invalidatedSurface)), + () => + serializeSurfacePersistence(() => + browserSurfaceLeaseStore.withLeaseTransaction((store) => markSurfaceUnhealthy(store, invalidatedSurface)) + ), (message) => log.warn?.(`[controller] persistence after surface invalidation failed: ${message}`) ); } @@ -1131,6 +1151,15 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br // biome-ignore lint/performance/noAwaitInLoops: Work is intentionally sequential to preserve ordering and state transitions. await store.upsertSurface(surface); } + for (const surface of allocatorReconcile.refreshed) { + // A lease may have acquired/released while this write waited behind + // another transaction. Persist current authority, never the idle snapshot. + const current = browserSurfaceLeaseManager?.getSurface(surface.surface_id); + if (current) { + // biome-ignore lint/performance/noAwaitInLoops: Persist in transaction order. + await store.upsertSurface(current); + } + } } function requireBrowserSurfacePersistenceUnitOfWork(): BrowserSurfacePersistenceUnitOfWork { @@ -1160,7 +1189,11 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br } function hasAllocatorSurfaceReconciliation(allocatorReconcile: AllocatorSurfaceReconciliation): boolean { - return allocatorReconcile.evicted.length > 0 || allocatorReconcile.downgraded.length > 0; + return ( + allocatorReconcile.evicted.length > 0 || + allocatorReconcile.downgraded.length > 0 || + allocatorReconcile.refreshed.length > 0 + ); } async function persistAllocatorSurfaceReconciliationInConfiguredStore( @@ -1183,7 +1216,10 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br if (!hasAllocatorSurfaceReconciliation(allocatorReconcile)) { return; } - await persistAllocatorSurfaceReconciliationInConfiguredStore(browserSurfaceLeaseStore, allocatorReconcile); + const leaseStore = browserSurfaceLeaseStore; + await serializeSurfacePersistence(() => + persistAllocatorSurfaceReconciliationInConfiguredStore(leaseStore, allocatorReconcile) + ); } function isLiveExternalLossCandidate(surface: BrowserSurface): boolean { @@ -1229,14 +1265,18 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br if (!(browserSurfaceLeaseManager && replacementAwareAllocator)) { return; } - try { - const allocatorReconcile = - await browserSurfaceLeaseManager.reconcileSurfacesWithAllocator(replacementAwareAllocator); - await persistAllocatorSurfaceReconciliation(allocatorReconcile); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - log.warn?.(`[controller] allocator-aware surface reconciliation failed: ${message}`); - } + const leaseManager = browserSurfaceLeaseManager; + const allocator = replacementAwareAllocator; + allocatorReconciliationInFlight = allocatorReconciliationInFlight.then(async () => { + try { + const observed = await leaseManager.reconcileSurfacesWithAllocator(allocator); + await persistAllocatorSurfaceReconciliation(observed); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn?.(`[controller] allocator-aware surface reconciliation failed: ${message}`); + } + }); + await allocatorReconciliationInFlight; } // ─── Acquisition pipeline ───────────────────────────────────────────────── @@ -1455,6 +1495,10 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br if (!browserSurfaceLeaseManager) { throw new Error("browser surface lease manager required to acquire a managed surface lease"); } + // Join an existing boot/periodic refresh; do not query the allocator on + // every acquire. This acquisition persists its full current surface before + // returning any dispatch environment, including after a failed refresh write. + await allocatorReconciliationInFlight; const { connectorId, connectorInstanceId, manifest, runId, traceContext } = ctx; const profileKey = readBrowserSurfaceProfileKey(connectorId, connectorInstanceId, manifest); const surfaceSubjectId = connectorInstanceId === connectorId ? undefined : connectorInstanceId; @@ -1642,12 +1686,15 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br log.warn?.(`[controller] browser-surface idle stop failed for ${failure.surface_id}: ${message}`); } if (browserSurfaceLeaseStore && cleanupResult.stopped.length > 0) { - await browserSurfaceLeaseStore.withLeaseTransaction(async (store) => { - for (const surface of cleanupResult.stopped) { - // biome-ignore lint/performance/noAwaitInLoops: Work is intentionally sequential to preserve ordering and state transitions. - await store.upsertSurface(surface); - } - }); + const leaseStore = browserSurfaceLeaseStore; + await serializeSurfacePersistence(() => + leaseStore.withLeaseTransaction(async (store) => { + for (const surface of cleanupResult.stopped) { + // biome-ignore lint/performance/noAwaitInLoops: Work is intentionally sequential to preserve ordering and state transitions. + await store.upsertSurface(surface); + } + }) + ); } await persistAndPromoteBrowserSurfaceLeases(cleanupResult.promoted, "browser-surface idle cleanup"); return cleanupResult.promoted.map((lease) => projectBrowserSurfaceLease(lease)); @@ -1797,6 +1844,10 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br releaseHostRun: true, boundRunIds, }); + // The first pass preserves active leases. Restart reconciliation may now + // have released a dead run's lease; refresh that newly idle surface before + // boot completes, rather than waiting for the first periodic sweep. + await reconcileBrowserSurfacesWithAllocatorAtBoot(); await windowSettleReconciliation.reconcileAtBoot(activeRunIds); } diff --git a/reference-implementation/server/db.ts b/reference-implementation/server/db.ts index 4621a8944..7bb43d7bd 100644 --- a/reference-implementation/server/db.ts +++ b/reference-implementation/server/db.ts @@ -1453,6 +1453,8 @@ CREATE TABLE IF NOT EXISTS browser_surfaces ( profile_dir TEXT, profile_volume TEXT, browser_generation_hash TEXT, + allocator_metadata_json TEXT, + retained INTEGER CHECK (retained IS NULL OR retained IN (0, 1)), active_lease_id TEXT, created_at TEXT NOT NULL, last_used_at TEXT NOT NULL, @@ -6501,6 +6503,10 @@ function buildSqliteSchema(raw: SqliteDatabase, opts: InitDbOptions): void { runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "browser_surfaces", "profile_dir", "TEXT")); runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "browser_surfaces", "profile_volume", "TEXT")); runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "browser_surfaces", "browser_generation_hash", "TEXT")); + runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "browser_surfaces", "allocator_metadata_json", "TEXT")); + runWithSqliteBusyRetrySync(() => + addColumnIfMissing(raw, "browser_surfaces", "retained", "INTEGER CHECK (retained IS NULL OR retained IN (0, 1))") + ); // Dataset summary projection fencing: a `generation` column lets rebuild // and reconcile writers guard their final summary write against // concurrent record/blob delta writers. Pre-existing rows seed with 0; diff --git a/reference-implementation/server/postgres-storage.ts b/reference-implementation/server/postgres-storage.ts index 1d6251036..05dbe3c00 100644 --- a/reference-implementation/server/postgres-storage.ts +++ b/reference-implementation/server/postgres-storage.ts @@ -3086,7 +3086,9 @@ async function bootstrapPostgresSchemaOnce({ ADD COLUMN IF NOT EXISTS profile_volume TEXT; ALTER TABLE browser_surfaces - ADD COLUMN IF NOT EXISTS browser_generation_hash TEXT; + ADD COLUMN IF NOT EXISTS browser_generation_hash TEXT, + ADD COLUMN IF NOT EXISTS allocator_metadata_json TEXT, + ADD COLUMN IF NOT EXISTS retained INTEGER CHECK (retained IS NULL OR retained IN (0, 1)); -- Kind-neutral, run-grain durable projection. Historically -- scheduler-only (scheduler_run_history); generalized so the diff --git a/reference-implementation/server/stores/browser-surface-lease-store.ts b/reference-implementation/server/stores/browser-surface-lease-store.ts index 4d11bac9a..75a5e393a 100644 --- a/reference-implementation/server/stores/browser-surface-lease-store.ts +++ b/reference-implementation/server/stores/browser-surface-lease-store.ts @@ -29,6 +29,7 @@ interface Queryable { interface BrowserSurfaceRow { account_key: string | null; active_lease_id: string | null; + allocator_metadata_json: string | null; backend: BrowserSurface["backend"]; browser_generation_hash: string | null; cdp_url: string; @@ -41,6 +42,7 @@ interface BrowserSurfaceRow { profile_dir: string | null; profile_key: string; profile_volume: string | null; + retained: number | null; stream_base_url: string; stream_origin: string | null; surface_id: string; @@ -175,7 +177,9 @@ function mapSurface(row: BrowserSurfaceRow | null | undefined): BrowserSurfaceWi if (!row) { return null; } - const surface = { + const surface: BrowserSurfaceWithPersistenceMetadata = { + ...(row.allocator_metadata_json ? { allocator_metadata: JSON.parse(row.allocator_metadata_json) } : {}), + ...(row.retained == null ? {} : { retained: row.retained === 1 }), backend: row.backend, cdp_url: row.cdp_url, connector_id: row.connector_id, @@ -270,6 +274,8 @@ function sqliteSurfaceParams(surface: BrowserSurfaceWithPersistenceMetadata): Bi surface.active_lease_id ?? null, surface.created_at, surface.last_used_at, + surface.allocator_metadata == null ? null : JSON.stringify(surface.allocator_metadata), + surface.retained == null ? null : Number(surface.retained), ]; } @@ -406,9 +412,9 @@ class SqliteBrowserSurfaceLeaseStore implements BrowserSurfaceLeaseStore { `INSERT INTO browser_surfaces( surface_id, backend, profile_key, connector_id, surface_subject_id, account_key, surface_mode, surface_source, cdp_url, stream_base_url, stream_origin, window_settle_endpoint, health, container_id, container_name, - profile_dir, profile_volume, browser_generation_hash, active_lease_id, created_at, last_used_at + profile_dir, profile_volume, browser_generation_hash, active_lease_id, created_at, last_used_at, allocator_metadata_json, retained ) - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(surface_id) DO UPDATE SET backend = excluded.backend, profile_key = excluded.profile_key, @@ -445,7 +451,9 @@ class SqliteBrowserSurfaceLeaseStore implements BrowserSurfaceLeaseStore { END, active_lease_id = excluded.active_lease_id, created_at = excluded.created_at, - last_used_at = excluded.last_used_at`, + last_used_at = excluded.last_used_at, + allocator_metadata_json = excluded.allocator_metadata_json, + retained = excluded.retained`, sqliteSurfaceParams(surface) ); return Promise.resolve(surface); @@ -662,9 +670,9 @@ class PostgresBrowserSurfaceLeaseStore implements BrowserSurfaceLeaseStore { `INSERT INTO browser_surfaces( surface_id, backend, profile_key, connector_id, surface_subject_id, account_key, surface_mode, surface_source, cdp_url, stream_base_url, stream_origin, window_settle_endpoint, health, container_id, container_name, - profile_dir, profile_volume, browser_generation_hash, active_lease_id, created_at, last_used_at + profile_dir, profile_volume, browser_generation_hash, active_lease_id, created_at, last_used_at, allocator_metadata_json, retained ) - VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) + VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) ON CONFLICT(surface_id) DO UPDATE SET backend = EXCLUDED.backend, profile_key = EXCLUDED.profile_key, @@ -701,7 +709,9 @@ class PostgresBrowserSurfaceLeaseStore implements BrowserSurfaceLeaseStore { END, active_lease_id = EXCLUDED.active_lease_id, created_at = EXCLUDED.created_at, - last_used_at = EXCLUDED.last_used_at`, + last_used_at = EXCLUDED.last_used_at, + allocator_metadata_json = EXCLUDED.allocator_metadata_json, + retained = EXCLUDED.retained`, sqliteSurfaceParams(surface) ); return surface; diff --git a/reference-implementation/test/controller-browser-surface-leases.test.ts b/reference-implementation/test/controller-browser-surface-leases.test.ts index 9275f43c0..5bfaf5ed1 100644 --- a/reference-implementation/test/controller-browser-surface-leases.test.ts +++ b/reference-implementation/test/controller-browser-surface-leases.test.ts @@ -3183,3 +3183,292 @@ test("a run acquiring during a pending idle stop does not lease the stopping sur assert.equal((await browserSurfaceLeaseStore.getLease("lease_2"))?.status, "released"); assert.equal(manager.getSurface("surface_2")?.active_lease_id, undefined); }); + +test("boot and periodic ready refresh survive durable reload before dispatch", async () => { + for (const health of ["ready", "unhealthy"] as const) { + for (const phase of ["boot", "periodic"] as const) { + const initial: BrowserSurfaceWithPersistenceMetadata = { + surface_id: "refresh-surface", + backend: "neko", + connector_id: "managed", + profile_key: "managed-profile", + health, + container_id: "prior-container", + cdp_url: "http://prior:9223/", + stream_base_url: "http://prior:8080/neko", + window_settle_endpoint: "http://prior:8080/window-settle", + allocator_metadata: { container_name: "prior", host_port: "31001" }, + retained: true, + created_at: "2026-05-12T12:00:00.000Z", + last_used_at: "2026-05-12T12:00:00.000Z", + }; + const replacement: BrowserSurfaceWithPersistenceMetadata = { + ...initial, + health: "ready", + container_id: "replacement-container", + cdp_url: "http://replacement:9223/", + stream_base_url: "http://replacement:8080/neko", + window_settle_endpoint: "http://replacement:8080/window-settle", + allocator_metadata: { container_name: "replacement", host_port: "31002" }, + }; + const store = createMemoryBrowserSurfaceLeaseStore({ surfaces: [initial] }); + const manager = createManager({ initialSurfaces: [initial], staticProfileKey: undefined }); + let statusCalls = 0; + const create = (leaseManager: BrowserSurfaceLeaseManager) => + createBrowserSurfaceManager({ + activeRunInteractions: new Map(), + browserSurfaceAllocator: { + ensureSurface: async () => { + throw new Error("unexpected allocation"); + }, + getSurfaceStatus: async () => { + statusCalls += 1; + return replacement; + }, + listSurfaces: async () => [replacement], + stopSurface: async () => null, + }, + browserSurfaceLeaseManager: leaseManager, + browserSurfaceLeaseStore: store, + browserSurfaceMidWaitPollIntervalMs: 0, + browserSurfaceReadinessProbe: null, + browserSurfaceReadinessTimeoutMs: undefined, + browserSurfaceReplacementReceiptStore: null, + listPersistedActiveRuns: async () => [], + log: { warn: () => undefined }, + pendingBrowserSurfaceLaunches: new Map(), + scheduleRun: () => undefined, + startupControllerRunReconciliation: Promise.resolve(), + }); + const controller = create(manager); + if (phase === "boot") { + await controller.reconcileBrowserSurfaceLeasesAfterBoot(); + } else { + await controller.sweepBrowserSurfaceLeases(); + } + const rows = await store.listSurfaces(); + assert.equal(rows[0]?.container_id, replacement.container_id); + assert.deepEqual(rows[0]?.allocator_metadata, replacement.allocator_metadata); + const reloaded = createManager({ initialSurfaces: rows, staticProfileKey: undefined }); + const refreshed = create(reloaded); + const observedCalls = statusCalls; + const acquired = await refreshed.acquireManagedBrowserSurfaceForRun({ + automationMetadata: { automation_mode: "manual_only", automation_summary: "", trigger_kind: "retry" }, + connectorId: "managed", + connectorInstanceId: "managed", + manifest: MANIFEST, + options: {}, + runId: "refresh-run", + traceContext: createTraceContext(), + }); + assert.equal(acquired.kind, "ready"); + if (acquired.kind !== "ready" || !acquired.lease) { + throw new Error("missing ready lease"); + } + assert.equal(acquired.lease.surface_id, initial.surface_id); + assert.ok(acquired.lease.surface_id); + const bound = reloaded.getSurface(acquired.lease.surface_id); + assert.equal(bound?.container_id, replacement.container_id); + assert.equal( + (bound as BrowserSurfaceWithPersistenceMetadata | undefined)?.window_settle_endpoint, + replacement.window_settle_endpoint + ); + assert.equal(bound?.retained, true); + assert.equal(bound?.created_at, initial.created_at); + assert.equal(acquired.env?.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL, replacement.cdp_url); + assert.equal(acquired.env?.PDPP_BROWSER_SURFACE_STREAM_BASE_URL, replacement.stream_base_url); + assert.equal(store.surfaces.get(acquired.lease.surface_id)?.active_lease_id, acquired.lease.lease_id); + assert.equal(statusCalls, observedCalls); + } + } +}); + +test("SQLite surface runtime metadata and retention survive database reopen", async (t) => { + closeDb(); + const path = tempDbPath(); + initDb(path); + t.after(() => closeDb()); + const surface: BrowserSurfaceWithPersistenceMetadata = { + surface_id: "persisted-refresh", + backend: "neko", + connector_id: "managed", + profile_key: "managed-profile", + health: "ready", + cdp_url: "http://replacement:9223/", + stream_base_url: "http://replacement:8080/neko", + window_settle_endpoint: "http://replacement:8080/window-settle", + allocator_metadata: { container_name: "replacement", host_port: "31002" }, + retained: true, + created_at: "2026-05-12T12:00:00.000Z", + last_used_at: "2026-05-12T12:00:00.000Z", + }; + await createSqliteBrowserSurfaceLeaseStore().upsertSurface(surface); + closeDb(); + initDb(path); + const reloaded = await createSqliteBrowserSurfaceLeaseStore().getSurface(surface.surface_id); + assert.deepEqual(reloaded, surface); + assert.ok(reloaded); + const manager = createDynamicManager({ initialSurfaces: [reloaded], now: () => new Date("2026-05-13T12:00:00Z") }); + const allocator = createReadyAllocator(); + assert.deepEqual((await manager.cleanupIdleSurfaces(allocator)).stopped, []); + assert.equal(allocator.stopRequests.length, 0); + assert.equal(manager.getSurface(surface.surface_id)?.retained, true); +}); + +function createReadyRefreshFixture() { + const initial: BrowserSurfaceWithPersistenceMetadata = { + surface_id: "refresh-race-surface", + backend: "neko", + connector_id: "managed", + profile_key: "managed-profile", + health: "ready", + container_id: "old-container", + cdp_url: "http://old:9223/", + stream_base_url: "http://old:8080/neko", + created_at: "2026-05-12T12:00:00.000Z", + last_used_at: "2026-05-12T12:00:00.000Z", + }; + const replacement: BrowserSurfaceWithPersistenceMetadata = { + ...initial, + container_id: "new-container", + cdp_url: "http://new:9223/", + stream_base_url: "http://new:8080/neko", + }; + const store = createMemoryBrowserSurfaceLeaseStore({ surfaces: [initial] }); + const leaseManager = createDynamicManager({ initialSurfaces: [initial] }); + const activeRunIds = new Set(); + const allocator: BrowserSurfaceAllocator = { + ensureSurface: async () => { throw new Error("unexpected allocation"); }, + getSurfaceStatus: async () => replacement, + listSurfaces: async () => [replacement], + stopSurface: async () => null, + }; + const manager = createBrowserSurfaceManager({ + activeRunInteractions: new Map(), + browserSurfaceAllocator: allocator, + browserSurfaceLeaseManager: leaseManager, + browserSurfaceLeaseStore: store, + browserSurfaceMidWaitPollIntervalMs: 0, + browserSurfaceReadinessProbe: null, + browserSurfaceReadinessTimeoutMs: undefined, + browserSurfaceReplacementReceiptStore: null, + listPersistedActiveRuns: async () => [...activeRunIds].map((run_id) => ({ run_id })), + log: { warn: () => undefined }, + pendingBrowserSurfaceLaunches: new Map(), + scheduleRun: () => undefined, + startupControllerRunReconciliation: Promise.resolve(), + }); + const acquire = (runId: string) => manager.acquireManagedBrowserSurfaceForRun({ + automationMetadata: { automation_mode: "manual_only", automation_summary: "", trigger_kind: "retry" }, + connectorId: "managed", + connectorInstanceId: "managed", + manifest: MANIFEST, + options: {}, + runId, + traceContext: createTraceContext(), + }); + return { acquire, activeRunIds, allocator, initial, leaseManager, manager, replacement, store }; +} + +test("boot refreshes a surface after releasing its dead-run lease before the first acquire", async () => { + const f = createReadyRefreshFixture(); + const prior = await f.acquire("dead-run"); + assert.equal(prior.kind, "ready"); + if (prior.kind !== "ready" || !prior.lease) throw new Error("expected prior lease"); + assert.equal(f.store.surfaces.get(f.initial.surface_id)?.active_lease_id, prior.lease.lease_id); + await f.manager.reconcileBrowserSurfaceLeasesAfterBoot(); + assert.equal(f.store.leases.get(prior.lease.lease_id)?.status, "released"); + assert.equal(f.store.surfaces.get(f.initial.surface_id)?.container_id, f.replacement.container_id); + const acquired = await f.acquire("after-boot"); + assert.equal(acquired.kind, "ready"); + if (acquired.kind !== "ready" || !acquired.lease) throw new Error("expected ready lease"); + assert.equal(acquired.lease.surface_id, f.initial.surface_id); + assert.ok(acquired.lease.fencing_token > prior.lease.fencing_token); + assert.equal(acquired.env?.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL, f.replacement.cdp_url); + assert.equal(f.store.surfaces.get(acquired.lease.surface_id)?.active_lease_id, acquired.lease.lease_id); +}); + +test("boot does not retarget a retained active run or change its lease fencing", async () => { + const f = createReadyRefreshFixture(); + const acquired = await f.acquire("live-run"); + if (acquired.kind !== "ready" || !acquired.lease) throw new Error("expected ready lease"); + f.activeRunIds.add("live-run"); + const prior = structuredClone(f.leaseManager.getLease(acquired.lease.lease_id)); + await f.manager.reconcileBrowserSurfaceLeasesAfterBoot(); + assert.deepEqual(f.leaseManager.getLease(acquired.lease.lease_id), prior); + assert.equal(f.leaseManager.getSurface(f.initial.surface_id)?.container_id, f.initial.container_id); + assert.equal(f.store.surfaces.get(f.initial.surface_id)?.container_id, f.initial.container_id); + assert.equal(f.store.surfaces.get(f.initial.surface_id)?.active_lease_id, acquired.lease.lease_id); +}); + +test("acquire joins a pending allocator observation before selecting its runtime", async () => { + const f = createReadyRefreshFixture(); + let enter: () => void = () => undefined; + let resume: () => void = () => undefined; + const entered = new Promise((resolve) => { enter = resolve; }); + const gate = new Promise((resolve) => { resume = resolve; }); + f.allocator.getSurfaceStatus = async () => { enter(); await gate; return f.replacement; }; + const sweep = f.manager.sweepBrowserSurfaceLeases(); + await entered; + const acquire = f.acquire("joined-run"); + resume(); + await sweep; + const acquired = await acquire; + assert.equal(acquired.kind, "ready"); + if (acquired.kind !== "ready" || !acquired.lease) throw new Error("expected ready lease"); + assert.equal(f.leaseManager.getSurface(acquired.lease.surface_id)?.container_id, f.replacement.container_id); + assert.equal(f.store.surfaces.get(acquired.lease.surface_id)?.container_id, f.replacement.container_id); + assert.equal(acquired.env?.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL, f.replacement.cdp_url); +}); + +test("a transient refresh-only write failure recovers through acquisition's own durable write", async () => { + const f = createReadyRefreshFixture(); + const upsert = f.store.upsertSurface.bind(f.store); + let failed = false; + f.store.upsertSurface = (surface) => { + if (!failed) { failed = true; return Promise.reject(new Error("transient refresh write failure")); } + return upsert(surface); + }; + await f.manager.sweepBrowserSurfaceLeases(); + assert.equal(failed, true); + assert.equal(f.store.surfaces.get(f.initial.surface_id)?.container_id, f.initial.container_id); + const acquired = await f.acquire("store-recovered"); + assert.equal(acquired.kind, "ready"); + if (acquired.kind !== "ready" || !acquired.lease) throw new Error("expected ready lease"); + assert.equal(acquired.env?.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL, f.replacement.cdp_url); + assert.equal(f.store.surfaces.get(acquired.lease.surface_id)?.container_id, f.replacement.container_id); + assert.equal(f.store.surfaces.get(acquired.lease.surface_id)?.active_lease_id, acquired.lease.lease_id); +}); + +test("refresh queued behind a release persists current authority with asynchronous transactions", async () => { + const f = createReadyRefreshFixture(); + const first = await f.acquire("first-run"); + if (first.kind !== "ready" || !first.lease) throw new Error("expected ready lease"); + let enter: () => void = () => undefined; + let resume: () => void = () => undefined; + const entered = new Promise((resolve) => { enter = resolve; }); + const gate = new Promise((resolve) => { resume = resolve; }); + const upsert = f.store.upsertSurface.bind(f.store); + let once = true; + f.store.upsertSurface = async (surface) => { + if (once && !surface.active_lease_id) { once = false; enter(); await gate; } + return upsert(surface); + }; + // The fake store deliberately does not serialize transactions itself. + const release = f.manager.releaseLease(first.lease, "managed", "first-run", createTraceContext()); + await entered; + const sweep = f.manager.sweepBrowserSurfaceLeases(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(f.leaseManager.getSurface(f.initial.surface_id)?.container_id, f.replacement.container_id); + const newer = f.leaseManager.acquire({ + connectorId: "managed", profileKey: f.initial.profile_key, runId: "concurrent-lease", + }); + assert.equal(newer.lease.status, "leased"); + resume(); + await Promise.all([release, sweep]); + const row = f.store.surfaces.get(f.initial.surface_id); + assert.equal(row?.container_id, f.replacement.container_id); + assert.equal(row?.active_lease_id, newer.lease.lease_id); + assert.ok(newer.lease.fencing_token > first.lease.fencing_token); + assert.equal(f.store.leases.get(first.lease.lease_id)?.status, "released"); +}); From 79ccb0cf64e4af3ee34b2471e0377fed5270d047 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Tue, 27 Oct 2026 19:47:57 -0500 Subject: [PATCH 2/3] test(browser-surface): align metadata persistence fixtures Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../test/browser-surface-lease-store.test.ts | 4 ++++ .../test/controller-browser-surface-leases.test.ts | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/reference-implementation/test/browser-surface-lease-store.test.ts b/reference-implementation/test/browser-surface-lease-store.test.ts index 13eab5bd2..bcfade3ba 100644 --- a/reference-implementation/test/browser-surface-lease-store.test.ts +++ b/reference-implementation/test/browser-surface-lease-store.test.ts @@ -1306,6 +1306,7 @@ test("Postgres store maps dynamic surface metadata with the same persistence sha const dynamicSurfaceRow = { account_key: dynamicSurface.account_key ?? null, active_lease_id: dynamicSurface.active_lease_id ?? null, + allocator_metadata_json: null, backend: dynamicSurface.backend, browser_generation_hash: dynamicSurface.browser_generation_hash ?? null, cdp_url: dynamicSurface.cdp_url, @@ -1318,6 +1319,7 @@ test("Postgres store maps dynamic surface metadata with the same persistence sha profile_dir: dynamicSurface.profile_dir ?? null, profile_key: dynamicSurface.profile_key, profile_volume: dynamicSurface.profile_volume ?? null, + retained: null, stream_base_url: dynamicSurface.stream_base_url, stream_origin: dynamicSurface.stream_origin ?? null, surface_id: dynamicSurface.surface_id, @@ -1362,6 +1364,8 @@ test("Postgres store maps dynamic surface metadata with the same persistence sha "lease_pg", "2026-05-12T12:00:00.000Z", "2026-05-12T12:00:00.000Z", + null, + null, ]); assert.deepEqual(await store.getSurface("surface_dynamic_pg"), dynamicSurface); }); diff --git a/reference-implementation/test/controller-browser-surface-leases.test.ts b/reference-implementation/test/controller-browser-surface-leases.test.ts index 5bfaf5ed1..0876947a9 100644 --- a/reference-implementation/test/controller-browser-surface-leases.test.ts +++ b/reference-implementation/test/controller-browser-surface-leases.test.ts @@ -3416,8 +3416,10 @@ test("acquire joins a pending allocator observation before selecting its runtime const acquired = await acquire; assert.equal(acquired.kind, "ready"); if (acquired.kind !== "ready" || !acquired.lease) throw new Error("expected ready lease"); - assert.equal(f.leaseManager.getSurface(acquired.lease.surface_id)?.container_id, f.replacement.container_id); - assert.equal(f.store.surfaces.get(acquired.lease.surface_id)?.container_id, f.replacement.container_id); + const surfaceId = acquired.lease.surface_id; + assert.ok(surfaceId); + assert.equal(f.leaseManager.getSurface(surfaceId)?.container_id, f.replacement.container_id); + assert.equal(f.store.surfaces.get(surfaceId)?.container_id, f.replacement.container_id); assert.equal(acquired.env?.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL, f.replacement.cdp_url); }); @@ -3435,9 +3437,11 @@ test("a transient refresh-only write failure recovers through acquisition's own const acquired = await f.acquire("store-recovered"); assert.equal(acquired.kind, "ready"); if (acquired.kind !== "ready" || !acquired.lease) throw new Error("expected ready lease"); + const surfaceId = acquired.lease.surface_id; + assert.ok(surfaceId); assert.equal(acquired.env?.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL, f.replacement.cdp_url); - assert.equal(f.store.surfaces.get(acquired.lease.surface_id)?.container_id, f.replacement.container_id); - assert.equal(f.store.surfaces.get(acquired.lease.surface_id)?.active_lease_id, acquired.lease.lease_id); + assert.equal(f.store.surfaces.get(surfaceId)?.container_id, f.replacement.container_id); + assert.equal(f.store.surfaces.get(surfaceId)?.active_lease_id, acquired.lease.lease_id); }); test("refresh queued behind a release persists current authority with asynchronous transactions", async () => { From d05bc7027074cc99e4de8284f6de0e3555036e33 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Tue, 27 Oct 2026 20:08:28 -0500 Subject: [PATCH 3/3] build(deps): require remote-surface 1.6.0 Both clients need the refreshed allocator metadata API before controller acquisition can persist replacement browser connections. Assisted-by: AI Signed-off-by: Tim Nunamaker --- apps/console/package.json | 2 +- package-lock.json | 10 +++++----- reference-implementation/package.json | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/console/package.json b/apps/console/package.json index 2d05a6ee7..6d6dd57e9 100644 --- a/apps/console/package.json +++ b/apps/console/package.json @@ -19,7 +19,7 @@ "dependencies": { "@base-ui/react": "^1.9.0", "@demodesk/neko": "1.6.32", - "@opendatalabs/remote-surface": "^1.5.2", + "@opendatalabs/remote-surface": "^1.6.0", "@pdpp/brand": "*", "@pdpp/brand-react": "*", "@pdpp/display": "*", diff --git a/package-lock.json b/package-lock.json index 3750d0652..e109d1536 100644 --- a/package-lock.json +++ b/package-lock.json @@ -86,7 +86,7 @@ "dependencies": { "@base-ui/react": "^1.9.0", "@demodesk/neko": "1.6.32", - "@opendatalabs/remote-surface": "^1.5.2", + "@opendatalabs/remote-surface": "^1.6.0", "@pdpp/brand": "*", "@pdpp/brand-react": "*", "@pdpp/display": "*", @@ -3574,9 +3574,9 @@ "license": "MIT" }, "node_modules/@opendatalabs/remote-surface": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/@opendatalabs/remote-surface/-/remote-surface-1.5.3.tgz", - "integrity": "sha512-hKMD47Mi9lis7o28F+MZDxhkb53ARxNohExJXzYC1WstzII9K7vxAjnCo8ljVMW1njneW9csi+HWMB8Z46uuXg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@opendatalabs/remote-surface/-/remote-surface-1.6.0.tgz", + "integrity": "sha512-CkftQiY2tVfmyI6pjrtKkKFZziUXK1gJ2ypsWjUlSa5qc7yWZC5RsPDrYrD07ZABIoBRgd0jUIrECjcF9wUbZw==", "license": "Apache-2.0", "dependencies": { "transformation-matrix": "^3.1.0" @@ -23816,7 +23816,7 @@ "ultracite": "^8.0.1" }, "optionalDependencies": { - "@opendatalabs/remote-surface": "^1.5.3" + "@opendatalabs/remote-surface": "^1.6.0" } }, "reference-implementation/node_modules/@babel/helper-string-parser": { diff --git a/reference-implementation/package.json b/reference-implementation/package.json index c0827d31a..967752fe8 100644 --- a/reference-implementation/package.json +++ b/reference-implementation/package.json @@ -108,13 +108,13 @@ "devDependencies": { "@babel/parser": "8.0.7", "@biomejs/biome": "2.5.15", - "@types/better-sqlite3": "^9.6.0", "@pdpp/polyfill-connectors": "file:./vendor/pdpp-polyfill-connectors-0.0.1.tgz", + "@types/better-sqlite3": "^9.6.0", "jsdom": "^30.1.3", "pino-pretty": "^13.2.0", "ultracite": "^8.0.1" }, "optionalDependencies": { - "@opendatalabs/remote-surface": "^1.5.3" + "@opendatalabs/remote-surface": "^1.6.0" } }