fix(browser-surface): persist refreshed browser connections

apps/console/package.json

@@ -19,7 +19,7 @@
   "dependencies": {
     "@base-ui/react": "^1.9.0",
     "@demodesk/neko": "1.6.32",
-    "@opendatalabs/remote-surface": "^1.5.2",
+    "@opendatalabs/remote-surface": "^1.6.0",
     "@pdpp/brand": "*",
     "@pdpp/brand-react": "*",
     "@pdpp/display": "*",

reference-implementation/package.json

@@ -108,13 +108,13 @@
   "devDependencies": {
     "@babel/parser": "8.0.7",
     "@biomejs/biome": "2.5.15",
-    "@types/better-sqlite3": "^9.6.0",
     "@pdpp/polyfill-connectors": "file:./vendor/pdpp-polyfill-connectors-0.0.1.tgz",
+    "@types/better-sqlite3": "^9.6.0",
     "jsdom": "^30.1.3",
     "pino-pretty": "^13.2.0",
     "ultracite": "^8.0.1"
   },
   "optionalDependencies": {
-    "@opendatalabs/remote-surface": "^1.5.3"
+    "@opendatalabs/remote-surface": "^1.6.0"
   }
 }

reference-implementation/runtime/browser-surface/run-coordinator.ts

@@ -157,6 +157,7 @@ interface ReclaimResolution {
 interface AllocatorSurfaceReconciliation {
   readonly downgraded: readonly BrowserSurface[];
   readonly evicted: readonly BrowserSurface[];
+  readonly refreshed: readonly BrowserSurface[];
 }
 
 // ─── Deps object ─────────────────────────────────────────────────────────────
@@ -323,6 +324,19 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
   const { allocator: replacementAwareAllocator } = replacementHooks;
   const connectorInstanceIdByRunId = new Map<string, string>();
   let browserSurfaceSweepInFlight = false;
+  let surfacePersistenceTail: Promise<void> = Promise.resolve();
+  let allocatorReconciliationInFlight: Promise<void> = Promise.resolve();
+
+  // Both stores allow asynchronous transactions. Keep this controller's
+  // full-row surface writes in order, including a refresh delayed by I/O.
+  function serializeSurfacePersistence<T>(action: () => Promise<T>): Promise<T> {
+    const work = surfacePersistenceTail.then(action);
+    surfacePersistenceTail = work.then(
+      () => undefined,
+      () => undefined
+    );
+    return work;
+  }
   const windowSettleReconciliation = createWindowSettleReconciliation({
     invalidateDeferredLease: invalidateBrowserSurfaceAfterProbeFailure,
     invalidateIdleSurface: invalidateIdleSurfaceAfterProbeFailure,
@@ -648,12 +662,15 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
     if (!browserSurfaceLeaseStore) {
       return;
     }
-    await browserSurfaceLeaseStore.withLeaseTransaction(async (store) => {
-      if (surface) {
-        await store.upsertSurface(surface);
-      }
-      await store.upsertLease(lease);
-    });
+    const leaseStore = browserSurfaceLeaseStore;
+    await serializeSurfacePersistence(() =>
+      leaseStore.withLeaseTransaction(async (store) => {
+        if (surface) {
+          await store.upsertSurface(surface);
+        }
+        await store.upsertLease(lease);
+      })
+    );
   }
 
   async function markSurfaceUnhealthy(
@@ -671,7 +688,10 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
       return;
     }
     await runBestEffort(
-      () => browserSurfaceLeaseStore.withLeaseTransaction((store) => markSurfaceUnhealthy(store, invalidatedSurface)),
+      () =>
+        serializeSurfacePersistence(() =>
+          browserSurfaceLeaseStore.withLeaseTransaction((store) => markSurfaceUnhealthy(store, invalidatedSurface))
+        ),
       (message) => log.warn?.(`[controller] persistence after surface invalidation failed: ${message}`)
     );
   }
@@ -1131,6 +1151,15 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
       // biome-ignore lint/performance/noAwaitInLoops: Work is intentionally sequential to preserve ordering and state transitions.
       await store.upsertSurface(surface);
     }
+    for (const surface of allocatorReconcile.refreshed) {
+      // A lease may have acquired/released while this write waited behind
+      // another transaction. Persist current authority, never the idle snapshot.
+      const current = browserSurfaceLeaseManager?.getSurface(surface.surface_id);
+      if (current) {
+        // biome-ignore lint/performance/noAwaitInLoops: Persist in transaction order.
+        await store.upsertSurface(current);
+      }
+    }
   }
 
   function requireBrowserSurfacePersistenceUnitOfWork(): BrowserSurfacePersistenceUnitOfWork {
@@ -1160,7 +1189,11 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
   }
 
   function hasAllocatorSurfaceReconciliation(allocatorReconcile: AllocatorSurfaceReconciliation): boolean {
-    return allocatorReconcile.evicted.length > 0 || allocatorReconcile.downgraded.length > 0;
+    return (
+      allocatorReconcile.evicted.length > 0 ||
+      allocatorReconcile.downgraded.length > 0 ||
+      allocatorReconcile.refreshed.length > 0
+    );
   }
 
   async function persistAllocatorSurfaceReconciliationInConfiguredStore(
@@ -1183,7 +1216,10 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
     if (!hasAllocatorSurfaceReconciliation(allocatorReconcile)) {
       return;
     }
-    await persistAllocatorSurfaceReconciliationInConfiguredStore(browserSurfaceLeaseStore, allocatorReconcile);
+    const leaseStore = browserSurfaceLeaseStore;
+    await serializeSurfacePersistence(() =>
+      persistAllocatorSurfaceReconciliationInConfiguredStore(leaseStore, allocatorReconcile)
+    );
   }
 
   function isLiveExternalLossCandidate(surface: BrowserSurface): boolean {
@@ -1229,14 +1265,18 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
     if (!(browserSurfaceLeaseManager && replacementAwareAllocator)) {
       return;
     }
-    try {
-      const allocatorReconcile =
-        await browserSurfaceLeaseManager.reconcileSurfacesWithAllocator(replacementAwareAllocator);
-      await persistAllocatorSurfaceReconciliation(allocatorReconcile);
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      log.warn?.(`[controller] allocator-aware surface reconciliation failed: ${message}`);
-    }
+    const leaseManager = browserSurfaceLeaseManager;
+    const allocator = replacementAwareAllocator;
+    allocatorReconciliationInFlight = allocatorReconciliationInFlight.then(async () => {
+      try {
+        const observed = await leaseManager.reconcileSurfacesWithAllocator(allocator);
+        await persistAllocatorSurfaceReconciliation(observed);
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        log.warn?.(`[controller] allocator-aware surface reconciliation failed: ${message}`);
+      }
+    });
+    await allocatorReconciliationInFlight;
   }
 
   // ─── Acquisition pipeline ─────────────────────────────────────────────────
@@ -1455,6 +1495,10 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
     if (!browserSurfaceLeaseManager) {
       throw new Error("browser surface lease manager required to acquire a managed surface lease");
     }
+    // Join an existing boot/periodic refresh; do not query the allocator on
+    // every acquire. This acquisition persists its full current surface before
+    // returning any dispatch environment, including after a failed refresh write.
+    await allocatorReconciliationInFlight;
     const { connectorId, connectorInstanceId, manifest, runId, traceContext } = ctx;
     const profileKey = readBrowserSurfaceProfileKey(connectorId, connectorInstanceId, manifest);
     const surfaceSubjectId = connectorInstanceId === connectorId ? undefined : connectorInstanceId;
@@ -1642,12 +1686,15 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
       log.warn?.(`[controller] browser-surface idle stop failed for ${failure.surface_id}: ${message}`);
     }
     if (browserSurfaceLeaseStore && cleanupResult.stopped.length > 0) {
-      await browserSurfaceLeaseStore.withLeaseTransaction(async (store) => {
-        for (const surface of cleanupResult.stopped) {
-          // biome-ignore lint/performance/noAwaitInLoops: Work is intentionally sequential to preserve ordering and state transitions.
-          await store.upsertSurface(surface);
-        }
-      });
+      const leaseStore = browserSurfaceLeaseStore;
+      await serializeSurfacePersistence(() =>
+        leaseStore.withLeaseTransaction(async (store) => {
+          for (const surface of cleanupResult.stopped) {
+            // biome-ignore lint/performance/noAwaitInLoops: Work is intentionally sequential to preserve ordering and state transitions.
+            await store.upsertSurface(surface);
+          }
+        })
+      );
     }
     await persistAndPromoteBrowserSurfaceLeases(cleanupResult.promoted, "browser-surface idle cleanup");
     return cleanupResult.promoted.map((lease) => projectBrowserSurfaceLease(lease));
@@ -1797,6 +1844,10 @@ export function createBrowserSurfaceManager(deps: BrowserSurfaceManagerDeps): Br
       releaseHostRun: true,
       boundRunIds,
     });
+    // The first pass preserves active leases. Restart reconciliation may now
+    // have released a dead run's lease; refresh that newly idle surface before
+    // boot completes, rather than waiting for the first periodic sweep.
+    await reconcileBrowserSurfacesWithAllocatorAtBoot();
     await windowSettleReconciliation.reconcileAtBoot(activeRunIds);
   }
 

reference-implementation/server/db.ts

@@ -1453,6 +1453,8 @@ CREATE TABLE IF NOT EXISTS browser_surfaces (
   profile_dir      TEXT,
   profile_volume   TEXT,
   browser_generation_hash TEXT,
+  allocator_metadata_json TEXT,
+  retained INTEGER CHECK (retained IS NULL OR retained IN (0, 1)),
   active_lease_id  TEXT,
   created_at       TEXT NOT NULL,
   last_used_at     TEXT NOT NULL,
@@ -6501,6 +6503,10 @@ function buildSqliteSchema(raw: SqliteDatabase, opts: InitDbOptions): void {
   runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "browser_surfaces", "profile_dir", "TEXT"));
   runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "browser_surfaces", "profile_volume", "TEXT"));
   runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "browser_surfaces", "browser_generation_hash", "TEXT"));
+  runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "browser_surfaces", "allocator_metadata_json", "TEXT"));
+  runWithSqliteBusyRetrySync(() =>
+    addColumnIfMissing(raw, "browser_surfaces", "retained", "INTEGER CHECK (retained IS NULL OR retained IN (0, 1))")
+  );
   // Dataset summary projection fencing: a `generation` column lets rebuild
   // and reconcile writers guard their final summary write against
   // concurrent record/blob delta writers. Pre-existing rows seed with 0;

reference-implementation/server/postgres-storage.ts

@@ -3086,7 +3086,9 @@ async function bootstrapPostgresSchemaOnce({
         ADD COLUMN IF NOT EXISTS profile_volume TEXT;
 
       ALTER TABLE browser_surfaces
-        ADD COLUMN IF NOT EXISTS browser_generation_hash TEXT;
+        ADD COLUMN IF NOT EXISTS browser_generation_hash TEXT,
+        ADD COLUMN IF NOT EXISTS allocator_metadata_json TEXT,
+        ADD COLUMN IF NOT EXISTS retained INTEGER CHECK (retained IS NULL OR retained IN (0, 1));
 
       -- Kind-neutral, run-grain durable projection. Historically
       -- scheduler-only (scheduler_run_history); generalized so the

reference-implementation/server/stores/browser-surface-lease-store.ts

@@ -29,6 +29,7 @@ interface Queryable {
 interface BrowserSurfaceRow {
   account_key: string | null;
   active_lease_id: string | null;
+  allocator_metadata_json: string | null;
   backend: BrowserSurface["backend"];
   browser_generation_hash: string | null;
   cdp_url: string;
@@ -41,6 +42,7 @@ interface BrowserSurfaceRow {
   profile_dir: string | null;
   profile_key: string;
   profile_volume: string | null;
+  retained: number | null;
   stream_base_url: string;
   stream_origin: string | null;
   surface_id: string;
@@ -175,7 +177,9 @@ function mapSurface(row: BrowserSurfaceRow | null | undefined): BrowserSurfaceWi
   if (!row) {
     return null;
   }
-  const surface = {
+  const surface: BrowserSurfaceWithPersistenceMetadata = {
+    ...(row.allocator_metadata_json ? { allocator_metadata: JSON.parse(row.allocator_metadata_json) } : {}),
+    ...(row.retained == null ? {} : { retained: row.retained === 1 }),
     backend: row.backend,
     cdp_url: row.cdp_url,
     connector_id: row.connector_id,
@@ -270,6 +274,8 @@ function sqliteSurfaceParams(surface: BrowserSurfaceWithPersistenceMetadata): Bi
     surface.active_lease_id ?? null,
     surface.created_at,
     surface.last_used_at,
+    surface.allocator_metadata == null ? null : JSON.stringify(surface.allocator_metadata),
+    surface.retained == null ? null : Number(surface.retained),
   ];
 }
 
@@ -406,9 +412,9 @@ class SqliteBrowserSurfaceLeaseStore implements BrowserSurfaceLeaseStore {
       `INSERT INTO browser_surfaces(
         surface_id, backend, profile_key, connector_id, surface_subject_id, account_key, surface_mode, surface_source,
         cdp_url, stream_base_url, stream_origin, window_settle_endpoint, health, container_id, container_name,
-        profile_dir, profile_volume, browser_generation_hash, active_lease_id, created_at, last_used_at
+        profile_dir, profile_volume, browser_generation_hash, active_lease_id, created_at, last_used_at, allocator_metadata_json, retained
       )
-      VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(surface_id) DO UPDATE SET
         backend = excluded.backend,
         profile_key = excluded.profile_key,
@@ -445,7 +451,9 @@ class SqliteBrowserSurfaceLeaseStore implements BrowserSurfaceLeaseStore {
         END,
         active_lease_id = excluded.active_lease_id,
         created_at = excluded.created_at,
-        last_used_at = excluded.last_used_at`,
+        last_used_at = excluded.last_used_at,
+        allocator_metadata_json = excluded.allocator_metadata_json,
+        retained = excluded.retained`,
       sqliteSurfaceParams(surface)
     );
     return Promise.resolve(surface);
@@ -662,9 +670,9 @@ class PostgresBrowserSurfaceLeaseStore implements BrowserSurfaceLeaseStore {
       `INSERT INTO browser_surfaces(
         surface_id, backend, profile_key, connector_id, surface_subject_id, account_key, surface_mode, surface_source,
         cdp_url, stream_base_url, stream_origin, window_settle_endpoint, health, container_id, container_name,
-        profile_dir, profile_volume, browser_generation_hash, active_lease_id, created_at, last_used_at
+        profile_dir, profile_volume, browser_generation_hash, active_lease_id, created_at, last_used_at, allocator_metadata_json, retained
       )
-      VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)
+      VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)
       ON CONFLICT(surface_id) DO UPDATE SET
         backend = EXCLUDED.backend,
         profile_key = EXCLUDED.profile_key,
@@ -701,7 +709,9 @@ class PostgresBrowserSurfaceLeaseStore implements BrowserSurfaceLeaseStore {
         END,
         active_lease_id = EXCLUDED.active_lease_id,
         created_at = EXCLUDED.created_at,
-        last_used_at = EXCLUDED.last_used_at`,
+        last_used_at = EXCLUDED.last_used_at,
+        allocator_metadata_json = EXCLUDED.allocator_metadata_json,
+        retained = EXCLUDED.retained`,
       sqliteSurfaceParams(surface)
     );
     return surface;

reference-implementation/test/browser-surface-lease-store.test.ts

@@ -1306,6 +1306,7 @@ test("Postgres store maps dynamic surface metadata with the same persistence sha
   const dynamicSurfaceRow = {
     account_key: dynamicSurface.account_key ?? null,
     active_lease_id: dynamicSurface.active_lease_id ?? null,
+    allocator_metadata_json: null,
     backend: dynamicSurface.backend,
     browser_generation_hash: dynamicSurface.browser_generation_hash ?? null,
     cdp_url: dynamicSurface.cdp_url,
@@ -1318,6 +1319,7 @@ test("Postgres store maps dynamic surface metadata with the same persistence sha
     profile_dir: dynamicSurface.profile_dir ?? null,
     profile_key: dynamicSurface.profile_key,
     profile_volume: dynamicSurface.profile_volume ?? null,
+    retained: null,
     stream_base_url: dynamicSurface.stream_base_url,
     stream_origin: dynamicSurface.stream_origin ?? null,
     surface_id: dynamicSurface.surface_id,
@@ -1362,6 +1364,8 @@ test("Postgres store maps dynamic surface metadata with the same persistence sha
     "lease_pg",
     "2026-05-12T12:00:00.000Z",
     "2026-05-12T12:00:00.000Z",
+    null,
+    null,
   ]);
   assert.deepEqual(await store.getSurface("surface_dynamic_pg"), dynamicSurface);
 });