fix(mounts): exponential backoff + cap on forced mount restarts (storm fix)

src/main/integration-mounts.test.ts

@@ -122,7 +122,7 @@ vi.mock('./relayfile-mount-launcher', () => ({
   createPearMountLauncher: vi.fn((options) => ({ start: mock.startMount, __options: options }))
 }))
 
-import { IntegrationMountManager, readStalledInjectedRevisions } from './integration-mounts'
+import { IntegrationMountManager, readStalledInjectedRevisions, type MountHealthAlert } from './integration-mounts'
 
 describe('IntegrationMountManager', () => {
   beforeEach(() => {
@@ -652,7 +652,7 @@ describe('IntegrationMountManager', () => {
     expect(mock.mountInputs.filter((input) => input.remotePath === '/slack/channels/C123/threads')).toHaveLength(2)
   })
 
-  it('still checks sync wedge logs after a replayed auth state error', async () => {
+  it('backs off forced restarts after a replayed auth state error instead of respawning every poll', async () => {
     vi.useFakeTimers()
     vi.setSystemTime(new Date('2026-06-06T14:00:00.000Z'))
     const manager = new IntegrationMountManager()
@@ -684,14 +684,62 @@ describe('IntegrationMountManager', () => {
       }
     ])
 
+    const restarts = () => mock.mountInputs.filter((input) => input.remotePath === '/slack/channels/C123/threads').length
+
+    // First wedge poll restarts once (initial mount + 1 restart = 2).
     await vi.advanceTimersByTimeAsync(45_000)
     await Promise.resolve()
     await Promise.resolve()
+    expect(restarts()).toBe(2)
+
+    // A poll within the backoff window (next floor is 120s) must NOT respawn —
+    // this is the storm guard: previously stopHandle wiped the throttle so every
+    // ~45-60s poll respawned a fresh (FD-leaking) process.
     await vi.advanceTimersByTimeAsync(60_000)
     await Promise.resolve()
     await Promise.resolve()
+    expect(restarts()).toBe(2)
+
+    // Past the backoff window, recovery still proceeds (not permanently wedged).
+    await vi.advanceTimersByTimeAsync(90_000)
+    await Promise.resolve()
+    await Promise.resolve()
+    expect(restarts()).toBe(3)
+  })
+
+  it('pauses auto-restart and escalates after the consecutive-restart cap', async () => {
+    vi.useFakeTimers()
+    vi.setSystemTime(new Date('2026-06-06T14:00:00.000Z'))
+    const manager = new IntegrationMountManager()
+    const alerts: MountHealthAlert[] = []
+    manager.setHealthObserver((alert) => alerts.push(alert))
+    mock.readFile.mockImplementation(async (path: string) => {
+      if (path.endsWith('/.relay/mount.log')) {
+        return '2026/06/06 14:00:00 mount sync cycle failed: context deadline exceeded'
+      }
+      return JSON.stringify({
+        status: 'writeback-pending',
+        pendingWriteback: 1,
+        lastSuccessfulReconcileAt: '2026-06-06T13:59:00.000Z',
+        lastError: { code: 'unauthorized', statusCode: 401, message: 'http 401 unauthorized: Token has expired', at: '2026-06-06T14:00:30.000Z' }
+      })
+    })
 
-    expect(mock.mountInputs.filter((input) => input.remotePath === '/slack/channels/C123/threads')).toHaveLength(3)
+    await manager.ensureMounted([{ provider: 'slack', mountPaths: ['/slack/channels/C123/threads'] }])
+    const restarts = () => mock.mountInputs.filter((input) => input.remotePath === '/slack/channels/C123/threads').length
+
+    // Over ~50min the wedge polls drive restarts with exponential backoff
+    // (~45s, 180s, 450s, 945s, 1935s cumulative — floors 60/120/240/480/960s);
+    // the 5th trips the cap. The 6th is then gated by RESET (30min from the 5th
+    // ≈ 3735s), which this 3000s window stays below.
+    for (let i = 0; i < 10; i += 1) {
+      await vi.advanceTimersByTimeAsync(300_000)
+      await Promise.resolve()
+      await Promise.resolve()
+    }
+    // initial mount + 5 restarts; the 5th trips the cap + escalates exactly once.
+    expect(restarts()).toBe(6)
+    expect(alerts.filter((a) => a.type === 'restart-cap-exceeded')).toHaveLength(1)
   })
 
   it('does not restart when sync deadline failures are followed by a completed cycle', async () => {

src/main/integration-mounts.ts

@@ -15,6 +15,20 @@ const MOUNT_SYNC_TIMEOUT = '180s'
 const MOUNT_REFRESH_FALLBACK_MARGIN_MS = 5 * 60_000
 const MOUNT_REFRESH_MIN_DELAY_MS = 1_000
 const MOUNT_AUTH_RESTART_THROTTLE_MS = 60_000
+// Forced restarts back off exponentially from the 60s floor (60s, 120s, 240s …)
+// capped at 30min, instead of a flat 60s. A persistently-wedged mount used to
+// respawn every ~60s forever (364 restarts observed since 06-10), and each
+// fresh process re-leaks file descriptors from zero while it mirrors. After
+// MOUNT_RESTART_CONSECUTIVE_CAP rapid restarts we pause auto-restart and
+// escalate. A mount that runs clean for MOUNT_RESTART_RESET_MS resets its
+// counter (so the hourly token-refresh restart, which is far apart, never
+// accrues backoff).
+const MOUNT_RESTART_BACKOFF_MAX_MS = 30 * 60_000
+// Must exceed the largest backoff window actually used (60s * 2^(CAP-1) = 16min
+// at CAP=5) so a genuinely-stable mount resets the counter, but a mount merely
+// waiting out a long backoff window does NOT reset and can still reach the cap.
+const MOUNT_RESTART_RESET_MS = 30 * 60_000
+const MOUNT_RESTART_CONSECUTIVE_CAP = 5
 // Stays under MOUNT_AUTH_RESTART_THROTTLE_MS so back-to-back poll hits
 // coalesce into one restart instead of racing it.
 const MOUNT_HEALTH_POLL_INTERVAL_MS = 45_000
@@ -75,6 +89,11 @@ export type MountHealthAlert = {
   remotePath: string
   status: string | null
   lastSuccessfulReconcileAt: string | null
+} | {
+  type: 'restart-cap-exceeded'
+  remotePath: string
+  attempts: number
+  reason: string
 }
 
 type MountAuthRequiredReason = 'cloud-auth-required' | 'account-workspace-required'
@@ -187,6 +206,7 @@ export class IntegrationMountManager {
   private handles = new Map<string, MountedWorkspaceHandle>()
   private refreshTimers = new Map<string, ReturnType<typeof setTimeout>>()
   private authRestartedAt = new Map<string, number>()
+  private restartAttempts = new Map<string, number>()
   private workspaceId: string | null = null
   private pending: Promise<void> | null = null
   private desiredMountPaths: string[] = []
@@ -269,6 +289,7 @@ export class IntegrationMountManager {
     for (const timer of this.refreshTimers.values()) clearTimeout(timer)
     this.refreshTimers.clear()
     this.authRestartedAt.clear()
+    this.restartAttempts.clear()
     this.handledHealthErrorKeys.clear()
     this.lastAuthRequiredReason = null
     const roots = Array.from(this.handles.keys())
@@ -282,7 +303,12 @@ export class IntegrationMountManager {
     const timer = this.refreshTimers.get(providerRoot)
     if (timer) clearTimeout(timer)
     this.refreshTimers.delete(providerRoot)
-    this.authRestartedAt.delete(providerRoot)
+    // NB: do NOT clear authRestartedAt / restartAttempts here. stopHandle runs
+    // as part of a forced RESTART (mount() tears the handle down before
+    // respawning), so clearing the backoff state every restart is exactly what
+    // defeated the throttle and produced the respawn storm. The backoff counters
+    // are cleared only on genuine teardown (stopAll) or when a path is dropped
+    // from the desired set (see mount()).
     const handle = this.handles.get(providerRoot)
     this.handles.delete(providerRoot)
     if (handle) {
@@ -511,8 +537,26 @@ export class IntegrationMountManager {
     if (!this.handles.has(remotePath)) return false
     const now = Date.now()
     const lastRestartedAt = this.authRestartedAt.get(remotePath) ?? 0
-    if (now - lastRestartedAt < MOUNT_AUTH_RESTART_THROTTLE_MS) return false
+    const sinceLast = now - lastRestartedAt
+    // A mount that ran clean for a while resets its backoff — isolated restarts
+    // (incl. the hourly token refresh, far apart) get fast recovery.
+    const attempts = sinceLast >= MOUNT_RESTART_RESET_MS ? 0 : (this.restartAttempts.get(remotePath) ?? 0)
+    // Capped: we already burned the last-ditch restart at the cap and escalated.
+    // Stay paused until a stable period (RESET) elapses, instead of hot-looping
+    // a fresh (FD-leaking) process every cycle.
+    if (attempts >= MOUNT_RESTART_CONSECUTIVE_CAP) return false
+    // Exponential backoff floor on the 60s throttle, capped at 30min.
+    const throttle = Math.min(MOUNT_AUTH_RESTART_THROTTLE_MS * 2 ** attempts, MOUNT_RESTART_BACKOFF_MAX_MS)
+    if (lastRestartedAt && sinceLast < throttle) return false
+    const nextAttempts = attempts + 1
     this.authRestartedAt.set(remotePath, now)
+    this.restartAttempts.set(remotePath, nextAttempts)
+    if (nextAttempts >= MOUNT_RESTART_CONSECUTIVE_CAP) {
+      console.warn(
+        `[integration-mounts] Mount ${remotePath} reached ${MOUNT_RESTART_CONSECUTIVE_CAP} consecutive restarts (${reason}); pausing auto-restart for ${Math.round(MOUNT_RESTART_RESET_MS / 60_000)}min pending manual recovery`
+      )
+      this.healthObserver?.({ type: 'restart-cap-exceeded', remotePath, attempts: nextAttempts, reason })
+    }
 
     const previous = this.pending ?? Promise.resolve()
     const pending = previous
@@ -587,10 +631,13 @@ export class IntegrationMountManager {
     const mountSpecs = this.mountSpecsFor(mountPaths, mountRoot)
     const expectedRemotePaths = mountSpecs.map((spec) => spec.remotePath)
 
-    // Drop mounts for paths that are no longer connected.
+    // Drop mounts for paths that are no longer connected. This is a genuine
+    // teardown (not a restart), so also clear the backoff counters for them.
     for (const existingRemotePath of Array.from(this.handles.keys())) {
       if (!expectedRemotePaths.includes(existingRemotePath)) {
         await this.stopHandle(existingRemotePath)
+        this.authRestartedAt.delete(existingRemotePath)
+        this.restartAttempts.delete(existingRemotePath)
       }
     }