fix(ssh): retire stranded relays and remote tabs after an app update

docs/reference/ssh-execution-boundary.md

@@ -40,11 +40,15 @@ Two ways remote work _can_ actually stop:
 
 Reconnect re-attaches to the same live PTYs and replays a bounded buffer (`REPLAY_BUFFER_MAX`, a 102,400-code-unit tail). Output beyond that while you were away is lost to the client even though the process was never interrupted: **the transcript is truncated; the work stays `live`.**
 
-## Updating Orca strands relay-backed terminals
+## Updating Orca retires relay-backed terminals
 
-There is a third outcome that is neither of the two above, and the vocabulary matters: the work does not stop, it becomes permanently unreachable.
+There is a third outcome that is neither of the two above: an app update that changes the relay bundle hash starts a new generation. The work on the old generation is not reattached.
 
-The relay's install directory — and therefore its socket path — is namespaced by a content hash of the relay bundle (`computeRemoteRelayDir` in `src/main/ssh/ssh-relay-versioned-install.ts`, consumed by `resolveRemoteInstallState` in `src/main/ssh/ssh-relay-deploy.ts`), and the daemon refuses any client whose bundle hash differs (`handleDaemonHandshakeFrame` in `src/relay/relay-handshake.ts`, exit `EXIT_CODE_VERSION_MISMATCH` 42). Two builds whose relay protocol is byte-identical still refuse each other. So the first reconnect after an app update deploys a new relay at a path the incumbent was never listening on, and cannot reach it even in principle. Every PTY the incumbent owns is `unverifiable` — running, unreachable, and never `exited`. The client's leases are attempted against a relay that never minted their ids, expired on the not-found answer (`handlePtyReattachFailure` in `src/main/ssh/ssh-relay-session.ts`), and the pane falls back to a cold-restore agent resume — or to a bare shell when no resumable provider session was captured for it. The old relay keeps its directory pinned against GC, because its socket really is live (`hasLiveRelaySocket` in `src/main/ssh/remote-install-gc.ts`). See #13852.
+The relay's install directory — and therefore its socket path — is namespaced by a content hash of the relay bundle (`computeRemoteRelayDir` in `src/main/ssh/ssh-relay-versioned-install.ts`, consumed by `resolveRemoteInstallState` in `src/main/ssh/ssh-relay-deploy.ts`), and the daemon refuses any client whose bundle hash differs (`handleDaemonHandshakeFrame` in `src/relay/relay-handshake.ts`, exit `EXIT_CODE_VERSION_MISMATCH` 42). Two builds whose relay protocol is byte-identical still refuse each other. So the first reconnect after an app update deploys a new relay at a path the incumbent was never listening on, and cannot reach it even in principle.
+
+That unreachable generation is then retired, not left running. If the old relay is the sole socket holder — no other Orca still connected — the superseded sweep SIGTERMs it even when it still holds PTYs (`isReapableSupersededRelay` in `src/main/ssh/ssh-relay-endpoint-incumbent.ts`). The client expires its leases, does not cold-restore those panes, and closes the remote terminal tabs for that host. Open new terminals after updating. A probe that cannot answer stays `unverifiable` and is not signalled; a relay with another client still attached is retained.
+
+Ordinary disconnect and app quit are unchanged: `--grace-time 0` still keeps remote work alive until you reconnect to the _same_ relay generation. See #13852.
 
 The peer model does not have this failure, and that is the concrete reason behind "One host, one model" below. The daemon's endpoint is namespaced by a **semantic protocol version** rather than a build (`daemon-v<N>.sock`, from `getDaemonSocketPath` in `src/main/daemon/daemon-spawner.ts`), every earlier protocol version stays attachable (`PROTOCOL_VERSION` in `src/main/daemon/daemon-protocol-version.ts`), and a daemon holding live sessions is preserved across a version change instead of replaced (`shouldPreserveDaemonWithLiveSessions` in `src/main/daemon/daemon-replacement-preflight.ts`).
 

src/main/ssh/ssh-relay-endpoint-incumbent-shell.integration.test.ts

@@ -15,7 +15,10 @@ import {
   relayEndpointIncumbentProbeCommand,
   type RelayEndpointIncumbent
 } from './ssh-relay-endpoint-incumbent'
-import { reapEmptyRelayHuskCommand } from './ssh-relay-endpoint-takeover'
+import {
+  reapEmptyRelayHuskCommand,
+  reapSupersededRelayCommand
+} from './ssh-relay-endpoint-takeover'
 import { RELAY_DAEMON_SERVICE_ENTRY_FILENAMES } from '../../shared/relay-artifacts'
 
 const posixOnly = process.platform === 'win32' ? describe.skip : describe
@@ -239,6 +242,13 @@ posixOnly('empty relay husk reap against a real process', () => {
     expect(relay.killed).toBe(false)
   })
 
+  it('signals a superseded relay that still holds work', async () => {
+    const sockPath = join(workDir, 'superseded-work.sock')
+    const relay = await startFakeRelay(sockPath, { withChild: true })
+    const output = await sh(reapSupersededRelayCommand(relay.pid!, sockPath))
+    expect(output.trim()).toBe('GONE')
+  })
+
   it('terminates a relay whose only children are its own service processes (#13614)', async () => {
     const sockPath = join(workDir, 'service-husk.sock')
     const relay = await startFakeRelay(sockPath, {

src/main/ssh/ssh-relay-endpoint-incumbent.test.ts

@@ -11,6 +11,7 @@ vi.mock('./ssh-relay-deploy-helpers', () => ({
 import {
   describeRelayEndpointIncumbent,
   isReapableRelayHusk,
+  isReapableSupersededRelay,
   mayLaunchOverRelayEndpoint,
   parseRelayEndpointIncumbentProbe,
   probeRelayEndpointIncumbent,
@@ -291,6 +292,39 @@ describe('isReapableRelayHusk', () => {
   })
 })
 
+describe('isReapableSupersededRelay', () => {
+  const husk = parseRelayEndpointIncumbentProbe(
+    SOCK,
+    probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=500 yes 0 0'])
+  )
+
+  it('accepts a sole holder even when it still has unaccounted-for children', () => {
+    expect(
+      isReapableSupersededRelay({
+        ...husk,
+        holders: [{ pid: 500, matchesRelayArgv: true, childCount: 13, unrecognizedChildCount: 11 }]
+      })
+    ).toBe(true)
+  })
+
+  it('still refuses when another process holds the socket', () => {
+    expect(
+      isReapableSupersededRelay({
+        ...husk,
+        holders: [
+          { pid: 500, matchesRelayArgv: true, childCount: 13, unrecognizedChildCount: 11 },
+          { pid: 501, matchesRelayArgv: false, childCount: 0, unrecognizedChildCount: 0 }
+        ]
+      })
+    ).toBe(false)
+  })
+
+  it('still refuses unverifiable endpoints', () => {
+    expect(isReapableSupersededRelay({ ...husk, verdict: 'unverifiable' })).toBe(false)
+    expect(isReapableSupersededRelay({ ...husk, holdersEnumerable: false })).toBe(false)
+  })
+})
+
 describe('describeRelayEndpointIncumbent', () => {
   it('distinguishes "no holders" from "could not enumerate holders"', () => {
     const none = parseRelayEndpointIncumbentProbe(

src/main/ssh/ssh-relay-endpoint-incumbent.ts

@@ -279,6 +279,16 @@ export function mayLaunchOverRelayEndpoint(incumbent: RelayEndpointIncumbent): b
   return incumbent.verdict !== 'live'
 }
 
+function isSoleIdentifiedRelayHolder(incumbent: RelayEndpointIncumbent): boolean {
+  if (incumbent.verdict !== 'live' || !incumbent.holdersEnumerable) {
+    return false
+  }
+  if (incumbent.holders.length !== 1) {
+    return false
+  }
+  return incumbent.holders[0]?.matchesRelayArgv === true
+}
+
 /**
  * A live relay that provably holds nothing: identity confirmed against its argv, exactly one
  * holder, and no child the host could not account for as one of the daemon's own service
@@ -289,14 +299,18 @@ export function mayLaunchOverRelayEndpoint(incumbent: RelayEndpointIncumbent): b
  * gate was unreachable for any relay that had ever served a vault request (#13614).
  */
 export function isReapableRelayHusk(incumbent: RelayEndpointIncumbent): boolean {
-  if (incumbent.verdict !== 'live' || !incumbent.holdersEnumerable) {
-    return false
-  }
-  if (incumbent.holders.length !== 1) {
-    return false
-  }
-  const [holder] = incumbent.holders
-  return holder.matchesRelayArgv && holder.unrecognizedChildCount === 0
+  return (
+    isSoleIdentifiedRelayHolder(incumbent) && incumbent.holders[0]?.unrecognizedChildCount === 0
+  )
+}
+
+/**
+ * A superseded generation this client can no longer handshake with. Sole holder means no other
+ * Orca is attached, so SIGTERM destroys only unreachable work — including live PTYs.
+ * Same-path takeover must keep using {@link isReapableRelayHusk}; that path still preserves work.
+ */
+export function isReapableSupersededRelay(incumbent: RelayEndpointIncumbent): boolean {
+  return isSoleIdentifiedRelayHolder(incumbent)
 }
 
 export function describeRelayEndpointIncumbent(incumbent: RelayEndpointIncumbent): string {

src/main/ssh/ssh-relay-endpoint-takeover.test.ts

@@ -15,6 +15,7 @@ import { RelayCredentialMismatchError } from './ssh-relay-credential-mismatch-er
 import {
   interpretRelayHuskReapOutput,
   reapEmptyRelayHuskCommand,
+  reapSupersededRelayCommand,
   resolveRelayEndpointBeforeRelaunch
 } from './ssh-relay-endpoint-takeover'
 import { RelayVersionMismatchError } from './ssh-relay-version-mismatch-error'
@@ -205,6 +206,18 @@ describe('reapEmptyRelayHuskCommand', () => {
   })
 })
 
+describe('reapSupersededRelayCommand', () => {
+  it('still re-checks argv, but does not refuse a daemon that still holds PTYs', () => {
+    const command = reapSupersededRelayCommand(4242, SOCK)
+    expect(command.indexOf('MISMATCH')).toBeLessThan(command.indexOf('kill -TERM'))
+    expect(command.indexOf('holder_count=')).toBeLessThan(command.indexOf('kill -TERM'))
+    expect(command.indexOf('BUSY')).toBeLessThan(command.indexOf('kill -TERM'))
+    expect(command).toContain('kill -TERM')
+    expect(command).not.toContain('unrecognized_kids')
+    expect(command).toContain('while [ $i -lt 50 ]')
+  })
+})
+
 describe('interpretRelayHuskReapOutput', () => {
   it('claims reaped only for a post-signal liveness check that failed', () => {
     expect(interpretRelayHuskReapOutput('GONE\n')).toBe('reaped')

src/main/ssh/ssh-relay-endpoint-takeover.ts

@@ -1,11 +1,10 @@
 /**
  * Deciding whether a relay socket path is ours to take, and acting on the answer.
  *
- * The only destructive action available here is a SIGTERM to a relay that has been proven —
- * by argv, by socket-holder enumeration, and by a child census re-run on the host immediately
- * before the signal — to hold nothing at all. Everything else is left running.
- * Per docs/reference/ssh-execution-boundary.md, a relay we merely failed to reach is
- * `unverifiable`, and `unverifiable` never authorizes a kill or a rebind.
+ * Same-path takeover may SIGTERM only a relay proven empty (argv, sole holder, no unaccounted
+ * children). The superseded-generation sweep uses a separate command that still re-checks argv
+ * but may signal a daemon that still holds PTYs, because that generation is unreachable after
+ * an app update. A relay we merely failed to reach is `unverifiable` and never authorized.
  */
 import type { SshConnection } from './ssh-connection'
 import { shellEscape } from './ssh-connection-utils'
@@ -32,28 +31,40 @@ import type { RemoteHostPlatform } from './ssh-remote-platform'
 export type RelayHuskReapResult = 'reaped' | 'reap-unconfirmed' | 'retained-live-work'
 
 const REAP_CONFIRM_ATTEMPTS = 15
+// Relay dispose waits up to IMMEDIATE_PTY_EXIT_TIMEOUT_MS (8s) for PTY SIGKILL to land.
+const SUPERSEDED_REAP_CONFIRM_ATTEMPTS = 50
 
-/**
- * Signal one relay, re-verifying identity and emptiness inside the same command.
- *
- * The re-verification is not belt-and-braces: a client can attach and spawn a PTY between the
- * probe and the signal, and pids are reused. `MISMATCH`/`BUSY` abort without signalling.
- */
-export function reapEmptyRelayHuskCommand(pid: number, sockPath: string): string {
+function reapRelayDaemonCommand(
+  pid: number,
+  sockPath: string,
+  options?: { requireEmptyUnrecognizedChildren?: boolean; confirmAttempts?: number }
+): string {
+  const requireEmpty = options?.requireEmptyUnrecognizedChildren !== false
+  const confirmAttempts = options?.confirmAttempts ?? REAP_CONFIRM_ATTEMPTS
   return [
     `pid=${shellEscape(String(pid))}`,
     `sock=${shellEscape(sockPath)}`,
     'args=$(ps -o args= -p "$pid" 2>/dev/null | tr "\\n" " ")',
     'case "$args" in *relay.js*"$sock"*) ;; *) printf \'MISMATCH\\n\'; exit 0 ;; esac',
-    // Why the same census as the probe: `unknown` (no pgrep) and any child this host could
-    // not account for as a relay service both land on BUSY, so nothing is signalled.
-    ...relayDaemonChildCensusShell(),
-    `[ "$${RELAY_UNRECOGNIZED_CHILD_COUNT_VAR}" = "0" ] || { printf 'BUSY\\n'; exit 0; }`,
+    ...(requireEmpty
+      ? [
+          // Why the same census as the probe: `unknown` (no pgrep) and any child this host could
+          // not account for as a relay service both land on BUSY, so nothing is signalled.
+          ...relayDaemonChildCensusShell(),
+          `[ "$${RELAY_UNRECOGNIZED_CHILD_COUNT_VAR}" = "0" ] || { printf 'BUSY\\n'; exit 0; }`
+        ]
+      : [
+          // Sole-holder was a probe-time fact. Another client can attach before this signal.
+          // Why -a: lsof ORs selectors without it and reports unrelated unix-socket holders.
+          "command -v lsof >/dev/null 2>&1 || { printf 'BUSY\\n'; exit 0; }",
+          'holder_count=$(lsof -t -a -U -- "$sock" 2>/dev/null | sort -u | wc -l | tr -d \' \')',
+          '[ "$holder_count" = "1" ] || { printf \'BUSY\\n\'; exit 0; }'
+        ]),
     // SIGTERM only: the relay's own handler disposes and unlinks. SIGKILL would leave the
     // socket inode behind and skip that shutdown path for no gain on an empty daemon.
     'kill -TERM "$pid" 2>/dev/null || true',
     'i=0',
-    `while [ $i -lt ${REAP_CONFIRM_ATTEMPTS} ]; do`,
+    `while [ $i -lt ${confirmAttempts} ]; do`,
     '  kill -0 "$pid" 2>/dev/null || { printf \'GONE\\n\'; exit 0; }',
     '  sleep 0.2',
     '  i=$((i+1))',
@@ -62,6 +73,24 @@ export function reapEmptyRelayHuskCommand(pid: number, sockPath: string): string
   ].join('\n')
 }
 
+/**
+ * Signal one empty relay, re-verifying identity and emptiness inside the same command.
+ *
+ * The re-verification is not belt-and-braces: a client can attach and spawn a PTY between the
+ * probe and the signal, and pids are reused. `MISMATCH`/`BUSY` abort without signalling.
+ */
+export function reapEmptyRelayHuskCommand(pid: number, sockPath: string): string {
+  return reapRelayDaemonCommand(pid, sockPath)
+}
+
+/** SIGTERM a superseded generation even when it still holds PTYs. Argv is still re-checked. */
+export function reapSupersededRelayCommand(pid: number, sockPath: string): string {
+  return reapRelayDaemonCommand(pid, sockPath, {
+    requireEmptyUnrecognizedChildren: false,
+    confirmAttempts: SUPERSEDED_REAP_CONFIRM_ATTEMPTS
+  })
+}
+
 export function interpretRelayHuskReapOutput(output: string): RelayHuskReapResult {
   const state = output.trim().split('\n').pop()?.trim()
   if (state === 'GONE') {
@@ -75,21 +104,21 @@ export function interpretRelayHuskReapOutput(output: string): RelayHuskReapResul
   return 'reap-unconfirmed'
 }
 
-export async function reapEmptyRelayHusk(
+async function signalRelayDaemon(
   conn: SshConnection,
   incumbent: RelayEndpointIncumbent,
+  command: string,
   options?: { signal?: AbortSignal }
 ): Promise<RelayHuskReapResult> {
   const holder = incumbent.holders[0]
   if (!holder) {
     return 'retained-live-work'
   }
   try {
-    const output = await execCommand(
-      conn,
-      reapEmptyRelayHuskCommand(holder.pid, incumbent.sockPath),
-      { wrapCommand: true, signal: options?.signal }
-    )
+    const output = await execCommand(conn, command, {
+      wrapCommand: true,
+      signal: options?.signal
+    })
     return interpretRelayHuskReapOutput(output)
   } catch (err) {
     if (isUnconfirmedSshCommandTermination(err)) {
@@ -99,6 +128,40 @@ export async function reapEmptyRelayHusk(
   }
 }
 
+export async function reapEmptyRelayHusk(
+  conn: SshConnection,
+  incumbent: RelayEndpointIncumbent,
+  options?: { signal?: AbortSignal }
+): Promise<RelayHuskReapResult> {
+  const holder = incumbent.holders[0]
+  if (!holder) {
+    return 'retained-live-work'
+  }
+  return signalRelayDaemon(
+    conn,
+    incumbent,
+    reapEmptyRelayHuskCommand(holder.pid, incumbent.sockPath),
+    options
+  )
+}
+
+export async function reapSupersededRelay(
+  conn: SshConnection,
+  incumbent: RelayEndpointIncumbent,
+  options?: { signal?: AbortSignal }
+): Promise<RelayHuskReapResult> {
+  const holder = incumbent.holders[0]
+  if (!holder) {
+    return 'retained-live-work'
+  }
+  return signalRelayDaemon(
+    conn,
+    incumbent,
+    reapSupersededRelayCommand(holder.pid, incumbent.sockPath),
+    options
+  )
+}
+
 /**
  * Called when `--connect` to an existing socket failed and the caller is about to launch a
  * replacement at the same path. Resolves when the launch may proceed; throws

src/main/ssh/ssh-relay-generation-reset.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+  isSshRelayGenerationChange,
+  retireSshRelayGenerationLeases,
+  shouldBroadcastSshRelayGenerationRetired
+} from './ssh-relay-generation-reset'
+
+describe('isSshRelayGenerationChange', () => {
+  it('is false until both sides have a build id', () => {
+    expect(isSshRelayGenerationChange(undefined, '0.1.0+new')).toBe(false)
+    expect(isSshRelayGenerationChange('0.1.0+old', undefined)).toBe(false)
+    expect(isSshRelayGenerationChange(undefined, undefined)).toBe(false)
+  })
+
+  it('is false when the relay bundle did not move', () => {
+    expect(isSshRelayGenerationChange('0.1.0+same', '0.1.0+same')).toBe(false)
+  })
+
+  it('is true only when this client last spoke a different bundle', () => {
+    expect(isSshRelayGenerationChange('0.1.0+old', '0.1.0+new')).toBe(true)
+  })
+})
+
+describe('retireSshRelayGenerationLeases', () => {
+  it('expires every lease on the target so the new generation cannot reattach them', () => {
+    const store = { markSshRemotePtyLeases: vi.fn() }
+    retireSshRelayGenerationLeases(store, 'ssh-1')
+    expect(store.markSshRemotePtyLeases).toHaveBeenCalledWith('ssh-1', 'expired')
+  })
+})
+
+describe('shouldBroadcastSshRelayGenerationRetired', () => {
+  it('does not broadcast for runtime-owned targets', () => {
+    expect(shouldBroadcastSshRelayGenerationRetired('runtime-ssh-vm-1')).toBe(false)
+  })
+
+  it('broadcasts for a user SSH target', () => {
+    expect(shouldBroadcastSshRelayGenerationRetired('ssh-1')).toBe(true)
+  })
+})

src/main/ssh/ssh-relay-generation-reset.ts

@@ -0,0 +1,21 @@
+import { isRuntimeOwnedSshTargetId } from '../../shared/execution-host'
+
+/** True when this connect landed on a different relay bundle than the last successful one. */
+export function isSshRelayGenerationChange(
+  previousBuildId: string | undefined,
+  nextBuildId: string | undefined
+): boolean {
+  return Boolean(previousBuildId && nextBuildId && previousBuildId !== nextBuildId)
+}
+
+/** Drop leases so the new generation cannot reattach or cold-restore the unreachable PTYs. */
+export function retireSshRelayGenerationLeases(
+  store: { markSshRemotePtyLeases: (targetId: string, state: 'expired') => void },
+  targetId: string
+): void {
+  store.markSshRemotePtyLeases(targetId, 'expired')
+}
+
+export function shouldBroadcastSshRelayGenerationRetired(targetId: string): boolean {
+  return !isRuntimeOwnedSshTargetId(targetId)
+}