From 16033c1d1d38f2d4e5d1023d7b5c95657eb4111c Mon Sep 17 00:00:00 2001 From: Neil Date: Thu, 15 Oct 2026 15:06:12 -0700 Subject: [PATCH 1/2] fix(ssh): skip the managed-server probe while the control socket is answering A remote call after more than 5 s idle first ran a status.get probe on a new one-shot socket (SSH channel, WebSocket upgrade, hello, auth, request). A ready shared control socket that heard from the server in the last 20 s already proves it serves; a kill, reboot, rebind or stop closes that socket, so the probe still runs then. --- ...me-environment-request-connections.test.ts | 20 +++++++ ...runtime-environment-request-connections.ts | 14 +++++ src/main/ssh/orcad-managed-serving.test.ts | 55 +++++++++++++++++++ src/main/ssh/orcad-managed-serving.ts | 11 ++++ src/main/ssh/orcad-managed-tunnel.ts | 8 ++- ...emote-runtime-shared-control-connection.ts | 1 + .../remote-runtime-shared-control-open.ts | 10 +++- .../remote-runtime-shared-control-types.ts | 2 + 8 files changed, 118 insertions(+), 3 deletions(-) diff --git a/src/main/ipc/runtime-environment-request-connections.test.ts b/src/main/ipc/runtime-environment-request-connections.test.ts index b02d1b06fb55..8da6a56051f9 100644 --- a/src/main/ipc/runtime-environment-request-connections.test.ts +++ b/src/main/ipc/runtime-environment-request-connections.test.ts @@ -15,6 +15,7 @@ import { import { closeRemoteRuntimeRequestConnection, ensureRemoteRuntimeSharedControlConnection, + getRemoteRuntimeSharedControlAnsweredAt, getRemoteRuntimeSharedControlDiagnostics, pauseRemoteRuntimeSharedControlRetry, reconnectRemoteRuntimeSharedControlConnection, @@ -65,6 +66,25 @@ describe('runtime environment shared-control connection cache', () => { await waitFor(() => server.connectionCount() === 2) }) + it('reports when the ready control socket last heard from its server, never once it drops', async () => { + const server = await createSharedControlTestServer() + expect(getRemoteRuntimeSharedControlAnsweredAt(ENVIRONMENT_ID)).toBeNull() + + const before = Date.now() + await sendRemoteRuntimeSharedControlRequest( + ENVIRONMENT_ID, + server.pairing, + 'status.get', + undefined, + 1000 + ) + expect(getRemoteRuntimeSharedControlAnsweredAt(ENVIRONMENT_ID)).toBeGreaterThanOrEqual(before) + + server.closeClients() + await waitFor(() => getRemoteRuntimeSharedControlDiagnostics(ENVIRONMENT_ID)?.state !== 'ready') + expect(getRemoteRuntimeSharedControlAnsweredAt(ENVIRONMENT_ID)).toBeNull() + }) + it('blocks standing creation while manual intent is disconnected', async () => { const server = await createSharedControlTestServer() markRuntimeEnvironmentManuallyDisconnected(ENVIRONMENT_ID) diff --git a/src/main/ipc/runtime-environment-request-connections.ts b/src/main/ipc/runtime-environment-request-connections.ts index 86b665f2dde7..acb0b46b0b79 100644 --- a/src/main/ipc/runtime-environment-request-connections.ts +++ b/src/main/ipc/runtime-environment-request-connections.ts @@ -31,6 +31,7 @@ type CachedRuntimeConnection = { type CachedSharedControlConnection = { pairingKey: string connection: RemoteRuntimeSharedControlConnection + answeredAt: () => number | null } const requestConnections = new Map() @@ -173,6 +174,13 @@ export function getRemoteRuntimeSharedControlDiagnostics( return sharedControlConnections.get(environmentId)?.connection.getDiagnostics() ?? null } +/** When the ready control socket last heard from its server; null while it is not ready. */ +export function getRemoteRuntimeSharedControlAnsweredAt(environmentId: string): number | null { + const cached = sharedControlConnections.get(environmentId) + // Why ready is enough: a socket reaches ready only on its own server frame, which stamps it. + return cached?.connection.getDiagnostics().state === 'ready' ? cached.answeredAt() : null +} + export function reconnectRemoteRuntimeSharedControlConnection(environmentId: string): void { sharedControlConnections.get(environmentId)?.connection.reconnectNow() } @@ -210,13 +218,19 @@ function getSharedControlConnection( advanceRuntimeEnvironmentTransportGeneration(environmentId) cached?.connection.close() const transportGeneration = getRuntimeEnvironmentTransportGeneration(environmentId) + let answeredAt: number | null = null cached = { pairingKey, + answeredAt: () => answeredAt, connection: new RemoteRuntimeSharedControlConnection(pairedConnection, { environmentId, clientCapabilities: ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, isManuallyDisconnected: () => isRuntimeEnvironmentManuallyDisconnected(environmentId), isCapabilityPaused: () => isRuntimeEnvironmentCapabilityPaused(environmentId), + // Why wall clock: it advances across a suspend, so a pre-sleep answer reads as stale. + onAnswered: () => { + answeredAt = Date.now() + }, onDiagnosticsChanged: (diagnostics) => { if (getRuntimeEnvironmentTransportGeneration(environmentId) !== transportGeneration) { return diff --git a/src/main/ssh/orcad-managed-serving.test.ts b/src/main/ssh/orcad-managed-serving.test.ts index a606e2ee7459..708d8026a83d 100644 --- a/src/main/ssh/orcad-managed-serving.test.ts +++ b/src/main/ssh/orcad-managed-serving.test.ts @@ -13,6 +13,7 @@ vi.mock('./orcad-managed-wake', () => ({ wakeStoppedManagedOrcad: vi.fn() })) import { wakeStoppedManagedOrcad } from './orcad-managed-wake' import { ensureManagedOrcadServing, + MANAGED_ORCAD_NOT_ACTIVATED_DETAIL, resetManagedOrcadServingForTests, setManagedOrcadStartListener, type OrcadManagedServingInput @@ -174,4 +175,58 @@ describe('ensureManagedOrcadServing', () => { await ensureManagedOrcadServing({ ...input(probe), remotePort: 40_001 }, () => 2) expect(probe).toHaveBeenCalledTimes(3) }) + + it('skips the probe after an idle pause while the control socket keeps answering', async () => { + let now = 0 + const probe = vi.fn(async () => true) + const answeredAt = vi.fn((): number | null => null) + const live = { ...input(probe), answeredAt } + await ensureManagedOrcadServing(live, () => now) + expect(probe).toHaveBeenCalledOnce() + + now = 6_000 + answeredAt.mockReturnValue(4_000) + expect(await ensureManagedOrcadServing(live, () => now)).toEqual({ state: 'serving' }) + expect(answeredAt).toHaveBeenCalledWith('env-1') + expect(probe).toHaveBeenCalledOnce() + }) + + it('shares one probe between concurrent calls once the last answer is stale', async () => { + let answer!: (value: boolean) => void + const probe = vi.fn(() => new Promise((resolve) => (answer = resolve))) + // Last heard before a suspend: too old to prove anything. + const stale = { ...input(probe), answeredAt: () => 0 } + const calls = [1, 2, 3].map(() => ensureManagedOrcadServing(stale, () => 60_000)) + answer(true) + expect(await Promise.all(calls)).toEqual([ + { state: 'serving' }, + { state: 'serving' }, + { state: 'serving' } + ]) + expect(probe).toHaveBeenCalledOnce() + }) + + it('still detects a stopped server once its control socket closed', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome: 'not-activated' }) + const answeredAt = vi.fn((): number | null => 0) + const probe = vi.fn(async () => true) + const watched = { ...input(probe), answeredAt } + expect(await ensureManagedOrcadServing(watched, () => 0)).toEqual({ state: 'serving' }) + expect(probe).not.toHaveBeenCalled() + + // Another desktop stopped the server: its socket closed, so there is no live answer. + answeredAt.mockReturnValue(null) + probe.mockResolvedValue(false) + expect(await ensureManagedOrcadServing(watched, () => 1_000)).toEqual({ + state: 'unverifiable', + detail: MANAGED_ORCAD_NOT_ACTIVATED_DETAIL + }) + expect(probe).toHaveBeenCalledOnce() + }) + + it('does not take an answer from a wall clock that moved backwards as proof', async () => { + const probe = vi.fn(async () => true) + await ensureManagedOrcadServing({ ...input(probe), answeredAt: () => 50_000 }, () => 10_000) + expect(probe).toHaveBeenCalledOnce() + }) }) diff --git a/src/main/ssh/orcad-managed-serving.ts b/src/main/ssh/orcad-managed-serving.ts index 19580399d176..1ccb7bac3737 100644 --- a/src/main/ssh/orcad-managed-serving.ts +++ b/src/main/ssh/orcad-managed-serving.ts @@ -27,6 +27,8 @@ export type OrcadManagedServingInput = { connection: OrcadHostTransport remotePort: number probe: (environment: KnownRuntimeEnvironment, timeoutMs: number) => Promise + /** When this environment's ready control socket last heard from its server, if it has. */ + answeredAt?: (environmentId: string) => number | null } /** A wake refused because another update, rollback or recovery holds the host; it clears itself. */ @@ -41,6 +43,9 @@ const PROBE_TIMEOUT_MS = 5_000 // Why: a connect checks right after its fresh tunnel did; one verdict serves both, on that // transport and port only, so a kill, reboot or rebind is never answered from cache. const VERDICT_REUSE_MS = 5_000 +// Why: the control socket hears a pong every 10 s while healthy, and a kill, reboot, rebind or +// stop closes it, so a recent answer on it proves serving without a fresh probe socket. +const LIVE_ANSWER_PROOF_MS = 20_000 type ServingTransport = { connection: OrcadHostTransport; generation: number; remotePort: number } // Why per transport: a check on a dropped connection fails, and a caller on the reconnected one @@ -85,6 +90,12 @@ export function ensureManagedOrcadServing( ) { return entry.check } + const answeredAt = input.answeredAt?.(id) ?? null + const answerAge = answeredAt === null ? null : now() - answeredAt + // Why the lower bound: a wall clock set backwards must not stretch the proof. + if (answerAge !== null && answerAge >= 0 && answerAge < LIVE_ANSWER_PROOF_MS) { + return Promise.resolve({ state: 'serving' }) + } const next: ServingCheck = { ...transport, check: checkAndStart(input) } checks.set(id, next) // A rejected check is never reused. diff --git a/src/main/ssh/orcad-managed-tunnel.ts b/src/main/ssh/orcad-managed-tunnel.ts index c206eb87aff5..5b4857351481 100644 --- a/src/main/ssh/orcad-managed-tunnel.ts +++ b/src/main/ssh/orcad-managed-tunnel.ts @@ -5,6 +5,7 @@ import type { SshConnection } from './ssh-connection' import { probeManagedOrcadTunnel } from './orcad-managed-tunnel-resume' import { ensureManagedOrcadServing } from './orcad-managed-serving' import { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager' +import { getRemoteRuntimeSharedControlAnsweredAt } from '../ipc/runtime-environment-request-connections' import { getSshConnectionManager, getSshTargetRegistryStore } from './ssh-target-registry' import { MANAGED_ORCAD_TUNNEL_TARGETING, @@ -17,7 +18,12 @@ const managedTunnels = new OrcadManagedTunnelManager({ getConnectionManager: getSshConnectionManager, getTargetStore: getSshTargetRegistryStore, targeting: MANAGED_ORCAD_TUNNEL_TARGETING, - ensureServing: (input) => ensureManagedOrcadServing({ ...input, probe: probeManagedOrcadTunnel }) + ensureServing: (input) => + ensureManagedOrcadServing({ + ...input, + probe: probeManagedOrcadTunnel, + answeredAt: getRemoteRuntimeSharedControlAnsweredAt + }) }) export async function ensureOrcadManagedTunnel( diff --git a/src/shared/remote-runtime-shared-control-connection.ts b/src/shared/remote-runtime-shared-control-connection.ts index c8189b430c4b..5baa3af9256a 100644 --- a/src/shared/remote-runtime-shared-control-connection.ts +++ b/src/shared/remote-runtime-shared-control-connection.ts @@ -186,6 +186,7 @@ export class RemoteRuntimeSharedControlConnection { }, onError: (error) => this.handleSocketClosed(error, socketGeneration), onTextFrame: (frame) => this.handleTextFrame(frame, socketGeneration), + onActivity: this.options.onAnswered, liveness: { options: this.options.liveness, onDead: (error) => this.handleSocketClosed(error, socketGeneration) diff --git a/src/shared/remote-runtime-shared-control-open.ts b/src/shared/remote-runtime-shared-control-open.ts index 02ed1337379c..56961236a2b4 100644 --- a/src/shared/remote-runtime-shared-control-open.ts +++ b/src/shared/remote-runtime-shared-control-open.ts @@ -19,6 +19,8 @@ export function openSharedControlSocket( onClose: (close: { code: number; reason: string }, error: RemoteRuntimeClientError) => void onError: (error: RemoteRuntimeClientError) => void onTextFrame: (frame: string) => void + /** Any inbound frame, ping or pong on the current socket. */ + onActivity?: () => void // Why: reconnect is edge-triggered on `close`, but a half-open tunnel // never delivers one. Liveness pings and declares the socket dead when // the server goes silent, so the close/reconnect path can run (#7718). @@ -28,7 +30,11 @@ export function openSharedControlSocket( } } ): { ok: true; socket: RemoteRuntimeWebSocket } | { ok: false; error: RemoteRuntimeClientError } { - let noteActivity: () => void = () => {} + let noteLiveness: () => void = () => {} + const noteActivity = (): void => { + noteLiveness() + callbacks.onActivity?.() + } const opened = openRemoteRuntimeWebSocket(connection, { onClose: (ws, code, reason) => { if (callbacks.getCurrentSocket() === ws) { @@ -90,7 +96,7 @@ export function openSharedControlSocket( }, options: liveness.options }) - noteActivity = monitor.noteActivity + noteLiveness = monitor.noteActivity return { ok: true, diff --git a/src/shared/remote-runtime-shared-control-types.ts b/src/shared/remote-runtime-shared-control-types.ts index 39d1b01f6133..7ea3b7fca75e 100644 --- a/src/shared/remote-runtime-shared-control-types.ts +++ b/src/shared/remote-runtime-shared-control-types.ts @@ -79,6 +79,8 @@ export type RemoteRuntimeSharedControlConnectionOptions = { isCapabilityPaused?: () => boolean /** Publishes local transport diagnostics after a meaningful state transition. */ onDiagnosticsChanged?: (diagnostics: RemoteRuntimeSharedConnectionDiagnostics) => void + /** Any inbound frame, ping or pong on the current socket. */ + onAnswered?: () => void reconnectStableResetMs?: number liveness?: RemoteRuntimeSocketLivenessOptions } From c1e74798a229938af627c373c67086f8e1133431 Mon Sep 17 00:00:00 2001 From: Neil Date: Thu, 15 Oct 2026 17:54:51 -0700 Subject: [PATCH 2/2] fix(ssh): read control-socket answers through a leaf module orcad-managed-tunnel imported the request-connection cache, which (through the status owner) imports the serving gate back. Tests that mock that cache with an async factory deadlocked importing it. A leaf module now carries the answer source between them. --- ...me-environment-request-connections.test.ts | 2 +- ...runtime-environment-request-connections.ts | 17 ++++++++------- ...time-environment-shared-control-answers.ts | 21 +++++++++++++++++++ src/main/ssh/orcad-managed-tunnel.ts | 2 +- 4 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 src/main/ipc/runtime-environment-shared-control-answers.ts diff --git a/src/main/ipc/runtime-environment-request-connections.test.ts b/src/main/ipc/runtime-environment-request-connections.test.ts index 8da6a56051f9..14ed126335a8 100644 --- a/src/main/ipc/runtime-environment-request-connections.test.ts +++ b/src/main/ipc/runtime-environment-request-connections.test.ts @@ -15,13 +15,13 @@ import { import { closeRemoteRuntimeRequestConnection, ensureRemoteRuntimeSharedControlConnection, - getRemoteRuntimeSharedControlAnsweredAt, getRemoteRuntimeSharedControlDiagnostics, pauseRemoteRuntimeSharedControlRetry, reconnectRemoteRuntimeSharedControlConnection, sendRemoteRuntimeSharedControlRequest, subscribeRemoteRuntimeSharedControlRequest } from './runtime-environment-request-connections' +import { getRemoteRuntimeSharedControlAnsweredAt } from './runtime-environment-shared-control-answers' const ENVIRONMENT_ID = 'standing-intent-test' diff --git a/src/main/ipc/runtime-environment-request-connections.ts b/src/main/ipc/runtime-environment-request-connections.ts index acb0b46b0b79..c2fdfbe2e139 100644 --- a/src/main/ipc/runtime-environment-request-connections.ts +++ b/src/main/ipc/runtime-environment-request-connections.ts @@ -18,6 +18,10 @@ import type { import { isRuntimeEnvironmentCapabilityPaused } from './runtime-environment-capability-evidence' import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' import { publishRuntimeEnvironmentDiagnostics } from './runtime-environment-diagnostics-broadcast' +import { + clearSharedControlAnswerSource, + setSharedControlAnswerSource +} from './runtime-environment-shared-control-answers' import { advanceRuntimeEnvironmentTransportGeneration, getRuntimeEnvironmentTransportGeneration @@ -165,6 +169,7 @@ export function subscribeRemoteRuntimeSharedControlRequest( export function closeRemoteRuntimeSharedControlConnection(environmentId: string): void { const cached = sharedControlConnections.get(environmentId) sharedControlConnections.delete(environmentId) + clearSharedControlAnswerSource(environmentId) cached?.connection.close() } @@ -174,13 +179,6 @@ export function getRemoteRuntimeSharedControlDiagnostics( return sharedControlConnections.get(environmentId)?.connection.getDiagnostics() ?? null } -/** When the ready control socket last heard from its server; null while it is not ready. */ -export function getRemoteRuntimeSharedControlAnsweredAt(environmentId: string): number | null { - const cached = sharedControlConnections.get(environmentId) - // Why ready is enough: a socket reaches ready only on its own server frame, which stamps it. - return cached?.connection.getDiagnostics().state === 'ready' ? cached.answeredAt() : null -} - export function reconnectRemoteRuntimeSharedControlConnection(environmentId: string): void { sharedControlConnections.get(environmentId)?.connection.reconnectNow() } @@ -254,6 +252,11 @@ function getSharedControlConnection( }) } sharedControlConnections.set(environmentId, cached) + const entry = cached + // Why ready is enough: a socket reaches ready only on its own server frame, which stamps it. + setSharedControlAnswerSource(environmentId, () => + entry.connection.getDiagnostics().state === 'ready' ? entry.answeredAt() : null + ) } return cached.connection } diff --git a/src/main/ipc/runtime-environment-shared-control-answers.ts b/src/main/ipc/runtime-environment-shared-control-answers.ts new file mode 100644 index 000000000000..5b310e9294da --- /dev/null +++ b/src/main/ipc/runtime-environment-shared-control-answers.ts @@ -0,0 +1,21 @@ +/** + * When each environment's ready control socket last heard from its server. A leaf module: the + * serving gate reads it without importing the connection cache, which imports the gate back. + */ +const sources = new Map number | null>() + +export function setSharedControlAnswerSource( + environmentId: string, + source: () => number | null +): void { + sources.set(environmentId, source) +} + +export function clearSharedControlAnswerSource(environmentId: string): void { + sources.delete(environmentId) +} + +/** Null while there is no ready control socket for the environment. */ +export function getRemoteRuntimeSharedControlAnsweredAt(environmentId: string): number | null { + return sources.get(environmentId)?.() ?? null +} diff --git a/src/main/ssh/orcad-managed-tunnel.ts b/src/main/ssh/orcad-managed-tunnel.ts index 5b4857351481..1ee13039d797 100644 --- a/src/main/ssh/orcad-managed-tunnel.ts +++ b/src/main/ssh/orcad-managed-tunnel.ts @@ -5,7 +5,7 @@ import type { SshConnection } from './ssh-connection' import { probeManagedOrcadTunnel } from './orcad-managed-tunnel-resume' import { ensureManagedOrcadServing } from './orcad-managed-serving' import { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager' -import { getRemoteRuntimeSharedControlAnsweredAt } from '../ipc/runtime-environment-request-connections' +import { getRemoteRuntimeSharedControlAnsweredAt } from '../ipc/runtime-environment-shared-control-answers' import { getSshConnectionManager, getSshTargetRegistryStore } from './ssh-target-registry' import { MANAGED_ORCAD_TUNNEL_TARGETING,