Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/main/ipc/runtime-environment-request-connections.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
sendRemoteRuntimeSharedControlRequest,
subscribeRemoteRuntimeSharedControlRequest
} from './runtime-environment-request-connections'
import { getRemoteRuntimeSharedControlAnsweredAt } from './runtime-environment-shared-control-answers'

const ENVIRONMENT_ID = 'standing-intent-test'

Expand Down Expand Up @@ -65,6 +66,25 @@ describe('runtime environment shared-control connection cache', () => {
await waitFor(() => server.connectionCount() === 2)
})

it('reports when the ready control socket last heard from its server, never once it drops', async () => {
const server = await createSharedControlTestServer()
expect(getRemoteRuntimeSharedControlAnsweredAt(ENVIRONMENT_ID)).toBeNull()

const before = Date.now()
await sendRemoteRuntimeSharedControlRequest(
ENVIRONMENT_ID,
server.pairing,
'status.get',
undefined,
1000
)
expect(getRemoteRuntimeSharedControlAnsweredAt(ENVIRONMENT_ID)).toBeGreaterThanOrEqual(before)

server.closeClients()
await waitFor(() => getRemoteRuntimeSharedControlDiagnostics(ENVIRONMENT_ID)?.state !== 'ready')
expect(getRemoteRuntimeSharedControlAnsweredAt(ENVIRONMENT_ID)).toBeNull()
})

it('blocks standing creation while manual intent is disconnected', async () => {
const server = await createSharedControlTestServer()
markRuntimeEnvironmentManuallyDisconnected(ENVIRONMENT_ID)
Expand Down
17 changes: 17 additions & 0 deletions src/main/ipc/runtime-environment-request-connections.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ import type {
import { isRuntimeEnvironmentCapabilityPaused } from './runtime-environment-capability-evidence'
import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect'
import { publishRuntimeEnvironmentDiagnostics } from './runtime-environment-diagnostics-broadcast'
import {
clearSharedControlAnswerSource,
setSharedControlAnswerSource
} from './runtime-environment-shared-control-answers'
import {
advanceRuntimeEnvironmentTransportGeneration,
getRuntimeEnvironmentTransportGeneration
Expand All @@ -31,6 +35,7 @@ type CachedRuntimeConnection = {
type CachedSharedControlConnection = {
pairingKey: string
connection: RemoteRuntimeSharedControlConnection
answeredAt: () => number | null
}

const requestConnections = new Map<string, CachedRuntimeConnection>()
Expand Down Expand Up @@ -164,6 +169,7 @@ export function subscribeRemoteRuntimeSharedControlRequest<TResult>(
export function closeRemoteRuntimeSharedControlConnection(environmentId: string): void {
const cached = sharedControlConnections.get(environmentId)
sharedControlConnections.delete(environmentId)
clearSharedControlAnswerSource(environmentId)
cached?.connection.close()
}

Expand Down Expand Up @@ -210,13 +216,19 @@ function getSharedControlConnection(
advanceRuntimeEnvironmentTransportGeneration(environmentId)
cached?.connection.close()
const transportGeneration = getRuntimeEnvironmentTransportGeneration(environmentId)
let answeredAt: number | null = null
cached = {
pairingKey,
answeredAt: () => answeredAt,
connection: new RemoteRuntimeSharedControlConnection(pairedConnection, {
environmentId,
clientCapabilities: ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES,
isManuallyDisconnected: () => isRuntimeEnvironmentManuallyDisconnected(environmentId),
isCapabilityPaused: () => isRuntimeEnvironmentCapabilityPaused(environmentId),
// Why wall clock: it advances across a suspend, so a pre-sleep answer reads as stale.
onAnswered: () => {
answeredAt = Date.now()
},
onDiagnosticsChanged: (diagnostics) => {
if (getRuntimeEnvironmentTransportGeneration(environmentId) !== transportGeneration) {
return
Expand All @@ -240,6 +252,11 @@ function getSharedControlConnection(
})
}
sharedControlConnections.set(environmentId, cached)
const entry = cached
// Why ready is enough: a socket reaches ready only on its own server frame, which stamps it.
setSharedControlAnswerSource(environmentId, () =>
entry.connection.getDiagnostics().state === 'ready' ? entry.answeredAt() : null
)
Comment on lines +257 to +259

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟡 Minor | ⚡ Quick win

🔎 Supported by static analysis

🏁 Script executed:

sed -n '1,210p' src/main/ssh/orcad-managed-serving.ts
sed -n '1,130p' src/main/ssh/orcad-managed-tunnel.ts
rg -n 'ensureServing\\(|ensureManagedOrcadServing|machineNameReady\\(' src/main/ssh src/main/runtime/rpc/methods/status.ts

Repository: stablyai/orca

Length of output: 11131


🏁 Script executed:

set -o pipefail
printf '%s\n' '--- probe and caller references ---'
rg -n -F --glob '*.ts' \
  -e 'probeManagedOrcadTunnel' \
  -e 'ensureServing' \
  -e 'status.get' \
  -e 'machineNameReady' \
  -e 'ensureOrcadManagedTunnel' \
  -e 'startOrcadManagedTunnel' \
  src/main/ssh src/main/runtime/rpc src/main/ipc || test "$?" -eq 1
printf '%s\n' '--- probe implementation ---'
sed -n '1,240p' src/main/ssh/orcad-managed-tunnel-resume.ts
printf '%s\n' '--- tunnel manager serving/start flow ---'
sed -n '1,320p' src/main/ssh/orcad-managed-tunnel-manager.ts
printf '%s\n' '--- status RPC ---'
sed -n '1,180p' src/main/runtime/rpc/methods/status.ts
printf '%s\n' '--- websocket lifecycle ---'
sed -n '1,280p' src/main/runtime/rpc/node-websocket-lifecycle.ts

Repository: stablyai/orca

Length of output: 42989


🏁 Script executed:

set -o pipefail
printf '%s\n' '--- probe and caller references ---'
rg -n -F --glob '*.ts' -e 'probeManagedOrcadTunnel' -e 'ensureServing' -e 'status.get' -e 'machineNameReady' -e 'ensureOrcadManagedTunnel' -e 'startOrcadManagedTunnel' src/main/ssh src/main/runtime/rpc src/main/ipc || test "$?" -eq 1
printf '%s\n' '--- probe implementation ---'
sed -n '1,240p' src/main/ssh/orcad-managed-tunnel-resume.ts
printf '%s\n' '--- tunnel manager serving/start flow ---'
sed -n '1,320p' src/main/ssh/orcad-managed-tunnel-manager.ts
printf '%s\n' '--- status RPC ---'
sed -n '1,180p' src/main/runtime/rpc/methods/status.ts
printf '%s\n' '--- websocket lifecycle ---'
sed -n '1,280p' src/main/runtime/rpc/node-websocket-lifecycle.ts

Repository: stablyai/orca

Length of output: 42263


🏁 Script executed:

printf '%s\n' '--- probe implementation ---'
sed -n '248,295p' src/main/ssh/orcad-managed-tunnel-resume.ts
printf '%s\n' '--- serving check ---'
sed -n '1,130p' src/main/ssh/orcad-managed-tunnel-serving.ts
printf '%s\n' '--- immediate SSH action caller ---'
sed -n '110,195p' src/main/ssh/runtime-ssh-access.ts
printf '%s\n' '--- probe transport helper definitions ---'
rg -n -F --glob '*.ts' -e 'sendRemoteRuntimeRequest' -e 'callRuntimeEnvironment' -e 'probeManagedOrcadTunnel' src/main/ssh src/main/ipc

Repository: stablyai/orca

Length of output: 32870


🏁 Script executed:

printf '%s\n' '--- request connection routing ---'
sed -n '35,85p' src/main/ipc/runtime-environment-request-connections.ts
printf '%s\n' '--- managed tunnel request handler ---'
sed -n '1,75p' src/main/ipc/runtime-environment-managed-tunnel.ts
printf '%s\n' '--- managed serving call site ---'
sed -n '1,65p' src/main/ssh/orcad-managed-tunnel-serving.ts

Repository: stablyai/orca

Length of output: 7489


Keep status.get as the serving check.

answeredAt records transport activity, not an application response. A recent timestamp returns serving before probeManagedOrcadTunnel sends status.get. If that probe times out, it returns false and enters wakeIfStopped, which can start a stopped server or return unverifiable for a live server that does not answer. The timestamp shortcut bypasses both outcomes, so resolveManagedRuntimeEnvironment can continue with the remote call on an unresponsive runtime.

Use an authoritative serving response for this shortcut, or retain the status.get probe while answeredAt represents only socket activity.

}
return cached.connection
}
Expand Down
21 changes: 21 additions & 0 deletions src/main/ipc/runtime-environment-shared-control-answers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/**
* When each environment's ready control socket last heard from its server. A leaf module: the
* serving gate reads it without importing the connection cache, which imports the gate back.
*/
const sources = new Map<string, () => number | null>()

export function setSharedControlAnswerSource(
environmentId: string,
source: () => number | null
): void {
sources.set(environmentId, source)
}

export function clearSharedControlAnswerSource(environmentId: string): void {
sources.delete(environmentId)
}

/** Null while there is no ready control socket for the environment. */
export function getRemoteRuntimeSharedControlAnsweredAt(environmentId: string): number | null {
return sources.get(environmentId)?.() ?? null
}
55 changes: 55 additions & 0 deletions src/main/ssh/orcad-managed-serving.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ vi.mock('./orcad-managed-wake', () => ({ wakeStoppedManagedOrcad: vi.fn() }))
import { wakeStoppedManagedOrcad } from './orcad-managed-wake'
import {
ensureManagedOrcadServing,
MANAGED_ORCAD_NOT_ACTIVATED_DETAIL,
resetManagedOrcadServingForTests,
setManagedOrcadStartListener,
type OrcadManagedServingInput
Expand Down Expand Up @@ -174,4 +175,58 @@ describe('ensureManagedOrcadServing', () => {
await ensureManagedOrcadServing({ ...input(probe), remotePort: 40_001 }, () => 2)
expect(probe).toHaveBeenCalledTimes(3)
})

it('skips the probe after an idle pause while the control socket keeps answering', async () => {
let now = 0
const probe = vi.fn(async () => true)
const answeredAt = vi.fn((): number | null => null)
const live = { ...input(probe), answeredAt }
await ensureManagedOrcadServing(live, () => now)
expect(probe).toHaveBeenCalledOnce()

now = 6_000
answeredAt.mockReturnValue(4_000)
expect(await ensureManagedOrcadServing(live, () => now)).toEqual({ state: 'serving' })
expect(answeredAt).toHaveBeenCalledWith('env-1')
expect(probe).toHaveBeenCalledOnce()
})

it('shares one probe between concurrent calls once the last answer is stale', async () => {
let answer!: (value: boolean) => void
const probe = vi.fn(() => new Promise<boolean>((resolve) => (answer = resolve)))
// Last heard before a suspend: too old to prove anything.
const stale = { ...input(probe), answeredAt: () => 0 }
const calls = [1, 2, 3].map(() => ensureManagedOrcadServing(stale, () => 60_000))
answer(true)
expect(await Promise.all(calls)).toEqual([
{ state: 'serving' },
{ state: 'serving' },
{ state: 'serving' }
])
expect(probe).toHaveBeenCalledOnce()
})

it('still detects a stopped server once its control socket closed', async () => {
vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome: 'not-activated' })
const answeredAt = vi.fn((): number | null => 0)
const probe = vi.fn(async () => true)
const watched = { ...input(probe), answeredAt }
expect(await ensureManagedOrcadServing(watched, () => 0)).toEqual({ state: 'serving' })
expect(probe).not.toHaveBeenCalled()

// Another desktop stopped the server: its socket closed, so there is no live answer.
answeredAt.mockReturnValue(null)
probe.mockResolvedValue(false)
expect(await ensureManagedOrcadServing(watched, () => 1_000)).toEqual({
state: 'unverifiable',
detail: MANAGED_ORCAD_NOT_ACTIVATED_DETAIL
})
expect(probe).toHaveBeenCalledOnce()
})

it('does not take an answer from a wall clock that moved backwards as proof', async () => {
const probe = vi.fn(async () => true)
await ensureManagedOrcadServing({ ...input(probe), answeredAt: () => 50_000 }, () => 10_000)
expect(probe).toHaveBeenCalledOnce()
})
})
11 changes: 11 additions & 0 deletions src/main/ssh/orcad-managed-serving.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ export type OrcadManagedServingInput = {
connection: OrcadHostTransport
remotePort: number
probe: (environment: KnownRuntimeEnvironment, timeoutMs: number) => Promise<boolean>
/** When this environment's ready control socket last heard from its server, if it has. */
answeredAt?: (environmentId: string) => number | null
}

/** A wake refused because another update, rollback or recovery holds the host; it clears itself. */
Expand All @@ -41,6 +43,9 @@ const PROBE_TIMEOUT_MS = 5_000
// Why: a connect checks right after its fresh tunnel did; one verdict serves both, on that
// transport and port only, so a kill, reboot or rebind is never answered from cache.
const VERDICT_REUSE_MS = 5_000
// Why: the control socket hears a pong every 10 s while healthy, and a kill, reboot, rebind or
// stop closes it, so a recent answer on it proves serving without a fresh probe socket.
const LIVE_ANSWER_PROOF_MS = 20_000

type ServingTransport = { connection: OrcadHostTransport; generation: number; remotePort: number }
// Why per transport: a check on a dropped connection fails, and a caller on the reconnected one
Expand Down Expand Up @@ -85,6 +90,12 @@ export function ensureManagedOrcadServing(
) {
return entry.check
}
const answeredAt = input.answeredAt?.(id) ?? null
const answerAge = answeredAt === null ? null : now() - answeredAt
// Why the lower bound: a wall clock set backwards must not stretch the proof.
if (answerAge !== null && answerAge >= 0 && answerAge < LIVE_ANSWER_PROOF_MS) {
return Promise.resolve({ state: 'serving' })
Comment on lines +96 to +97

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | 🏗️ Heavy lift

🔎 Supported by static analysis

🏁 Script executed:

set -o pipefail
printf '%s\n' '--- diff ---'
git diff --find-renames 33759f91f5f5a113d9b2eff5bca6e3a30ec3636e 16033c1d1d38f2d4e5d1023d7b5c95657eb4111c -- src/main/ssh/orcad-managed-serving.ts src/main/ssh src/shared/remote-runtime-shared-control-connection.ts src/shared/remote-runtime-shared-control-test-server.ts src/main/ssh/orcad-managed-serving.test.ts
printf '%s\n' '--- relevant references ---'
rg -n -F --glob '*.ts' -e 'answeredAt' -e 'onAnswered' -e 'autoPong' -e 'status.get' -e 'LIVE_ANSWER_PROOF_MS' src/main/ssh src/shared || test "$?" -eq 1
printf '%s\n' '--- changed implementation ---'
nl -ba src/main/ssh/orcad-managed-serving.ts | sed -n '1,180p'
printf '%s\n' '--- shared connection ---'
nl -ba src/shared/remote-runtime-shared-control-connection.ts | sed -n '1,260p'
printf '%s\n' '--- test server ---'
nl -ba src/shared/remote-runtime-shared-control-test-server.ts | sed -n '1,280p'

Repository: stablyai/orca

Length of output: 45010


🏁 Script executed:

set -o pipefail
printf '%s\n' '--- callback and answeredAt definitions ---'
rg -n -F --glob '*.ts' -e 'onActivity' -e 'getRemoteRuntimeSharedControlAnsweredAt' -e 'onAnswered' -e 'answeredAt' src/shared src/main/ipc
printf '%s\n' '--- socket opener and liveness ---'
nl -ba src/shared/remote-runtime-shared-control-open.ts | sed -n '1,280p'
nl -ba src/shared/remote-runtime-shared-control-types.ts | sed -n '50,110p'
rg -n -F --glob '*.ts' -e 'pong' -e 'ping' src/shared/remote-runtime-shared-control-*.ts src/shared/remote-runtime-socket*.ts || test "$?" -eq 1
printf '%s\n' '--- answeredAt owner and changed tests ---'
nl -ba src/main/ipc/runtime-environment-request-connections.ts | sed -n '1,280p'
git diff --find-renames 33759f91f5f5a113d9b2eff5bca6e3a30ec3636e 16033c1d1d38f2d4e5d1023d7b5c95657eb4111c -- src/main/ipc/runtime-environment-request-connections.ts src/main/ipc/runtime-environment-request-connections.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-open.ts src/shared/remote-runtime-shared-control-types.ts

Repository: stablyai/orca

Length of output: 30213


🏁 Script executed:

set -o pipefail
printf '%s\n' '--- repository-owned control server bindings ---'
rg -n -F --glob '*.ts' --glob '*.js' --glob '*.tsx' --glob '*.jsx' -e 'new WebSocketServer' -e 'WebSocketServer(' -e 'autoPong' -e 'status.get' -e 'silentMethods' -e 'createSharedControlTestServer' . || test "$?" -eq 1
printf '%s\n' '--- server/control-related files ---'
rg --files | rg 'remote-runtime|runtime-control|shared-control|websocket|ws' | sed -n '1,240p'

Repository: stablyai/orca

Length of output: 41187


Do not treat a pong alone as proof that status.get works.

openSharedControlSocket records pongs as activity, and answeredAt records that activity. A recent timestamp then skips the status.get probe. A server can keep the WebSocket open, answer pings automatically, and stop sending application-level RPC responses. Require application-level response evidence before updating answeredAt, or run the probe when only control-frame activity is available.

}
const next: ServingCheck = { ...transport, check: checkAndStart(input) }
checks.set(id, next)
// A rejected check is never reused.
Expand Down
8 changes: 7 additions & 1 deletion src/main/ssh/orcad-managed-tunnel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { SshConnection } from './ssh-connection'
import { probeManagedOrcadTunnel } from './orcad-managed-tunnel-resume'
import { ensureManagedOrcadServing } from './orcad-managed-serving'
import { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager'
import { getRemoteRuntimeSharedControlAnsweredAt } from '../ipc/runtime-environment-shared-control-answers'
import { getSshConnectionManager, getSshTargetRegistryStore } from './ssh-target-registry'
import {
MANAGED_ORCAD_TUNNEL_TARGETING,
Expand All @@ -17,7 +18,12 @@ const managedTunnels = new OrcadManagedTunnelManager({
getConnectionManager: getSshConnectionManager,
getTargetStore: getSshTargetRegistryStore,
targeting: MANAGED_ORCAD_TUNNEL_TARGETING,
ensureServing: (input) => ensureManagedOrcadServing({ ...input, probe: probeManagedOrcadTunnel })
ensureServing: (input) =>
ensureManagedOrcadServing({
...input,
probe: probeManagedOrcadTunnel,
answeredAt: getRemoteRuntimeSharedControlAnsweredAt
})
})

export async function ensureOrcadManagedTunnel(
Expand Down
1 change: 1 addition & 0 deletions src/shared/remote-runtime-shared-control-connection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ export class RemoteRuntimeSharedControlConnection {
},
onError: (error) => this.handleSocketClosed(error, socketGeneration),
onTextFrame: (frame) => this.handleTextFrame(frame, socketGeneration),
onActivity: this.options.onAnswered,
liveness: {
options: this.options.liveness,
onDead: (error) => this.handleSocketClosed(error, socketGeneration)
Expand Down
10 changes: 8 additions & 2 deletions src/shared/remote-runtime-shared-control-open.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ export function openSharedControlSocket(
onClose: (close: { code: number; reason: string }, error: RemoteRuntimeClientError) => void
onError: (error: RemoteRuntimeClientError) => void
onTextFrame: (frame: string) => void
/** Any inbound frame, ping or pong on the current socket. */
onActivity?: () => void
// Why: reconnect is edge-triggered on `close`, but a half-open tunnel
// never delivers one. Liveness pings and declares the socket dead when
// the server goes silent, so the close/reconnect path can run (#7718).
Expand All @@ -28,7 +30,11 @@ export function openSharedControlSocket(
}
}
): { ok: true; socket: RemoteRuntimeWebSocket } | { ok: false; error: RemoteRuntimeClientError } {
let noteActivity: () => void = () => {}
let noteLiveness: () => void = () => {}
const noteActivity = (): void => {
noteLiveness()
callbacks.onActivity?.()
}
const opened = openRemoteRuntimeWebSocket(connection, {
onClose: (ws, code, reason) => {
if (callbacks.getCurrentSocket() === ws) {
Expand Down Expand Up @@ -90,7 +96,7 @@ export function openSharedControlSocket(
},
options: liveness.options
})
noteActivity = monitor.noteActivity
noteLiveness = monitor.noteActivity

return {
ok: true,
Expand Down
2 changes: 2 additions & 0 deletions src/shared/remote-runtime-shared-control-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ export type RemoteRuntimeSharedControlConnectionOptions = {
isCapabilityPaused?: () => boolean
/** Publishes local transport diagnostics after a meaningful state transition. */
onDiagnosticsChanged?: (diagnostics: RemoteRuntimeSharedConnectionDiagnostics) => void
/** Any inbound frame, ping or pong on the current socket. */
onAnswered?: () => void
reconnectStableResetMs?: number
liveness?: RemoteRuntimeSocketLivenessOptions
}
Loading