From 39b591064bf5e754a799d88fa6c41328e7bc632d Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:54:56 -0700 Subject: [PATCH] feat(orcad): deploy, activate and roll back a versioned orcad install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan items 6 and 7 from docs/design/shipping-orcad.html. Install reuses the relay's transaction verbatim — per-version lock, staged SFTP write, .install-complete sentinel, stale-lock recovery — under a parameterized namespace, so orcad-/ sits beside relay-/ permanently (§06). Parameterizing GC is the trap that creates: each model now collects only its own directories, enforced twice (prefix-scoped remote listing plus a local ownership re-check), and a client picks its model from how the host is registered, never from what it finds on disk. Activation is separate from installation, because a versioned directory selects nothing. A candidate is launched, publishes orca_server_ready, and only becomes active if its cross-process health payload passes: right build hash, listening, daemon live, PTY self-test green. A rejected candidate is stopped and the incumbent restarted, so a careful deploy cannot cause the outage it was being careful about. Update and rollback are shaped by the daemon. An update restarts orcad, the daemon outlives it, and the surviving daemon was forked from the outgoing bundle — so live terminals defer the update rather than proceed, and GC pins the active version, the rollback target and the live daemon's bundle. Orca's persisted state carries no schema version, so rollback restores a pre-activation snapshot rather than trusting backward-readability; the point past which it is unsafe is the first terminal created after activation, which the snapshot cannot describe and the surviving daemon still owns. Running the generated shell for real found two bugs the text assertions missed: tar members re-quoted inside a shell variable captured nothing, and kill -0 reports a zombie as alive. --- config/scripts/build-orcad.mjs | 36 +- src/main/ssh/orcad-activation-gate.test.ts | 164 +++++++++ src/main/ssh/orcad-activation-gate.ts | 155 ++++++++ src/main/ssh/orcad-activation-record-store.ts | 48 +++ src/main/ssh/orcad-activation-record.test.ts | 102 ++++++ src/main/ssh/orcad-activation-record.ts | 175 +++++++++ src/main/ssh/orcad-local-build-hash.ts | 22 ++ src/main/ssh/orcad-remote-deploy.test.ts | 264 ++++++++++++++ src/main/ssh/orcad-remote-deploy.ts | 340 ++++++++++++++++++ src/main/ssh/orcad-remote-gc.test.ts | 153 ++++++++ src/main/ssh/orcad-remote-gc.ts | 74 ++++ src/main/ssh/orcad-remote-host-support.ts | 50 +++ src/main/ssh/orcad-remote-launch.test.ts | 121 +++++++ src/main/ssh/orcad-remote-launch.ts | 178 +++++++++ src/main/ssh/orcad-remote-process-control.ts | 73 ++++ src/main/ssh/orcad-remote-rollback.test.ts | 206 +++++++++++ src/main/ssh/orcad-remote-rollback.ts | 244 +++++++++++++ ...-remote-shell-commands.integration.test.ts | 189 ++++++++++ src/main/ssh/orcad-state-snapshot.test.ts | 100 ++++++ src/main/ssh/orcad-state-snapshot.ts | 173 +++++++++ src/main/ssh/orcad-update-plan.test.ts | 177 +++++++++ src/main/ssh/orcad-update-plan.ts | 234 ++++++++++++ .../ssh/remote-install-coexistence.test.ts | 76 ++++ src/main/ssh/remote-install-coexistence.ts | 83 +++++ src/main/ssh/remote-install-gc.ts | 286 +++++++++++++++ src/main/ssh/remote-install-model.test.ts | 70 ++++ src/main/ssh/remote-install-model.ts | 143 ++++++++ src/main/ssh/ssh-relay-install-namespace.ts | 28 +- src/main/ssh/ssh-relay-versioned-install.ts | 252 ++----------- src/main/ssh/ssh-remote-commands.ts | 51 ++- src/shared/orcad-artifacts.ts | 42 +++ 31 files changed, 4087 insertions(+), 222 deletions(-) create mode 100644 src/main/ssh/orcad-activation-gate.test.ts create mode 100644 src/main/ssh/orcad-activation-gate.ts create mode 100644 src/main/ssh/orcad-activation-record-store.ts create mode 100644 src/main/ssh/orcad-activation-record.test.ts create mode 100644 src/main/ssh/orcad-activation-record.ts create mode 100644 src/main/ssh/orcad-local-build-hash.ts create mode 100644 src/main/ssh/orcad-remote-deploy.test.ts create mode 100644 src/main/ssh/orcad-remote-deploy.ts create mode 100644 src/main/ssh/orcad-remote-gc.test.ts create mode 100644 src/main/ssh/orcad-remote-gc.ts create mode 100644 src/main/ssh/orcad-remote-host-support.ts create mode 100644 src/main/ssh/orcad-remote-launch.test.ts create mode 100644 src/main/ssh/orcad-remote-launch.ts create mode 100644 src/main/ssh/orcad-remote-process-control.ts create mode 100644 src/main/ssh/orcad-remote-rollback.test.ts create mode 100644 src/main/ssh/orcad-remote-rollback.ts create mode 100644 src/main/ssh/orcad-remote-shell-commands.integration.test.ts create mode 100644 src/main/ssh/orcad-state-snapshot.test.ts create mode 100644 src/main/ssh/orcad-state-snapshot.ts create mode 100644 src/main/ssh/orcad-update-plan.test.ts create mode 100644 src/main/ssh/orcad-update-plan.ts create mode 100644 src/main/ssh/remote-install-coexistence.test.ts create mode 100644 src/main/ssh/remote-install-coexistence.ts create mode 100644 src/main/ssh/remote-install-gc.ts create mode 100644 src/main/ssh/remote-install-model.test.ts create mode 100644 src/main/ssh/remote-install-model.ts create mode 100644 src/shared/orcad-artifacts.ts diff --git a/config/scripts/build-orcad.mjs b/config/scripts/build-orcad.mjs index 9a6bacd5eddc..69c12f0ecc95 100644 --- a/config/scripts/build-orcad.mjs +++ b/config/scripts/build-orcad.mjs @@ -9,10 +9,25 @@ */ import { fork, spawnSync } from 'node:child_process' import { build } from 'esbuild' -import { chmodSync, copyFileSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' import { arch, platform, tmpdir } from 'node:os' import { join } from 'node:path' import process from 'node:process' +import { + ORCAD_VERSION, + ORCAD_VERSION_FILENAME, + orcadArtifactFilenames +} from '../../src/shared/orcad-artifacts.ts' const ROOT = join(import.meta.dirname, '..', '..') const OUT_DIR = join(ROOT, 'out', 'orcad') @@ -226,9 +241,26 @@ if (graphErrors.length > 0) { } } +// Why a content hash and not ORCAD_VERSION alone: the remote install directory is keyed on +// this string, so two different builds carrying one version would share a directory — and an +// already-`.install-complete` dir is never re-uploaded. The deploy would silently run stale +// bytes while reporting the new version. if (process.exitCode !== 1) { + const hash = createHash('sha256') + for (const filename of orcadArtifactFilenames()) { + const artifactPath = join(OUT_DIR, filename) + if (!existsSync(artifactPath)) { + throw new Error( + `orcad declares ${filename} in ORCAD_ARTIFACTS but never emitted it. Add the build ` + + 'step, or drop it from src/shared/orcad-artifacts.ts.' + ) + } + hash.update(readFileSync(artifactPath)) + } + const fullVersion = `${ORCAD_VERSION}+${hash.digest('hex').slice(0, 12)}` + writeFileSync(join(OUT_DIR, ORCAD_VERSION_FILENAME), fullVersion) console.log( - `[build-orcad] ok — ${(output.bytes / 1024 / 1024).toFixed(2)} MB, ${Object.keys(output.inputs).length} modules, zero electron and node:sqlite imports.` + `[build-orcad] ok — ${fullVersion}, ${(output.bytes / 1024 / 1024).toFixed(2)} MB, ${Object.keys(output.inputs).length} modules, zero electron and node:sqlite imports.` ) } diff --git a/src/main/ssh/orcad-activation-gate.test.ts b/src/main/ssh/orcad-activation-gate.test.ts new file mode 100644 index 000000000000..534d41c08f02 --- /dev/null +++ b/src/main/ssh/orcad-activation-gate.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest' + +import { evaluateOrcadActivation } from './orcad-activation-gate' +import type { ServeReadiness } from '../server/serve-readiness' +import type { OrcadHealth, TerminalDaemonHealth } from '../orcad/orcad-health' + +const EXPECTED = { buildHash: 'abc123def4567890', fullVersion: '0.2.0+bb01' } + +function daemon(overrides: Partial = {}): TerminalDaemonHealth { + return { + state: 'live', + ownsFreshSessions: true, + pid: 4242, + buildVersion: '0.2.0+bb01', + entryPath: '/home/u/.orca-remote/orcad-0.2.0+bb01/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 12 }, + ...overrides + } +} + +function health(overrides: Partial = {}): OrcadHealth { + return { + buildHash: EXPECTED.buildHash, + buildVersion: EXPECTED.fullVersion, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 4200, + terminalDaemon: daemon(), + ...overrides + } +} + +function readiness(overrides: Partial = {}): ServeReadiness { + return { + runtimeId: 'runtime-1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: health(), + ...overrides + } +} + +describe('evaluateOrcadActivation', () => { + it('activates a candidate that proved a real PTY round trip', () => { + const verdict = evaluateOrcadActivation(readiness(), EXPECTED) + expect(verdict).toEqual({ decision: 'activate', coverage: 'pty-spawn', warnings: [] }) + }) + + it('refuses when the candidate never published readiness', () => { + const verdict = evaluateOrcadActivation(null, EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_no_readiness' }) + }) + + it('refuses a readiness payload with no health, rather than reading silence as healthy', () => { + const { health: _dropped, ...withoutHealth } = readiness() + const verdict = evaluateOrcadActivation(withoutHealth as ServeReadiness, EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_no_health' }) + }) + + it('refuses when a different build answered — a stale process holding the port', () => { + const verdict = evaluateOrcadActivation( + readiness({ health: health({ buildHash: '0000000000000000' }) }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_build_mismatch' }) + }) + + it('refuses a listening orcad whose terminal daemon is absent', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + terminalDaemon: daemon({ + state: 'absent', + ownsFreshSessions: false, + selfTest: { ok: false, coverage: 'pty-spawn', verdict: 'no-daemon', durationMs: 1 } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_daemon_absent' }) + }) + + it('refuses a degraded daemon, whose fresh terminals would not survive a restart', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ terminalDaemon: daemon({ state: 'degraded', ownsFreshSessions: false }) }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_daemon_degraded' }) + }) + + it('refuses a live daemon that failed its PTY spawn probe', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + terminalDaemon: daemon({ + selfTest: { + ok: false, + coverage: 'pty-spawn', + verdict: 'pty-spawn-unhealthy', + durationMs: 30 + } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ + decision: 'reject', + code: 'orcad_activation_pty_self_test_failed' + }) + }) + + it('refuses a green daemon that does not own fresh sessions', () => { + const verdict = evaluateOrcadActivation( + readiness({ health: health({ terminalDaemon: daemon({ ownsFreshSessions: false }) }) }), + EXPECTED + ) + expect(verdict).toMatchObject({ + decision: 'reject', + code: 'orcad_activation_no_persistent_terminals' + }) + }) + + it('refuses a candidate that is not listening', () => { + const verdict = evaluateOrcadActivation(readiness({ boundEndpoint: null }), EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_not_listening' }) + }) + + it('activates handshake-only coverage but never records it as a proven PTY', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + platform: 'win32', + terminalDaemon: daemon({ + selfTest: { ok: true, coverage: 'handshake', verdict: 'healthy', durationMs: 5 } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'activate', coverage: 'handshake' }) + expect(verdict.decision === 'activate' && verdict.warnings[0]).toContain( + 'covered the daemon handshake only' + ) + }) + + it('checks identity before health, so a wrong-build green payload cannot pass', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ buildHash: 'ffffffffffffffff', terminalDaemon: daemon() }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ code: 'orcad_activation_build_mismatch' }) + }) +}) diff --git a/src/main/ssh/orcad-activation-gate.ts b/src/main/ssh/orcad-activation-gate.ts new file mode 100644 index 000000000000..c95819191179 --- /dev/null +++ b/src/main/ssh/orcad-activation-gate.ts @@ -0,0 +1,155 @@ +/** + * Whether a freshly launched orcad has earned the right to become the active one. + * + * The failure this exists to prevent is the one `docs/design/shipping-orcad.html` names + * throughout: a deployment that reports success because a port opened. orcad answers RPC + * from its own process, so "listening" stays true while the terminal daemon that owns every + * terminal is dead — a green host that cannot run a single command. Activation therefore + * reads the cross-process health payload the candidate published, not the exit code of the + * command that started it. + * + * A refusal here is not a failure to deploy. The bytes are installed and the previous + * version is still active; nothing was lost. Activating on a bad verdict is what loses + * things. + */ +import type { ServeReadiness } from '../server/serve-readiness' + +export type OrcadActivationRejectCode = + | 'orcad_activation_no_readiness' + | 'orcad_activation_no_health' + | 'orcad_activation_build_mismatch' + | 'orcad_activation_not_listening' + | 'orcad_activation_daemon_absent' + | 'orcad_activation_daemon_degraded' + | 'orcad_activation_pty_self_test_failed' + | 'orcad_activation_no_persistent_terminals' + +export type OrcadActivationVerdict = + | { + decision: 'activate' + /** + * `pty-spawn` means a real PTY was created and torn down inside the daemon. + * `handshake` means the daemon answered but its spawn probe is a no-op on this + * platform (win32). Carried through so an activation is never recorded as proving + * more than it did. + */ + coverage: 'pty-spawn' | 'handshake' + warnings: string[] + } + | { decision: 'reject'; code: OrcadActivationRejectCode; reason: string } + +export type OrcadActivationExpectation = { + /** sha256(orcad.js).slice(0,16) computed from the bytes this client just uploaded. */ + buildHash: string + /** The full content-hashed version this deploy installed. */ + fullVersion: string +} + +/** + * Gate an activation on what the candidate actually reported. + * + * Order matters: identity before health. A health payload from the wrong process is worse + * than no payload, because it is green and about something else. + */ +export function evaluateOrcadActivation( + readiness: ServeReadiness | null, + expected: OrcadActivationExpectation +): OrcadActivationVerdict { + if (!readiness) { + return { + decision: 'reject', + code: 'orcad_activation_no_readiness', + reason: + 'The candidate orcad never published an `orca_server_ready` line. It may have exited, ' + + 'failed to bind, or be wedged before readiness. Nothing was activated.' + } + } + const health = readiness.health + if (!health) { + return { + decision: 'reject', + code: 'orcad_activation_no_health', + reason: + 'The candidate published readiness without a health payload, so its terminal daemon ' + + 'is unverified. Absence of a verdict is not a healthy verdict — treat this build as ' + + 'too old to gate on and do not activate it.' + } + } + // Why identity first: a stale orcad already holding the port would answer readiness and + // report its own (healthy) daemon. Activating on that record points the pointer at bytes + // nobody is running. + if (health.buildHash !== expected.buildHash) { + return { + decision: 'reject', + code: 'orcad_activation_build_mismatch', + reason: + `The process that answered is running build ${health.buildHash}, not the ` + + `${expected.buildHash} this deploy installed. Something else owns that port, or the ` + + 'upload did not land. Nothing was activated.' + } + } + if (!readiness.boundEndpoint) { + return { + decision: 'reject', + code: 'orcad_activation_not_listening', + reason: + 'The candidate reported no bound endpoint, so no client could reach it. Nothing was ' + + 'activated.' + } + } + const daemon = health.terminalDaemon + if (daemon.state === 'absent') { + return { + decision: 'reject', + code: 'orcad_activation_daemon_absent', + reason: + 'The candidate has no terminal daemon. Every terminal on this host would run in the ' + + 'orcad process and die with it, which is the exact regression the daemon exists to ' + + 'prevent. Nothing was activated.' + } + } + if (daemon.state === 'degraded') { + return { + decision: 'reject', + code: 'orcad_activation_daemon_degraded', + reason: + `The candidate's terminal daemon is degraded (self-test: ${daemon.selfTest.verdict}). ` + + 'Existing sessions keep working, but fresh terminals would not survive a restart. ' + + 'Nothing was activated; the previous version is still serving.' + } + } + if (!daemon.selfTest.ok) { + return { + decision: 'reject', + code: 'orcad_activation_pty_self_test_failed', + reason: + `The candidate's PTY self-test failed (${daemon.selfTest.verdict}). The host is ` + + 'listening but cannot create a terminal. Nothing was activated.' + } + } + if (!daemon.ownsFreshSessions) { + return { + decision: 'reject', + code: 'orcad_activation_no_persistent_terminals', + reason: + 'The candidate answered healthy but does not own fresh sessions, so new terminals ' + + 'would not survive its own restart. Nothing was activated.' + } + } + const warnings: string[] = [] + if (daemon.selfTest.coverage === 'handshake') { + warnings.push( + 'The PTY self-test covered the daemon handshake only — this platform does not spawn a ' + + 'probe PTY. Terminal creation is unproven on this host.' + ) + } + if (health.buildVersion !== expected.fullVersion) { + // Not a rejection: the hash already proved identity, and ORCA_VERSION is whatever the + // launch command exported. Worth saying, because a mismatch means the launch env is wrong. + warnings.push( + `The candidate reports version ${health.buildVersion} but was installed as ` + + `${expected.fullVersion}; check ORCA_VERSION in the launch command.` + ) + } + return { decision: 'activate', coverage: daemon.selfTest.coverage, warnings } +} diff --git a/src/main/ssh/orcad-activation-record-store.ts b/src/main/ssh/orcad-activation-record-store.ts new file mode 100644 index 000000000000..d60634ec525d --- /dev/null +++ b/src/main/ssh/orcad-activation-record-store.ts @@ -0,0 +1,48 @@ +/** + * Reading the activation record off a host. + * + * Split out from the deploy driver because the rollback path needs it too, and because an + * unreadable record must fail loudly in both: treating "I cannot parse this" as "nothing is + * activated" would deploy over a live install and lose its rollback target. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_ACTIVATION_FILENAME, + emptyOrcadActivationRecord, + parseOrcadActivationRecord, + type OrcadActivationRecord +} from './orcad-activation-record' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +export function orcadActivationPath(host: RemoteHostPlatform, remoteHome: string): string { + return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_ACTIVATION_FILENAME) +} + +export async function readOrcadActivationRecord(options: { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + signal?: AbortSignal +}): Promise { + const path = orcadActivationPath(options.host, options.remoteHome) + const raw = await execCommand(options.conn, `cat ${shellQuote(path)} 2>/dev/null || true`, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + }).catch(() => '') + const parsed = parseOrcadActivationRecord(raw) + if (parsed.state === 'ok') { + return parsed.record + } + if (parsed.state === 'unreadable') { + // Why throw: an unreadable record is not an empty one. Treating it as empty would + // activate over a live install and orphan its rollback target. + throw new Error(`Cannot read this host's orcad activation record: ${parsed.reason}`) + } + return emptyOrcadActivationRecord() +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} diff --git a/src/main/ssh/orcad-activation-record.test.ts b/src/main/ssh/orcad-activation-record.test.ts new file mode 100644 index 000000000000..dd625238bb12 --- /dev/null +++ b/src/main/ssh/orcad-activation-record.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest' + +import { + emptyOrcadActivationRecord, + orcadGcPinnedDirNames, + parseOrcadActivationRecord, + serializeOrcadActivationRecord, + withActivatedVersion, + withRolledBackVersion, + type OrcadStateSnapshot +} from './orcad-activation-record' + +const SNAPSHOT: OrcadStateSnapshot = { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: '0.2.0+bb01', + readableByVersion: '0.1.0+aa01', + takenAt: '2026-01-01T00:00:00.000Z' +} +const NOW = new Date('2026-01-02T00:00:00.000Z') + +describe('orcad activation record', () => { + it('round-trips through the host', () => { + const record = withActivatedVersion( + { ...emptyOrcadActivationRecord(), active: '0.1.0+aa01' }, + '0.2.0+bb01', + SNAPSHOT, + NOW + ) + const parsed = parseOrcadActivationRecord(serializeOrcadActivationRecord(record)) + expect(parsed).toEqual({ state: 'ok', record }) + }) + + it('reports an absent record as absent', () => { + expect(parseOrcadActivationRecord(null)).toEqual({ state: 'absent' }) + expect(parseOrcadActivationRecord(' ')).toEqual({ state: 'absent' }) + }) + + it('reports a newer schema as unreadable, never as absent', () => { + const parsed = parseOrcadActivationRecord(JSON.stringify({ schemaVersion: 2, active: 'x' })) + expect(parsed.state).toBe('unreadable') + }) + + it('reports corrupt JSON as unreadable, never as absent', () => { + expect(parseOrcadActivationRecord('{not json').state).toBe('unreadable') + }) + + it('names the outgoing version as the rollback target', () => { + const record = withActivatedVersion( + { ...emptyOrcadActivationRecord(), active: '0.1.0+aa01' }, + '0.2.0+bb01', + SNAPSHOT, + NOW + ) + expect(record).toMatchObject({ active: '0.2.0+bb01', previous: '0.1.0+aa01' }) + }) + + it('does not let a re-deploy of the active version erase the rollback target', () => { + const before = { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + snapshot: SNAPSHOT + } + const after = withActivatedVersion(before, '0.2.0+bb01', null, NOW) + expect(after).toMatchObject({ active: '0.2.0+bb01', previous: '0.1.0+aa01' }) + expect(after.snapshot).toEqual(SNAPSHOT) + }) + + it('clears the rollback target after rolling back, so it cannot walk into the bad build', () => { + const before = { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + snapshot: SNAPSHOT + } + expect(withRolledBackVersion(before, NOW)).toMatchObject({ + active: '0.1.0+aa01', + previous: null, + snapshot: null + }) + }) + + it('pins the active version, the rollback target and the live daemon"s bundle against GC', () => { + const pinned = orcadGcPinnedDirNames( + { + ...emptyOrcadActivationRecord(), + active: '0.3.0+cc01', + previous: '0.2.0+bb01' + }, + '0.1.0+aa01' + ) + expect(pinned).toEqual(['orcad-0.3.0+cc01', 'orcad-0.2.0+bb01', 'orcad-0.1.0+aa01']) + }) + + it('deduplicates pins when the live daemon came from the active bundle', () => { + const pinned = orcadGcPinnedDirNames( + { ...emptyOrcadActivationRecord(), active: '0.3.0+cc01', previous: null }, + '0.3.0+cc01' + ) + expect(pinned).toEqual(['orcad-0.3.0+cc01']) + }) +}) diff --git a/src/main/ssh/orcad-activation-record.ts b/src/main/ssh/orcad-activation-record.ts new file mode 100644 index 000000000000..8c32e6c355dc --- /dev/null +++ b/src/main/ssh/orcad-activation-record.ts @@ -0,0 +1,175 @@ +/** + * Which installed orcad is the live one, and which one a rollback goes back to. + * + * A versioned install directory decides where bytes land; it does not decide which version + * runs. Without this record, "roll back" means "deploy the old version again" — which needs + * the client that has those bytes, on a host that may be the only thing still working. The + * record is the host-side half: it names an active version, a rollback target, and the + * pre-activation state snapshot that makes going back to that target sound. + * + * It lives beside the version dirs (`~/.orca-remote/orcad-active.json`), not inside one, + * because it has to outlive whichever version GC removes. + */ +import { remoteInstallDirName, ORCAD_INSTALL_MODEL } from './remote-install-model' + +export const ORCAD_ACTIVATION_FILENAME = 'orcad-active.json' +export const ORCAD_ACTIVATION_SCHEMA_VERSION = 1 + +/** Where a pre-activation copy of the shared data root lives, relative to `.orca-remote/`. */ +export const ORCAD_STATE_SNAPSHOT_DIR = 'orcad-state-snapshots' + +export type OrcadStateSnapshot = { + /** Directory name under `ORCAD_STATE_SNAPSHOT_DIR`. */ + dirName: string + /** The version whose activation this snapshot was taken FOR — i.e. taken before it ran. */ + takenBeforeVersion: string + /** The version that produced the state, i.e. the rollback target it is readable by. */ + readableByVersion: string | null + takenAt: string +} + +export type OrcadActivationRecord = { + schemaVersion: typeof ORCAD_ACTIVATION_SCHEMA_VERSION + /** Full content-hashed version, e.g. `0.1.0+9f2a1c`. Null before the first activation. */ + active: string | null + /** The version `active` replaced. The rollback target, and pinned against GC. */ + previous: string | null + activatedAt: string | null + snapshot: OrcadStateSnapshot | null +} + +export function emptyOrcadActivationRecord(): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: null, + previous: null, + activatedAt: null, + snapshot: null + } +} + +/** + * Parse the record read off the host. + * + * Why a null return and not a throw on a newer schema: a client older than the host must not + * treat "I cannot read this" as "nothing is activated" — that would deploy over a live + * install. Callers distinguish the two through `OrcadActivationReadResult`. + */ +export type OrcadActivationReadResult = + | { state: 'absent' } + | { state: 'ok'; record: OrcadActivationRecord } + | { state: 'unreadable'; reason: string } + +export function parseOrcadActivationRecord(raw: string | null): OrcadActivationReadResult { + if (raw === null || raw.trim() === '') { + return { state: 'absent' } + } + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch (error) { + return { + state: 'unreadable', + reason: `activation record is not JSON: ${error instanceof Error ? error.message : String(error)}` + } + } + if (typeof parsed !== 'object' || parsed === null) { + return { state: 'unreadable', reason: 'activation record is not an object' } + } + const record = parsed as Partial + if (record.schemaVersion !== ORCAD_ACTIVATION_SCHEMA_VERSION) { + return { + state: 'unreadable', + reason: + `activation record schemaVersion ${String(record.schemaVersion)} is not ` + + `${ORCAD_ACTIVATION_SCHEMA_VERSION}; this client cannot safely interpret it` + } + } + return { + state: 'ok', + record: { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: typeof record.active === 'string' ? record.active : null, + previous: typeof record.previous === 'string' ? record.previous : null, + activatedAt: typeof record.activatedAt === 'string' ? record.activatedAt : null, + snapshot: parseSnapshot(record.snapshot) + } + } +} + +function parseSnapshot(value: unknown): OrcadStateSnapshot | null { + if (typeof value !== 'object' || value === null) { + return null + } + const snapshot = value as Partial + if (typeof snapshot.dirName !== 'string' || typeof snapshot.takenBeforeVersion !== 'string') { + return null + } + return { + dirName: snapshot.dirName, + takenBeforeVersion: snapshot.takenBeforeVersion, + readableByVersion: + typeof snapshot.readableByVersion === 'string' ? snapshot.readableByVersion : null, + takenAt: typeof snapshot.takenAt === 'string' ? snapshot.takenAt : '' + } +} + +export function serializeOrcadActivationRecord(record: OrcadActivationRecord): string { + return `${JSON.stringify(record, null, 2)}\n` +} + +/** The record that results from activating `version`, keeping the outgoing one as the target. */ +export function withActivatedVersion( + record: OrcadActivationRecord, + version: string, + snapshot: OrcadStateSnapshot | null, + now: Date +): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: version, + // Why keep the OLD previous when re-activating the same version: a repeated deploy of + // an already-active build is not a version change, so it must not erase the rollback + // target by naming the active version as its own predecessor. + previous: record.active === version ? record.previous : record.active, + activatedAt: now.toISOString(), + snapshot: record.active === version ? record.snapshot : snapshot + } +} + +/** The record that results from rolling `active` back to `previous`. */ +export function withRolledBackVersion( + record: OrcadActivationRecord, + now: Date +): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: record.previous, + // Why null and not the version we just left: it is the build we are rolling back FROM, + // so offering it as the next rollback target would walk straight back into the failure. + previous: null, + activatedAt: now.toISOString(), + // The snapshot was taken before `active` ran; once restored it has been consumed. + snapshot: null + } +} + +/** + * Version dirs GC must not remove, as directory names. + * + * `previous` is here because a rollback target that GC deleted is not a rollback target. + * `daemonEntryVersion` is here because an update preserves a live daemon forked from the + * OUTGOING bundle (see orcad-update-plan.ts) — deleting the tree under a running process is + * how a later respawn finds no entry point. + */ +export function orcadGcPinnedDirNames( + record: OrcadActivationRecord, + daemonEntryVersion?: string | null +): string[] { + const versions = [record.active, record.previous, daemonEntryVersion ?? null].filter( + (v): v is string => typeof v === 'string' && v.length > 0 + ) + return [...new Set(versions)].map((version) => + remoteInstallDirName(ORCAD_INSTALL_MODEL, version) + ) +} diff --git a/src/main/ssh/orcad-local-build-hash.ts b/src/main/ssh/orcad-local-build-hash.ts new file mode 100644 index 000000000000..3740ea6d55c5 --- /dev/null +++ b/src/main/ssh/orcad-local-build-hash.ts @@ -0,0 +1,22 @@ +/** + * The build identity the deploy expects the host to answer with. + * + * It must be computed the same way `computeOrcadBuildHash` computes it on the host — + * sha256 of `orcad.js`, first 16 hex characters — or the activation gate would reject every + * healthy candidate. Keeping the two in one comment is deliberate: they are one contract + * split across a network, and the version string cannot stand in for it, because + * `ORCA_VERSION` is whatever the launch command exported and two builds can carry one value. + */ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +export const ORCAD_BUILD_HASH_LENGTH = 16 + +export function computeLocalOrcadBuildHash(localOrcadDir: string): string { + const entry = join(localOrcadDir, 'orcad.js') + return createHash('sha256') + .update(readFileSync(entry)) + .digest('hex') + .slice(0, ORCAD_BUILD_HASH_LENGTH) +} diff --git a/src/main/ssh/orcad-remote-deploy.test.ts b/src/main/ssh/orcad-remote-deploy.test.ts new file mode 100644 index 000000000000..799193f3384f --- /dev/null +++ b/src/main/ssh/orcad-remote-deploy.test.ts @@ -0,0 +1,264 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', () => ({ + acquireInstallLock: vi.fn().mockResolvedValue(undefined), + RELAY_INSTALL_LOCK_NAME: '.install-lock' +})) +vi.mock('./ssh-relay-install-transfers', () => ({ + uploadRelayDirectory: vi.fn().mockResolvedValue(undefined), + writeRelayFile: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./orcad-local-build-hash', () => ({ + computeLocalOrcadBuildHash: () => 'abc123def4567890' +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy' +import { emptyOrcadActivationRecord, withActivatedVersion } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const mockExec = vi.mocked(execCommand) +const NEW_VERSION = '0.2.0+bb01' +const OLD_VERSION = '0.1.0+aa01' + +vi.mock('./ssh-relay-versioned-install', async (importOriginal) => ({ + ...(await importOriginal>()), + readLocalFullVersion: () => '0.2.0+bb01', + isRemoteInstallComplete: vi.fn().mockResolvedValue(false), + finalizeInstall: vi.fn().mockResolvedValue(undefined), + abandonInstall: vi.fn().mockResolvedValue(undefined) +})) + +function readyLine(overrides: { + buildHash?: string + daemonState?: 'live' | 'degraded' | 'absent' + selfTestOk?: boolean +}): string { + return JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: overrides.buildHash ?? 'abc123def4567890', + buildVersion: NEW_VERSION, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: overrides.daemonState ?? 'live', + ownsFreshSessions: (overrides.daemonState ?? 'live') === 'live', + pid: 2, + buildVersion: NEW_VERSION, + entryPath: '/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { + ok: overrides.selfTestOk ?? true, + coverage: 'pty-spawn', + verdict: (overrides.selfTestOk ?? true) ? 'healthy' : 'pty-spawn-unhealthy', + durationMs: 5 + } + } + } + }) +} + +type HostScript = { + activationRecord: string + /** Readiness content per version dir, keyed by the version in the path. */ + readiness: Record + log: string[] +} + +function scriptHost(script: HostScript): void { + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.startsWith('cat ') && text.includes('orcad-active.json')) { + return script.activationRecord + } + if (text.includes('.orcad-readiness') && text.startsWith('cat ')) { + const version = Object.keys(script.readiness).find((v) => text.includes(v)) + return version ? script.readiness[version] : '' + } + if (text.includes('nohup')) { + script.log.push(`launch:${text.includes(NEW_VERSION) ? NEW_VERSION : OLD_VERSION}`) + return '9999' + } + if (text.includes('kill -TERM')) { + script.log.push(`stop:${text.includes(NEW_VERSION) ? NEW_VERSION : OLD_VERSION}`) + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-cf')) { + script.log.push('snapshot') + return 'CAPTURED' + } + return '' + }) +} + +function options(overrides: Partial = {}): OrcadDeployOptions { + return { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + localOrcadDir: '/local/out/orcad', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + census: { liveSessions: 0, startedSinceActivation: 0 }, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z'), + ...overrides + } +} + +const ACTIVE_OLD = JSON.stringify( + withActivatedVersion(emptyOrcadActivationRecord(), OLD_VERSION, null, new Date(0)) +) + +describe('deployOrcad', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('installs under the orcad namespace, not the relay one', async () => { + const script: HostScript = { + activationRecord: '', + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await deployOrcad(options()) + expect(vi.mocked(acquireInstallLock).mock.calls[0][1]).toBe( + `/home/u/.orca-remote/orcad-${NEW_VERSION}` + ) + expect(vi.mocked(uploadRelayDirectory).mock.calls[0][2]).toContain(`orcad-${NEW_VERSION}`) + }) + + it('activates a healthy candidate and records the outgoing version as the rollback target', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ outcome: 'installed-and-activated', fullVersion: NEW_VERSION }) + const written = vi + .mocked(writeRelayFile) + .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[3]))).toMatchObject({ + active: NEW_VERSION, + previous: OLD_VERSION + }) + }) + + it('snapshots the shared data root before the candidate ever runs', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await deployOrcad(options()) + expect(script.log.indexOf('snapshot')).toBeGreaterThan(-1) + expect(script.log.indexOf('snapshot')).toBeLessThan(script.log.indexOf(`launch:${NEW_VERSION}`)) + }) + + it('installs but does not activate when terminals are running', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad( + options({ census: { liveSessions: 2, startedSinceActivation: 0 } }) + ) + expect(result).toMatchObject({ + outcome: 'installed-not-activated', + code: 'orcad_update_terminals_running' + }) + // The bytes landed; nothing was stopped, launched or snapshotted. + expect(vi.mocked(uploadRelayDirectory)).toHaveBeenCalled() + expect(script.log).toEqual([]) + }) + + it('does not write the activation record when the candidate fails its health gate', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({ daemonState: 'degraded' }) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' }) + expect( + vi.mocked(writeRelayFile).mock.calls.some((call) => String(call[2]).endsWith('orcad-active.json')) + ).toBe(false) + }) + + it('puts the previous version back after a rejected candidate, rather than leaving the host down', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { + [NEW_VERSION]: readyLine({ selfTestOk: false }), + [OLD_VERSION]: readyLine({}) + }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ outcome: 'installed-not-activated' }) + expect(script.log).toEqual([ + 'snapshot', + `stop:${OLD_VERSION}`, + `launch:${NEW_VERSION}`, + `stop:${NEW_VERSION}`, + `launch:${OLD_VERSION}` + ]) + expect(result.outcome === 'installed-not-activated' && result.reason).toContain( + `orcad ${OLD_VERSION} was restarted and is serving again` + ) + }) + + it('refuses to activate when a different build answered the port', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { + [NEW_VERSION]: readyLine({ buildHash: 'deadbeefdeadbeef' }), + [OLD_VERSION]: readyLine({}) + }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ code: 'orcad_activation_build_mismatch' }) + }) + + it('refuses to treat an unreadable activation record as an empty one', async () => { + const script: HostScript = { + activationRecord: JSON.stringify({ schemaVersion: 99, active: 'x' }), + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await expect(deployOrcad(options())).rejects.toThrow('activation record') + expect(vi.mocked(uploadRelayDirectory)).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/orcad-remote-deploy.ts b/src/main/ssh/orcad-remote-deploy.ts new file mode 100644 index 000000000000..1f8a4099a826 --- /dev/null +++ b/src/main/ssh/orcad-remote-deploy.ts @@ -0,0 +1,340 @@ +/** + * Installing orcad on a host and, only if it proves itself, making it the active one. + * + * The install half is the relay's transaction, parameterized: the same per-version lock, + * staged SFTP write, `.install-complete` sentinel and stale-lock recovery, under + * `orcad-/` instead of `relay-/`. That is what §02 marks reusable. + * + * The activation half has no relay equivalent, because the relay has no notion of a version + * being *selected*. Bytes landing in a versioned directory neither picks a version nor rolls + * one back; the activation record does, and it is written only after the candidate publishes + * a health payload that survives `evaluateOrcadActivation`. A rejected candidate leaves the + * previous version running and its own bytes on disk — nothing is lost, and a retry costs no + * upload. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { + abandonInstall, + computeRemoteInstallDir, + finalizeInstall, + isRemoteInstallComplete, + readLocalFullVersion +} from './ssh-relay-versioned-install' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_STATE_SNAPSHOT_DIR, + serializeOrcadActivationRecord, + withActivatedVersion, + type OrcadActivationRecord, + type OrcadStateSnapshot +} from './orcad-activation-record' +import { + orcadActivationPath, + readOrcadActivationRecord +} from './orcad-activation-record-store' +import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' +import { planOrcadUpdate, type OrcadTerminalCensus } from './orcad-update-plan' +import { + ORCAD_LOG_FILENAME, + orcadLaunchCommand, + parseOrcadReadinessOutput, + readOrcadReadinessCommand, + type OrcadLaunchSpec +} from './orcad-remote-launch' +import { + captureOrcadStateSnapshotCommand, + orcadSnapshotDirName, + parseOrcadSnapshotCapture +} from './orcad-state-snapshot' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { computeLocalOrcadBuildHash } from './orcad-local-build-hash' + +export type OrcadDeployOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + /** Local `out/orcad`, containing the artifacts and the `.version` marker. */ + localOrcadDir: string + nodePath: string + userDataDir: string + bindHost: string + port: number + /** + * Live-terminal counts, supplied by the caller from the runtime it is already connected + * to. Not probed here: counting the daemon's sessions needs its protocol, and a deploy + * that guessed zero from silence would be the "loss of contact means death" mistake. + */ + census: OrcadTerminalCensus + force?: boolean + readinessTimeoutMs?: number + now?: () => Date + sleep?: (ms: number) => Promise + signal?: AbortSignal +} + +export type OrcadDeployResult = + | { outcome: 'installed-and-activated'; fullVersion: string; verdict: OrcadActivationVerdict } + | { outcome: 'already-active'; fullVersion: string } + | { outcome: 'installed-not-activated'; fullVersion: string; code: string; reason: string } + +const DEFAULT_READINESS_TIMEOUT_MS = 90_000 +const READINESS_POLL_MS = 500 +const STOP_WAIT_SECONDS = 20 + +function exec( + options: OrcadDeployOptions, + command: string, + signal = options.signal +): Promise { + return execCommand(options.conn, command, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal + }) +} + +function baseDir(options: OrcadDeployOptions): string { + return joinRemotePath(options.host, options.remoteHome, RELAY_REMOTE_DIR) +} + +/** Install the bytes under `orcad-/`, using the relay's install transaction. */ +async function installOrcadBundle( + options: OrcadDeployOptions, + fullVersion: string, + remoteDir: string +): Promise { + if (await isRemoteInstallComplete(options.conn, ORCAD_INSTALL_MODEL, remoteDir, options.host, { + signal: options.signal + })) { + return + } + await acquireInstallLock(options.conn, remoteDir, options.host, { signal: options.signal }) + try { + // Re-probe under the lock: a sibling deploy may have finished while we waited. + if ( + await isRemoteInstallComplete(options.conn, ORCAD_INSTALL_MODEL, remoteDir, options.host, { + signal: options.signal + }) + ) { + return + } + await uploadRelayDirectory(options.conn, options.localOrcadDir, remoteDir, options.host, { + signal: options.signal + }) + await writeRelayFile( + options.conn, + options.host, + joinRemotePath(options.host, remoteDir, ORCAD_INSTALL_MODEL.versionFilename), + fullVersion, + { signal: options.signal } + ) + await finalizeInstall(options.conn, remoteDir, options.host, { signal: options.signal }) + } catch (error) { + // Leave a recoverable partial rather than a dir that probes complete. + await abandonInstall(options.conn, remoteDir, options.host) + throw error + } +} + +async function captureSnapshot( + options: OrcadDeployOptions, + fullVersion: string, + outgoingVersion: string | null, + takenAt: Date +): Promise { + const dirName = orcadSnapshotDirName(fullVersion, takenAt.getTime()) + const snapshotDir = joinRemotePath(options.host, baseDir(options), ORCAD_STATE_SNAPSHOT_DIR, dirName) + const capture = parseOrcadSnapshotCapture( + await exec( + options, + captureOrcadStateSnapshotCommand(options.host, options.userDataDir, snapshotDir) + ) + ) + if (capture === 'failed') { + throw new Error( + `Could not snapshot ${options.userDataDir} before activating ${fullVersion}. Orca's ` + + 'persisted state carries no schema version, so without a snapshot a rollback has no ' + + 'way back. Refusing to activate.' + ) + } + if (capture === 'empty') { + // Nothing on the host to lose: a first deployment. Rollback will correctly report that + // it has no snapshot, rather than restoring an archive of nothing over a populated root. + return null + } + return { + dirName, + takenBeforeVersion: fullVersion, + readableByVersion: outgoingVersion, + takenAt: takenAt.toISOString() + } +} + +async function launchAndAwaitReadiness( + options: OrcadDeployOptions, + spec: OrcadLaunchSpec +): Promise> { + await exec(options, orcadLaunchCommand(options.host, spec)) + const deadline = Date.now() + (options.readinessTimeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS) + const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) + let last = parseOrcadReadinessOutput('') + while (Date.now() < deadline) { + options.signal?.throwIfAborted() + last = parseOrcadReadinessOutput( + await exec(options, readOrcadReadinessCommand(options.host, spec.remoteInstallDir)) + ) + if (last.state !== 'pending') { + return last + } + await sleep(READINESS_POLL_MS) + } + return last +} + +/** + * Put the previous version back after a rejected candidate. + * + * Why this exists at all: activating means swapping which process owns the data root and the + * port, so the incumbent has to stop before the candidate can start. A gate that rejected + * and returned would leave the host with nothing running — a careful deploy causing the + * outage it was being careful about. The returned sentence goes into the caller's reason so + * the operator learns the host's actual state, not just why the candidate failed. + */ +async function restoreIncumbent( + options: OrcadDeployOptions, + record: OrcadActivationRecord, + candidateDir: string +): Promise { + const stopped = parseOrcadStopOutcome( + await exec( + options, + stopOrcadCommand(options.host, candidateDir, { waitSeconds: STOP_WAIT_SECONDS }) + ) + ) + if (!orcadStopFreedTheHost(stopped)) { + return `The candidate itself did not stop (${stopped}); the host may still be serving the rejected build.` + } + if (!record.active) { + return 'No previous version was active, so this host is now serving nothing.' + } + const incumbentDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + record.active + ) + const parsed = await launchAndAwaitReadiness(options, { + remoteInstallDir: incumbentDir, + nodePath: options.nodePath, + fullVersion: record.active, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + return parsed.state === 'ready' + ? `orcad ${record.active} was restarted and is serving again.` + : `orcad ${record.active} was relaunched but has not published readiness; this host may be down.` +} + +/** + * Install, then activate only on a green cross-process health verdict. + * + * Every early return past the install leaves the bytes on disk and the previous version + * serving, which is why they all report `installed-not-activated` rather than throwing: a + * refusal to switch is a successful outcome of a deploy that was asked to be careful. + */ +export async function deployOrcad(options: OrcadDeployOptions): Promise { + const now = options.now ?? ((): Date => new Date()) + const fullVersion = readLocalFullVersion(options.localOrcadDir) + const remoteDir = computeRemoteInstallDir(ORCAD_INSTALL_MODEL, options.remoteHome, fullVersion) + const record = await readOrcadActivationRecord(options) + + await installOrcadBundle(options, fullVersion, remoteDir) + + const plan = planOrcadUpdate({ + record, + candidateVersion: fullVersion, + census: options.census, + ...(options.force !== undefined ? { force: options.force } : {}) + }) + if (plan.action === 'noop') { + return { outcome: 'already-active', fullVersion } + } + if (plan.action === 'defer') { + return { + outcome: 'installed-not-activated', + fullVersion, + code: plan.code, + reason: plan.reason + } + } + + const snapshot = record.active + ? await captureSnapshot(options, fullVersion, record.active, now()) + : null + + if (record.active) { + const outgoingDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + record.active + ) + const stopped = parseOrcadStopOutcome( + await exec(options, stopOrcadCommand(options.host, outgoingDir, { + waitSeconds: STOP_WAIT_SECONDS + })) + ) + if (!orcadStopFreedTheHost(stopped)) { + return { + outcome: 'installed-not-activated', + fullVersion, + code: 'orcad_outgoing_stop_incomplete', + reason: + `orcad ${record.active} did not exit within ${STOP_WAIT_SECONDS}s of SIGTERM ` + + `(${stopped}). It is still holding the data root and the port, so the candidate ` + + 'cannot start. Not escalating to SIGKILL: that skips the shutdown that releases ' + + 'the instance lock, and the successor would then refuse to start.' + } + } + } + + const parsed = await launchAndAwaitReadiness(options, { + remoteInstallDir: remoteDir, + nodePath: options.nodePath, + fullVersion, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { + buildHash: computeLocalOrcadBuildHash(options.localOrcadDir), + fullVersion + }) + if (verdict.decision === 'reject') { + const restored = await restoreIncumbent(options, record, remoteDir) + return { + outcome: 'installed-not-activated', + fullVersion, + code: verdict.code, + reason: + `${verdict.reason} Candidate stderr is at ` + + `${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}` + } + } + + await writeRelayFile( + options.conn, + options.host, + orcadActivationPath(options.host, options.remoteHome), + serializeOrcadActivationRecord(withActivatedVersion(record, fullVersion, snapshot, now())), + { signal: options.signal } + ) + return { outcome: 'installed-and-activated', fullVersion, verdict } +} diff --git a/src/main/ssh/orcad-remote-gc.test.ts b/src/main/ssh/orcad-remote-gc.test.ts new file mode 100644 index 000000000000..3716aaddaf99 --- /dev/null +++ b/src/main/ssh/orcad-remote-gc.test.ts @@ -0,0 +1,153 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-gc-claim', () => ({ + isRelayGcClaimOwned: vi.fn().mockResolvedValue(true), + releaseRelayGcClaimWithRetry: vi.fn().mockResolvedValue('released'), + tryAcquireRelayGcClaim: vi.fn().mockResolvedValue('token') +})) +vi.mock('./ssh-relay-gc-tombstone', () => ({ + cleanupRelayGcTombstones: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./ssh-relay-install-lock', () => ({ + RELAY_INSTALL_LOCK_NAME: '.install-lock', + isRelayInstallLockStale: vi.fn().mockResolvedValue(false) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { gcOldOrcadVersions } from './orcad-remote-gc' +import { emptyOrcadActivationRecord } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const conn = {} as SshConnection +const host = getRemoteHostPlatform('linux-x64') +const mockExec = vi.mocked(execCommand) + +/** + * Drive one GC pass against a scripted host. `listing` is what the remote listing command + * returns; every other command answers from `responses`, defaulting to the happy path + * (unlocked, complete, dead) so a candidate is removed unless a test says otherwise. + */ +function scriptHost(options: { + listing: string[] + liveness?: Record + removed: string[] +}): void { + mockExec.mockImplementation(async (_conn, command: string) => { + if (command.includes('-mindepth 1 -maxdepth 1')) { + return options.listing.join('\n') + } + if (command.includes('.install-lock')) { + return 'OPEN' + } + if (command.includes('.install-complete')) { + return 'COMPLETE' + } + if (command.includes('.orcad-pid')) { + const dir = Object.keys(options.liveness ?? {}).find((name) => command.includes(name)) + return dir ? (options.liveness?.[dir] ?? 'DEAD') : 'DEAD' + } + if (command.startsWith('mv ')) { + const match = command.match(/'([^']*)'/) + if (match) { + options.removed.push(match[1].split('/').pop() ?? '') + } + return 'MOVED' + } + return '' + }) +} + +describe('orcad GC', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('scopes its remote listing to the orcad namespace', async () => { + const removed: string[] = [] + scriptHost({ listing: [], removed }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb', + record: emptyOrcadActivationRecord() + }) + const listCommand = mockExec.mock.calls.map((call) => String(call[1])).find((c) => c.includes('find')) + expect(listCommand).toContain("-name 'orcad-*'") + expect(listCommand).not.toContain("-name 'relay-*'") + }) + + it('leaves relay directories alone even when the host hands them to it', async () => { + // The host is lying — or a future listing bug widened the glob. GC must still refuse. + const removed: string[] = [] + scriptHost({ + listing: ['relay-0.1.0+aa', 'relay-0.0.9+ff', 'orcad-0.1.0+aa'], + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb', + record: emptyOrcadActivationRecord() + }) + expect(removed).toEqual(['orcad-0.1.0+aa']) + }) + + it('never removes the active or the previous version', async () => { + const removed: string[] = [] + scriptHost({ + listing: ['orcad-0.1.0+01d', 'orcad-0.2.0+9ee0', 'orcad-0.3.0+cc0'], + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: { + ...emptyOrcadActivationRecord(), + active: '0.3.0+cc0', + previous: '0.2.0+9ee0' + } + }) + expect(removed).toEqual(['orcad-0.1.0+01d']) + }) + + it('never removes the version a live daemon was forked from', async () => { + const removed: string[] = [] + scriptHost({ listing: ['orcad-0.1.0+01d', 'orcad-0.0.9+01de'], removed }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: { ...emptyOrcadActivationRecord(), active: '0.3.0+cc0' }, + liveDaemonVersion: '0.1.0+01d' + }) + expect(removed).toEqual(['orcad-0.0.9+01de']) + }) + + it('treats an unanswerable liveness probe as in use', async () => { + const removed: string[] = [] + scriptHost({ + listing: ['orcad-0.1.0+bb0', 'orcad-0.0.9+dead'], + liveness: { 'orcad-0.1.0+bb0': 'UNKNOWN', 'orcad-0.0.9+dead': 'DEAD' }, + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: emptyOrcadActivationRecord() + }) + expect(removed).toEqual(['orcad-0.0.9+dead']) + }) +}) diff --git a/src/main/ssh/orcad-remote-gc.ts b/src/main/ssh/orcad-remote-gc.ts new file mode 100644 index 000000000000..a96b386fc1bb --- /dev/null +++ b/src/main/ssh/orcad-remote-gc.ts @@ -0,0 +1,74 @@ +/** + * orcad's garbage collection, and the half of §06 falsifier 1 that says who owns it. + * + * **Each model GCs only its own namespace, permanently.** orcad removes `orcad-/` + * directories; the relay removes `relay-/` directories; neither ever removes the other's, + * and no plan item makes one the winner. That is not a migration compromise — the two models + * serve different users on the same machine (SSH target vs paired peer), so there is no + * moment at which one of them is entitled to clean up after the other. A pass that deleted + * the sibling's tree would be reaching across the execution boundary the whole design exists + * to keep intact. + * + * On top of the ownership rule, orcad pins three directories that are idle-looking but + * load-bearing: the active version, the rollback target, and whichever version the LIVE + * terminal daemon was forked from. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { gcOldRemoteInstallVersions } from './ssh-relay-versioned-install' +import { orcadGcPinnedDirNames, type OrcadActivationRecord } from './orcad-activation-record' +import { + orcadLivenessBlocksGc, + orcadLivenessProbeCommand, + parseOrcadLiveness +} from './orcad-remote-launch' +import type { RemoteHostPlatform } from './ssh-remote-platform' + +export type OrcadGcOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + /** Absolute path of the version dir this client just used; never a candidate. */ + currentDirAbsPath: string + record: OrcadActivationRecord + /** + * The full version the live daemon's PID record names, when it can be read. + * + * An update preserves a daemon forked from the OUTGOING bundle whenever terminals are + * live, so this is routinely a version that is neither active nor previous. Deleting it + * would remove the tree under a running process. + */ + liveDaemonVersion?: string | null + signal?: AbortSignal +} + +export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise { + await gcOldRemoteInstallVersions( + options.conn, + ORCAD_INSTALL_MODEL, + options.remoteHome, + options.currentDirAbsPath, + options.host, + { + pinnedDirNames: orcadGcPinnedDirNames(options.record, options.liveDaemonVersion), + isDirLive: async (dir) => { + try { + const probe = await execCommand( + options.conn, + orcadLivenessProbeCommand(options.host, dir), + { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + } + ) + return orcadLivenessBlocksGc(parseOrcadLiveness(probe)) + } catch { + // Why true: an unanswered probe is not evidence a tree is idle. Same rule the + // relay's socket probe applies, for the same reason. + return true + } + } + } + ) +} diff --git a/src/main/ssh/orcad-remote-host-support.ts b/src/main/ssh/orcad-remote-host-support.ts new file mode 100644 index 000000000000..dfcb0f4b6e4d --- /dev/null +++ b/src/main/ssh/orcad-remote-host-support.ts @@ -0,0 +1,50 @@ +/** + * Which hosts the orcad launch/lifecycle path actually supports, declared rather than + * discovered at runtime. + * + * The install transaction is host-agnostic — it is the relay's, and the relay runs on + * Windows. The launch, liveness and stop path is not: it uses `nohup`, a redirected stdout, + * `kill -0` and `ps`. Emitting a PowerShell-shaped approximation of that would produce a + * deploy that reports success on a host where nothing is running. + */ +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' + +export class OrcadRemoteLaunchUnsupportedError extends Error { + readonly code = 'orcad_remote_launch_unsupported_host' + constructor(hostLabel: string) { + super( + `Deploying orcad to a ${hostLabel} host is not implemented. The install transaction is ` + + 'host-agnostic, but the launch and readiness path is POSIX-only: it uses nohup, a ' + + 'redirected stdout and `kill -0` liveness. Use the relay for this host.' + ) + this.name = 'OrcadRemoteLaunchUnsupportedError' + } +} + +export function assertPosixOrcadHost(host: RemoteHostPlatform): void { + if (isWindowsRemoteHost(host)) { + throw new OrcadRemoteLaunchUnsupportedError('Windows') + } +} + +/** PID of the launched orcad, written into its own version dir at launch. */ +export const ORCAD_PID_FILENAME = '.orcad-pid' + +/** + * A shell function answering whether a PID is a *running* process. + * + * `kill -0` alone is not that question. It succeeds for a zombie — a process that has + * exited but whose parent has not reaped it — so a stop loop built on it reports + * `STILL_RUNNING` for a process that is already gone, and GC reports a dead version dir as + * in use. Verified against a real zombie on macOS; the `ps` state check is what separates + * the two. + * + * A host without `ps` yields an empty state, which falls through to "alive" — the safe + * direction for both callers. + */ +export function posixProcessAliveShellFunction(): string { + return ( + 'orcad_alive() { kill -0 "$1" 2>/dev/null || return 1; ' + + 'case "$(ps -o stat= -p "$1" 2>/dev/null)" in Z*) return 1;; esac; return 0; };' + ) +} diff --git a/src/main/ssh/orcad-remote-launch.test.ts b/src/main/ssh/orcad-remote-launch.test.ts new file mode 100644 index 000000000000..068ae588f657 --- /dev/null +++ b/src/main/ssh/orcad-remote-launch.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest' + +import { + ORCAD_READINESS_FILENAME, + orcadLaunchCommand, + orcadLivenessBlocksGc, + orcadLivenessProbeCommand, + OrcadRemoteLaunchUnsupportedError, + parseOrcadLiveness, + parseOrcadReadinessOutput +} from './orcad-remote-launch' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') + +const SPEC = { + remoteInstallDir: '/home/u/.orca-remote/orcad-0.2.0+bb01', + nodePath: '/usr/bin/node', + fullVersion: '0.2.0+bb01', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777 +} + +const READY_LINE = JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { buildHash: 'abc', terminalDaemon: { state: 'live' } } +}) + +describe('orcadLaunchCommand', () => { + it('states the bind posture rather than inheriting the build default', () => { + expect(orcadLaunchCommand(posix, SPEC)).toContain("--bind '127.0.0.1'") + }) + + it('truncates the readiness file, so a stale line cannot be activated on', () => { + const command = orcadLaunchCommand(posix, SPEC) + const truncate = command.indexOf(`: > '${SPEC.remoteInstallDir}/${ORCAD_READINESS_FILENAME}'`) + const launch = command.indexOf('nohup') + expect(truncate).toBeGreaterThan(-1) + expect(truncate).toBeLessThan(launch) + }) + + it('exports the version and the shared data root the deploy decided on', () => { + const command = orcadLaunchCommand(posix, SPEC) + expect(command).toContain(`ORCA_VERSION '${SPEC.fullVersion}'`.replace(' ', '=')) + expect(command).toContain(`ORCA_USER_DATA='${SPEC.userDataDir}'`) + }) + + it('declares the Windows refusal instead of emitting a command that cannot work', () => { + expect(() => orcadLaunchCommand(windows, SPEC)).toThrow(OrcadRemoteLaunchUnsupportedError) + }) +}) + +describe('readiness parsing', () => { + it('extracts the orca_server_ready payload', () => { + const parsed = parseOrcadReadinessOutput(`${READY_LINE}\n`) + expect(parsed).toMatchObject({ state: 'ready' }) + expect(parsed.state === 'ready' && parsed.readiness.boundEndpoint).toBe('ws://127.0.0.1:7777') + }) + + it('treats an empty or half-written file as pending, not as a failure', () => { + expect(parseOrcadReadinessOutput('')).toEqual({ state: 'pending' }) + expect(parseOrcadReadinessOutput('{"type":"orca_serv')).toEqual({ state: 'pending' }) + }) + + it('reports a complete JSON line that is not a readiness payload as malformed', () => { + expect(parseOrcadReadinessOutput('{"type":"something_else"}')).toMatchObject({ + state: 'malformed' + }) + }) +}) + +describe('liveness', () => { + it('reads the pid recorded in the version dir', () => { + expect(orcadLivenessProbeCommand(posix, SPEC.remoteInstallDir)).toContain('.orcad-pid') + }) + + it.each([ + ['LIVE', 'LIVE', true], + ['DEAD', 'DEAD', false], + ['', 'UNKNOWN', true], + ['garbage', 'UNKNOWN', true] + ])('parses %s and blocks GC = %s', (output, expected, blocks) => { + expect(parseOrcadLiveness(output)).toBe(expected) + expect(orcadLivenessBlocksGc(parseOrcadLiveness(output))).toBe(blocks) + }) +}) + +describe('stopping a running orcad', () => { + it('sends SIGTERM and never SIGKILL', () => { + const command = stopOrcadCommand(posix, SPEC.remoteInstallDir, { waitSeconds: 20 }) + expect(command).toContain('kill -TERM') + for (const kill of ['kill -9', 'kill -KILL', 'kill -SIGKILL', 'pkill']) { + expect(command).not.toContain(kill) + } + }) + + it.each([ + ['STOPPED', 'stopped', true], + ['ALREADY_EXITED', 'already-exited', true], + ['NO_PID', 'no-pid', true], + ['STILL_RUNNING', 'still-running', false], + ['SIGNAL_FAILED', 'signal-failed', false], + ['', 'unknown', false] + ])('parses %s and frees the host = %s', (output, expected, frees) => { + expect(parseOrcadStopOutcome(output)).toBe(expected) + expect(orcadStopFreedTheHost(parseOrcadStopOutcome(output))).toBe(frees) + }) +}) diff --git a/src/main/ssh/orcad-remote-launch.ts b/src/main/ssh/orcad-remote-launch.ts new file mode 100644 index 000000000000..e82be95faec4 --- /dev/null +++ b/src/main/ssh/orcad-remote-launch.ts @@ -0,0 +1,178 @@ +/** + * Starting a candidate orcad on the host and reading back what it says about itself. + * + * This is the piece `docs/design/shipping-orcad.html` §02 marks **fork**, not reuse: the + * relay launches detached and proves itself by printing an `ORCA-RELAY` sentinel, and orcad + * has no such interface. It publishes a single `orca_server_ready` JSON line on stdout, + * carrying the health payload activation is gated on — so the handshake here is "capture + * that line", not "match a marker". + * + * The candidate is launched detached with stdout redirected to a file inside its own version + * directory. Reading readiness off the exec channel would mean holding the channel open for + * the process's whole life; redirecting means the deploy can disconnect and the supervisor + * still owns a running service. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + assertPosixOrcadHost as assertPosixHost, + ORCAD_PID_FILENAME, + posixProcessAliveShellFunction +} from './orcad-remote-host-support' +import type { ServeReadiness } from '../server/serve-readiness' + +/** Stdout of the launched candidate: exactly one `orca_server_ready` line, then nothing. */ +export const ORCAD_READINESS_FILENAME = '.orcad-readiness' +/** Stderr, including the bind-exposure line and every supervision message. */ +export const ORCAD_LOG_FILENAME = 'orcad.log' +export { + ORCAD_PID_FILENAME, + OrcadRemoteLaunchUnsupportedError +} from './orcad-remote-host-support' + +export type OrcadLaunchSpec = { + remoteInstallDir: string + nodePath: string + fullVersion: string + /** Shared across versions, and the reason rollback needs a snapshot. */ + userDataDir: string + /** Loopback by default; the client reaches it through an SSH local port-forward. */ + bindHost: string + port: number +} + +/** + * Launch the candidate detached and echo its PID. + * + * Why `--bind` is always passed explicitly: orcad defaults to loopback, but a default is a + * thing a future version can change. The deploy states the posture it intends rather than + * inheriting whatever the installed build happens to default to. + */ +export function orcadLaunchCommand(host: RemoteHostPlatform, spec: OrcadLaunchSpec): string { + assertPosixHost(host) + const dir = shellEscape(spec.remoteInstallDir) + const readiness = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_READINESS_FILENAME)) + const log = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_LOG_FILENAME)) + const pidFile = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_PID_FILENAME)) + const entry = shellEscape(joinRemotePath(host, spec.remoteInstallDir, 'orcad.js')) + return [ + `cd ${dir} &&`, + // Why truncate: a re-launch into a dir that already holds a previous readiness line would + // otherwise let the deploy activate on the OLD process's health payload. + `: > ${readiness} &&`, + 'umask 077 &&', + `ORCA_VERSION=${shellEscape(spec.fullVersion)}`, + `ORCA_USER_DATA=${shellEscape(spec.userDataDir)}`, + `nohup ${shellEscape(spec.nodePath)} ${entry}`, + `--json --bind ${shellEscape(spec.bindHost)} --port ${String(spec.port)}`, + `> ${readiness} 2>> ${log} < /dev/null &`, + `echo $! > ${pidFile} && cat ${pidFile}` + ].join(' ') +} + +export function readOrcadReadinessCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + assertPosixHost(host) + const readiness = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME)) + return `cat ${readiness} 2>/dev/null || true` +} + +/** + * Is the process recorded in this version dir still running? + * + * Answers `LIVE`, `DEAD`, or `UNKNOWN`. `UNKNOWN` covers a missing or unparseable PID file + * and a `kill -0` that failed for a reason other than "no such process" — a permission + * error means someone else's process holds that PID, which is not evidence of death. + */ +export function orcadLivenessProbeCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + assertPosixHost(host) + const pidFile = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_PID_FILENAME)) + return [ + posixProcessAliveShellFunction(), + `pid=$(cat ${pidFile} 2>/dev/null);`, + 'case "$pid" in', + '"" ) echo UNKNOWN;;', + '*[!0-9]* ) echo UNKNOWN;;', + // Why EPERM is LIVE and not DEAD: a permission error means some process holds that PID, + // and deleting a tree because we could not signal its owner is the wrong direction. + '* ) if orcad_alive "$pid"; then echo LIVE;', + 'elif kill -0 "$pid" 2>&1 | grep -qi "not permitted"; then echo LIVE;', + 'else echo DEAD; fi;;', + 'esac' + ].join(' ') +} + +export type OrcadLiveness = 'LIVE' | 'DEAD' | 'UNKNOWN' + +export function parseOrcadLiveness(output: string): OrcadLiveness { + const value = output.trim().split('\n').pop()?.trim() + return value === 'LIVE' || value === 'DEAD' ? value : 'UNKNOWN' +} + +/** True when GC must leave this directory alone. Inconclusive counts as in use. */ +export function orcadLivenessBlocksGc(liveness: OrcadLiveness): boolean { + return liveness !== 'DEAD' +} + +export type OrcadReadinessParse = + | { state: 'ready'; readiness: ServeReadiness } + | { state: 'pending' } + | { state: 'malformed'; reason: string } + +/** + * Pull the `orca_server_ready` payload out of whatever the candidate has written so far. + * + * Why scan for the type tag rather than parsing the last line: stdout is a file being + * appended to, so a poll can catch a half-written line. A partial JSON line is `pending`, + * not `malformed` — reporting a parse failure for a race would fail deploys that were fine. + */ +export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse { + const lines = raw.split('\n') + let sawCandidate = false + for (const line of lines) { + const trimmed = line.trim() + if (!trimmed.startsWith('{')) { + continue + } + sawCandidate = true + let parsed: unknown + try { + parsed = JSON.parse(trimmed) + } catch { + continue + } + if (typeof parsed !== 'object' || parsed === null) { + continue + } + const payload = parsed as { type?: unknown } + if (payload.type !== 'orca_server_ready') { + return { + state: 'malformed', + reason: `expected an orca_server_ready line, got type=${JSON.stringify(payload.type)}` + } + } + return { state: 'ready', readiness: toServeReadiness(payload as Record) } + } + return sawCandidate ? { state: 'pending' } : { state: 'pending' } +} + +function toServeReadiness(payload: Record): ServeReadiness { + return { + runtimeId: typeof payload.runtimeId === 'string' ? payload.runtimeId : '', + boundEndpoint: typeof payload.boundEndpoint === 'string' ? payload.boundEndpoint : null, + advertisedEndpoint: + typeof payload.advertisedEndpoint === 'string' ? payload.advertisedEndpoint : null, + managedWslCliReconciliation: + payload.managedWslCliReconciliation === 'pending' || + payload.managedWslCliReconciliation === 'failed' + ? payload.managedWslCliReconciliation + : 'settled', + pairing: payload.pairing as ServeReadiness['pairing'], + ...(payload.health ? { health: payload.health as ServeReadiness['health'] } : {}) + } +} diff --git a/src/main/ssh/orcad-remote-process-control.ts b/src/main/ssh/orcad-remote-process-control.ts new file mode 100644 index 000000000000..6a9038ea3d62 --- /dev/null +++ b/src/main/ssh/orcad-remote-process-control.ts @@ -0,0 +1,73 @@ +/** + * Stopping a running orcad on the host without taking its terminals with it. + * + * `SIGKILL` is absent on purpose. orcad's own escalation contract (`orcad-entry.ts`) is + * SIGTERM, then a second SIGTERM meaning "your deadline elapsed, exit now"; a kill skips the + * teardown that releases the instance lock and disconnects — rather than shuts down — the + * terminal daemon. The daemon is detached and would survive a kill, but a stop that leaves + * the lock file behind makes the successor refuse to start with + * `orcad_data_root_shared`, so the update turns into an outage for no gain. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + assertPosixOrcadHost as assertPosixHost, + ORCAD_PID_FILENAME, + posixProcessAliveShellFunction +} from './orcad-remote-host-support' + +/** + * Signal the orcad recorded in a version dir and wait for it to go. + * + * `escalate` sends the second SIGTERM orcad reads as "exit immediately". Callers use it only + * after the first deadline elapses, so the two signals are never in the same command. + */ +export function stopOrcadCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + options: { waitSeconds: number } +): string { + assertPosixHost(host) + const pidFile = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_PID_FILENAME)) + return [ + posixProcessAliveShellFunction(), + `pid=$(cat ${pidFile} 2>/dev/null);`, + 'case "$pid" in "" | *[!0-9]* ) echo NO_PID; exit 0;; esac;', + 'orcad_alive "$pid" || { echo ALREADY_EXITED; exit 0; };', + 'kill -TERM "$pid" 2>/dev/null || { echo SIGNAL_FAILED; exit 0; };', + `i=0; while [ "$i" -lt ${options.waitSeconds} ]; do`, + 'orcad_alive "$pid" || { echo STOPPED; exit 0; };', + 'sleep 1; i=$((i + 1)); done;', + 'echo STILL_RUNNING' + ].join(' ') +} + +export type OrcadStopOutcome = + | 'stopped' + | 'already-exited' + | 'no-pid' + | 'still-running' + | 'signal-failed' + | 'unknown' + +export function parseOrcadStopOutcome(output: string): OrcadStopOutcome { + switch (output.trim().split('\n').pop()?.trim() ?? '') { + case 'STOPPED': + return 'stopped' + case 'ALREADY_EXITED': + return 'already-exited' + case 'NO_PID': + return 'no-pid' + case 'STILL_RUNNING': + return 'still-running' + case 'SIGNAL_FAILED': + return 'signal-failed' + default: + return 'unknown' + } +} + +/** True when the port is free and a successor may bind. */ +export function orcadStopFreedTheHost(outcome: OrcadStopOutcome): boolean { + return outcome === 'stopped' || outcome === 'already-exited' || outcome === 'no-pid' +} diff --git a/src/main/ssh/orcad-remote-rollback.test.ts b/src/main/ssh/orcad-remote-rollback.test.ts new file mode 100644 index 000000000000..59f3d46e330d --- /dev/null +++ b/src/main/ssh/orcad-remote-rollback.test.ts @@ -0,0 +1,206 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-transfers', () => ({ + writeRelayFile: vi.fn().mockResolvedValue(undefined), + uploadRelayDirectory: vi.fn().mockResolvedValue(undefined) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { writeRelayFile } from './ssh-relay-install-transfers' +import { rollbackOrcad, type OrcadRollbackOptions } from './orcad-remote-rollback' +import { + emptyOrcadActivationRecord, + type OrcadActivationRecord +} from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const mockExec = vi.mocked(execCommand) +const ACTIVE = '0.2.0+bb01' +const TARGET = '0.1.0+aa01' +const BUILD_HASH = 'abc123def4567890' + +function record(overrides: Partial = {}): OrcadActivationRecord { + return { + ...emptyOrcadActivationRecord(), + active: ACTIVE, + previous: TARGET, + activatedAt: '2026-01-01T00:00:00.000Z', + snapshot: { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: ACTIVE, + readableByVersion: TARGET, + takenAt: '2026-01-01T00:00:00.000Z' + }, + ...overrides + } +} + +function readyLine(version: string): string { + return JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: BUILD_HASH, + buildVersion: version, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 2, + buildVersion: version, + entryPath: '/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 5 } + } + } + }) +} + +function scriptHost(log: string[], overrides: { restore?: string } = {}): void { + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { + return 'PRESENT' + } + if (text.includes('find ') && text.includes('stat')) { + return 'UNKNOWN' + } + if (text.includes('kill -TERM')) { + log.push(`stop:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-xf')) { + log.push('restore') + return overrides.restore ?? 'RESTORED' + } + if (text.includes('nohup')) { + log.push(`launch:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) + return '9999' + } + if (text.startsWith('cat ') && text.includes('.orcad-readiness')) { + return readyLine(TARGET) + } + return '' + }) +} + +function options(overrides: Partial = {}): OrcadRollbackOptions { + return { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + record: record(), + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + census: { liveSessions: 0, startedSinceActivation: 0 }, + targetBuildHash: BUILD_HASH, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z'), + ...overrides + } +} + +describe('rollbackOrcad', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('stops, restores state, then starts the target — in that order', async () => { + const log: string[] = [] + scriptHost(log) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'rolled-back', target: TARGET }) + // Restoring under a running orcad would replace the store beneath a process holding it; + // starting first would let the older build migrate the newer build's state. + expect(log).toEqual([`stop:${ACTIVE}`, 'restore', `launch:${TARGET}`]) + }) + + it('refuses before touching anything when terminals started after activation', async () => { + const log: string[] = [] + scriptHost(log) + const result = await rollbackOrcad( + options({ census: { liveSessions: 3, startedSinceActivation: 2 } }) + ) + expect(result).toMatchObject({ + outcome: 'refused', + code: 'orcad_rollback_orphans_live_terminals' + }) + expect(log).toEqual([]) + expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + }) + + it('refuses when the snapshot is gone from the host', async () => { + const log: string[] = [] + mockExec.mockImplementation(async (_conn, command: string) => + String(command).includes('state.tar') ? 'ABSENT' : '' + ) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'refused', code: 'orcad_rollback_snapshot_missing' }) + expect(log).toEqual([]) + }) + + it('does not start the old build when the restore failed', async () => { + const log: string[] = [] + scriptHost(log, { restore: 'FAILED' }) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_rollback_restore_failed' }) + expect(log).toEqual([`stop:${ACTIVE}`, 'restore']) + expect(result.outcome === 'failed' && result.reason).toContain('Do NOT start the older build') + }) + + it('leaves the record naming the newer version when the target fails to come up', async () => { + const log: string[] = [] + scriptHost(log) + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { + return 'PRESENT' + } + if (text.includes('kill -TERM')) { + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-xf')) { + return 'RESTORED' + } + // The target never publishes readiness. + return '' + }) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' }) + // Until the target is proven serving, `active` must still name the version an operator + // would have to bring back. + expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + }) + + it('records the rollback only after the target answers healthy', async () => { + const log: string[] = [] + scriptHost(log) + await rollbackOrcad(options()) + const written = vi + .mocked(writeRelayFile) + .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[3]))).toMatchObject({ + active: TARGET, + previous: null, + snapshot: null + }) + }) +}) diff --git a/src/main/ssh/orcad-remote-rollback.ts b/src/main/ssh/orcad-remote-rollback.ts new file mode 100644 index 000000000000..165ccfc037f2 --- /dev/null +++ b/src/main/ssh/orcad-remote-rollback.ts @@ -0,0 +1,244 @@ +/** + * Going back to the previously active orcad. + * + * Rollback is a state operation, not a binary swap. The version dirs are immutable and both + * are still on disk, so pointing at the old one is trivial; what is not trivial is that both + * versions share ONE data root, outside either dir. A newer orcad migrates that root on load + * — and Orca's persisted state carries no schema version to migrate against, so the older + * build cannot be shown to read the result. Rollback therefore restores the pre-activation + * snapshot, and refuses when restoring it would orphan work (`assessOrcadRollback`). + * + * The order below is the whole safety argument: stop, then restore, then start. Restoring + * under a running orcad would replace the store beneath a process holding it open, and + * starting before restoring would let the old build migrate the new build's state — the + * failure this is meant to avoid, arrived at from the other side. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { computeRemoteInstallDir } from './ssh-relay-versioned-install' +import { writeRelayFile } from './ssh-relay-install-transfers' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_STATE_SNAPSHOT_DIR, + serializeOrcadActivationRecord, + withRolledBackVersion, + type OrcadActivationRecord +} from './orcad-activation-record' +import { assessOrcadRollback, type OrcadTerminalCensus } from './orcad-update-plan' +import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' +import { + ORCAD_LOG_FILENAME, + orcadLaunchCommand, + parseOrcadReadinessOutput, + readOrcadReadinessCommand +} from './orcad-remote-launch' +import { + newestStateMtimeCommand, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotRestore, + probeOrcadStateSnapshotCommand, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { orcadActivationPath } from './orcad-activation-record-store' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +export type OrcadRollbackOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + record: OrcadActivationRecord + nodePath: string + userDataDir: string + bindHost: string + port: number + census: OrcadTerminalCensus + /** Expected build hash of the rollback target, from the client's copy of those bytes. */ + targetBuildHash: string + readinessTimeoutMs?: number + now?: () => Date + sleep?: (ms: number) => Promise + signal?: AbortSignal +} + +export type OrcadRollbackResult = + | { outcome: 'rolled-back'; target: string; discarded: string[]; verdict: OrcadActivationVerdict } + | { outcome: 'refused'; code: string; reason: string } + | { outcome: 'failed'; code: string; reason: string } + +const DEFAULT_READINESS_TIMEOUT_MS = 90_000 +const READINESS_POLL_MS = 500 +const STOP_WAIT_SECONDS = 20 + +function exec(options: OrcadRollbackOptions, command: string): Promise { + return execCommand(options.conn, command, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + }) +} + +function snapshotDirPath(options: OrcadRollbackOptions, dirName: string): string { + return joinRemotePath( + options.host, + options.remoteHome, + RELAY_REMOTE_DIR, + ORCAD_STATE_SNAPSHOT_DIR, + dirName + ) +} + +/** Has the store been written since activation? `null` when it cannot be established. */ +async function readStateWritesSinceActivation( + options: OrcadRollbackOptions +): Promise { + if (!options.record.activatedAt) { + return null + } + const activatedAtSeconds = Math.floor(Date.parse(options.record.activatedAt) / 1000) + if (!Number.isFinite(activatedAtSeconds)) { + return null + } + const newest = parseNewestStateMtimeSeconds( + await exec(options, newestStateMtimeCommand(options.host, options.userDataDir)).catch(() => '') + ) + return newest === null ? null : newest >= activatedAtSeconds +} + +export async function rollbackOrcad( + options: OrcadRollbackOptions +): Promise { + const now = options.now ?? ((): Date => new Date()) + const snapshotPresent = options.record.snapshot + ? ( + await exec( + options, + probeOrcadStateSnapshotCommand( + options.host, + snapshotDirPath(options, options.record.snapshot.dirName) + ) + ).catch(() => 'ABSENT') + ).trim() === 'PRESENT' + : false + + const safety = assessOrcadRollback({ + record: options.record, + snapshotPresent, + census: options.census, + stateWritesSinceActivation: await readStateWritesSinceActivation(options) + }) + if (safety.safety === 'unsafe') { + return { outcome: 'refused', code: safety.code, reason: safety.reason } + } + + if (options.record.active) { + const outgoingDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + options.record.active + ) + const stopped = parseOrcadStopOutcome( + await exec( + options, + stopOrcadCommand(options.host, outgoingDir, { waitSeconds: STOP_WAIT_SECONDS }) + ) + ) + if (!orcadStopFreedTheHost(stopped)) { + return { + outcome: 'failed', + code: 'orcad_rollback_stop_incomplete', + reason: + `orcad ${options.record.active} did not exit within ${STOP_WAIT_SECONDS}s of SIGTERM ` + + `(${stopped}). Nothing was restored — the store is untouched and the host is still ` + + 'serving the version you tried to leave.' + } + } + } + + // Why between stop and start: the store must be replaced while no orcad holds it, and + // before the older build gets a chance to migrate the newer build's state. + const restored = parseOrcadSnapshotRestore( + await exec( + options, + restoreOrcadStateSnapshotCommand( + options.host, + options.userDataDir, + // Guarded by `assessOrcadRollback`: `unsafe` covers a missing snapshot. + snapshotDirPath(options, options.record.snapshot?.dirName ?? '') + ) + ).catch(() => 'FAILED') + ) + if (restored !== 'restored') { + return { + outcome: 'failed', + code: 'orcad_rollback_restore_failed', + reason: + `The pre-activation snapshot could not be restored (${restored}). orcad is stopped and ` + + 'the data root may be partially replaced. Do NOT start the older build against it; ' + + `re-deploy ${options.record.active ?? 'the newer version'}, which can read what is there.` + } + } + + const targetDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + safety.target + ) + await exec( + options, + orcadLaunchCommand(options.host, { + remoteInstallDir: targetDir, + nodePath: options.nodePath, + fullVersion: safety.target, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + ) + const deadline = Date.now() + (options.readinessTimeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS) + const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) + let parsed = parseOrcadReadinessOutput('') + while (Date.now() < deadline && parsed.state === 'pending') { + options.signal?.throwIfAborted() + parsed = parseOrcadReadinessOutput( + await exec(options, readOrcadReadinessCommand(options.host, targetDir)) + ) + if (parsed.state === 'pending') { + await sleep(READINESS_POLL_MS) + } + } + const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { + buildHash: options.targetBuildHash, + fullVersion: safety.target + }) + if (verdict.decision === 'reject') { + return { + outcome: 'failed', + code: verdict.code, + reason: + `The rollback target ${safety.target} did not come up healthy: ${verdict.reason} The ` + + `store has been restored to its pre-activation state. Its stderr is at ` + + `${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.` + } + } + + // Why the record is written last: until the target is proven serving, `active` still names + // the version an operator would need to bring back, and `previous` still names this target. + await writeRelayFile( + options.conn, + options.host, + orcadActivationPath(options.host, options.remoteHome), + serializeOrcadActivationRecord(withRolledBackVersion(options.record, now())), + { signal: options.signal } + ) + return { + outcome: 'rolled-back', + target: safety.target, + discarded: safety.safety === 'lossy' ? safety.discards : [], + verdict + } +} diff --git a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts new file mode 100644 index 000000000000..d1f8ac0048f7 --- /dev/null +++ b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts @@ -0,0 +1,189 @@ +/** + * Runs the generated POSIX commands through a real `/bin/sh`. + * + * The unit tests assert on command *text*, which is exactly the kind of test that stays + * green while the shell it produces does not work — a quoting slip, a `case` pattern that + * never matches, a `tar` invocation that silently captures nothing. These run the strings. + */ +import { execFileSync, spawn } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + orcadLivenessProbeCommand, + ORCAD_PID_FILENAME, + parseOrcadLiveness +} from './orcad-remote-launch' +import { parseOrcadStopOutcome, stopOrcadCommand } from './orcad-remote-process-control' +import { + captureOrcadStateSnapshotCommand, + newestStateMtimeCommand, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotCapture, + parseOrcadSnapshotRestore, + probeOrcadStateSnapshotCommand, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const host = getRemoteHostPlatform('linux-x64') +let root = '' +let dataDir = '' +let snapshotDir = '' +let versionDir = '' + +function sh(command: string): string { + return execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' }) +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orcad-shell-')) + dataDir = join(root, '.orca') + snapshotDir = join(root, 'snapshots', 'pre-0.2.0+bb01-1000') + versionDir = join(root, '.orca-remote', 'orcad-0.2.0+bb01') + mkdirSync(join(dataDir, 'profiles', 'p1'), { recursive: true }) + mkdirSync(join(dataDir, 'daemon'), { recursive: true }) + mkdirSync(versionDir, { recursive: true }) + writeFileSync(join(dataDir, 'orca-profile-index.json'), '{"v":"before"}') + writeFileSync(join(dataDir, 'profiles', 'p1', 'orca-data.json'), '{"repos":"before"}') + writeFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'live-daemon-token') +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) +}) + +describe('state snapshot commands, run for real', () => { + it('captures, then restores state the newer build overwrote', () => { + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + ).toBe('captured') + expect(sh(probeOrcadStateSnapshotCommand(host, snapshotDir)).trim()).toBe('PRESENT') + + // The new version migrates the store and adds a file of its own. + writeFileSync(join(dataDir, 'orca-profile-index.json'), '{"v":"migrated"}') + writeFileSync(join(dataDir, 'profiles', 'p1', 'new-build-only.json'), '{}') + + expect( + parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + ).toBe('restored') + expect(readFileSync(join(dataDir, 'orca-profile-index.json'), 'utf8')).toBe('{"v":"before"}') + // Removed before extraction, so the older build never sees a file it cannot interpret. + expect(() => readFileSync(join(dataDir, 'profiles', 'p1', 'new-build-only.json'))).toThrow() + }) + + it('leaves the live daemon runtime dir untouched through capture and restore', () => { + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + // The daemon is running across the rollback and rewrites its token; a restore that + // reached /daemon would break the fence that keeps its terminals adoptable. + writeFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'token-after-restart') + sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + expect(readFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'utf8')).toBe( + 'token-after-restart' + ) + }) + + it('reports EMPTY on a data root with nothing to lose, instead of an archive of nothing', () => { + const emptyRoot = join(root, 'fresh') + mkdirSync(emptyRoot) + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, emptyRoot, snapshotDir))) + ).toBe('empty') + expect(sh(probeOrcadStateSnapshotCommand(host, snapshotDir)).trim()).toBe('ABSENT') + }) + + it('reports MISSING rather than claiming a restore it did not perform', () => { + expect( + parseOrcadSnapshotRestore( + sh(restoreOrcadStateSnapshotCommand(host, dataDir, join(root, 'nope'))) + ) + ).toBe('missing') + }) + + it('reads a real mtime for the store', () => { + const seconds = parseNewestStateMtimeSeconds(sh(newestStateMtimeCommand(host, dataDir))) + expect(seconds).toBeGreaterThan(1_600_000_000) + expect(seconds).toBeLessThanOrEqual(Math.floor(Date.now() / 1000) + 5) + }) + + it('survives a data root whose path contains a quote and a space', () => { + const nasty = join(root, `it's a dir`) + mkdirSync(join(nasty, 'profiles'), { recursive: true }) + writeFileSync(join(nasty, 'orca-profile-index.json'), '{"v":"quoted"}') + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, nasty, snapshotDir))) + ).toBe('captured') + writeFileSync(join(nasty, 'orca-profile-index.json'), '{"v":"changed"}') + expect( + parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, nasty, snapshotDir))) + ).toBe('restored') + expect(readFileSync(join(nasty, 'orca-profile-index.json'), 'utf8')).toBe('{"v":"quoted"}') + }) +}) + +describe('liveness and stop commands, run for real', () => { + it('reports UNKNOWN with no pid file, and DEAD for a pid that has exited', () => { + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('UNKNOWN') + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), 'not-a-pid') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('UNKNOWN') + // A pid that has certainly exited: our own `sh` child from the line above. + const exited = Number(sh('sh -c "echo $$"').trim()) + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(exited)) + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + }) + + it('reports LIVE for a running process and stops it with SIGTERM', async () => { + const child = spawn('/bin/sh', ['-c', 'sleep 30'], { stdio: 'ignore' }) + try { + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(child.pid)) + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('LIVE') + + const exited = new Promise((resolve) => + child.once('exit', (_code, signal) => resolve(signal)) + ) + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 10 })))).toBe( + 'stopped' + ) + expect(await exited).toBe('SIGTERM') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + } finally { + child.kill('SIGKILL') + } + }) + + // `kill -0` succeeds on a zombie, so a probe built on it alone calls an exited process + // live: the stop loop would time out on a process that is already gone, and GC would keep + // a dead version dir forever. Verified as a real macOS behaviour, not a hypothetical. + it('reports a zombie as DEAD, not as a running process', () => { + const child = spawn('/bin/sh', ['-c', 'exit 0'], { stdio: 'ignore' }) + try { + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(child.pid)) + // Block the event loop so Node never reaps it; the process is now a zombie. + sh('sleep 1') + expect(sh(`ps -o stat= -p ${child.pid} || echo GONE`).trim()).toMatch(/^Z/) + expect(sh(`kill -0 ${child.pid} 2>/dev/null && echo LIVE || echo DEAD`).trim()).toBe('LIVE') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + expect( + parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 }))) + ).toBe('already-exited') + } finally { + child.unref() + } + }) + + it('reports ALREADY_EXITED for a stale pid file rather than signalling a stranger', () => { + const exited = Number(sh('sh -c "echo $$"').trim()) + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(exited)) + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 })))).toBe( + 'already-exited' + ) + }) + + it('reports NO_PID when the version dir was never launched', () => { + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 })))).toBe( + 'no-pid' + ) + }) +}) diff --git a/src/main/ssh/orcad-state-snapshot.test.ts b/src/main/ssh/orcad-state-snapshot.test.ts new file mode 100644 index 000000000000..0c50dc55fdbe --- /dev/null +++ b/src/main/ssh/orcad-state-snapshot.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest' + +import { + ORCAD_SNAPSHOT_EXCLUDED, + ORCAD_SNAPSHOT_MEMBERS, + captureOrcadStateSnapshotCommand, + newestStateMtimeCommand, + orcadSnapshotDirName, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotCapture, + parseOrcadSnapshotRestore, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') +const ROOT = '/home/u/.orca' +const SNAP = '/home/u/.orca-remote/orcad-state-snapshots/pre-0.2.0+bb01-1000' + +describe('capturing the pre-activation snapshot', () => { + it('captures the profile state a rollback needs', () => { + const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP) + for (const member of ORCAD_SNAPSHOT_MEMBERS) { + expect(command).toContain(`'${member}'`) + } + }) + + // The live daemon owns /daemon and outlives every restart. Restoring a stale copy of + // its socket, PID record and token would break the fence that keeps its terminals adoptable. + it.each(ORCAD_SNAPSHOT_EXCLUDED)('never captures %s', (excluded) => { + expect(captureOrcadStateSnapshotCommand(posix, ROOT, SNAP)).not.toContain(`'${excluded}'`) + }) + + it.each(ORCAD_SNAPSHOT_EXCLUDED)('never removes or restores over %s', (excluded) => { + expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP)).not.toContain(`'${excluded}'`) + }) + + it('writes the archive under a temp name and renames, so a killed deploy leaves no torn tar', () => { + const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP) + expect(command).toContain('.partial') + expect(command.indexOf('tar -C')).toBeLessThan(command.indexOf('mv ')) + }) + + it.each([ + ['CAPTURED', 'captured'], + ['EMPTY', 'empty'], + ['tar: broken', 'failed'], + ['', 'failed'] + ])('parses %s as %s', (output, expected) => { + expect(parseOrcadSnapshotCapture(output)).toBe(expected) + }) + + it('keys the snapshot dir on both version and time, so a retry cannot overwrite one', () => { + expect(orcadSnapshotDirName('0.2.0+bb01', 1000)).not.toBe( + orcadSnapshotDirName('0.2.0+bb01', 2000) + ) + }) +}) + +describe('restoring the snapshot', () => { + it('clears the members before extracting, so files the new build added do not survive', () => { + const command = restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP) + expect(command.indexOf('rm -rf')).toBeLessThan(command.indexOf('tar -C')) + }) + + it('reports a missing archive instead of extracting nothing and claiming success', () => { + expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP)).toContain('echo MISSING') + expect(parseOrcadSnapshotRestore('MISSING')).toBe('missing') + expect(parseOrcadSnapshotRestore('RESTORED')).toBe('restored') + expect(parseOrcadSnapshotRestore('FAILED')).toBe('failed') + }) +}) + +describe('detecting writes since activation', () => { + it.each([ + ['1700000000', 1_700_000_000], + ['UNKNOWN', null], + ['', null] + ])('parses %s', (output, expected) => { + expect(parseNewestStateMtimeSeconds(output)).toBe(expected) + }) + + it('looks at the same members the snapshot covers', () => { + const command = newestStateMtimeCommand(posix, ROOT) + for (const member of ORCAD_SNAPSHOT_MEMBERS) { + expect(command).toContain(`'${member}'`) + } + }) +}) + +describe('Windows hosts', () => { + it.each([ + ['capture', () => captureOrcadStateSnapshotCommand(windows, ROOT, SNAP)], + ['restore', () => restoreOrcadStateSnapshotCommand(windows, ROOT, SNAP)], + ['mtime', () => newestStateMtimeCommand(windows, ROOT)] + ])('refuses %s rather than emitting a POSIX command', (_label, build) => { + expect(build).toThrow('orcad to a Windows host is not implemented') + }) +}) diff --git a/src/main/ssh/orcad-state-snapshot.ts b/src/main/ssh/orcad-state-snapshot.ts new file mode 100644 index 000000000000..78ad8b47d2ac --- /dev/null +++ b/src/main/ssh/orcad-state-snapshot.ts @@ -0,0 +1,173 @@ +/** + * The pre-activation copy of shared profile state that makes rollback sound. + * + * `docs/design/shipping-orcad.html` §04's state-schema row asks for "backward-readable + * migrations or a pre-activation snapshot". Only the second is available here, and not as a + * preference: Orca's persisted state carries **no schema version**. Migrations are cohort + * and shape heuristics that run on load and rewrite in place, and the load path rebuilds + * `settings` and `ui` from known fields — so a newer build's nested additions are silently + * dropped by an older one rather than rejected. There is nothing to compare and nothing that + * fails loudly, which rules out proving backward-readability and leaves the snapshot. + * + * What is snapshotted is deliberately narrow. `/daemon` is EXCLUDED: it holds the live + * daemon's socket, PID record and auth token, and that daemon outlives every orcad restart + * by design. Restoring a stale copy of it over a running daemon would break the endpoint + * fence that keeps its terminals adoptable — turning a rollback into the exact terminal + * massacre the daemon exists to prevent. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { assertPosixOrcadHost as assertPosixHost } from './orcad-remote-host-support' + +/** + * Root-relative paths a rollback needs restored. Everything else under the data root is + * either regenerable, or owned by a process that survives the rollback. + */ +export const ORCAD_SNAPSHOT_MEMBERS = [ + 'orca-profile-index.json', + // Pre-profiles layout; still read as a migration source. + 'orca-data.json', + 'profiles' +] as const + +/** Never captured and never restored — see the module comment. */ +export const ORCAD_SNAPSHOT_EXCLUDED = ['daemon', 'logs'] as const + +/** + * The member names go into the command unquoted (see `captureOrcadStateSnapshotCommand`), so + * they must be inert. They are compile-time constants; this catches the edit that adds one + * with a space or a metacharacter in it. + */ +function assertPlainMemberName(member: string): string { + if (!/^[A-Za-z0-9._-]+$/.test(member)) { + throw new Error(`Unsafe orcad snapshot member name: ${JSON.stringify(member)}`) + } + return member +} + +export function orcadSnapshotDirName(fullVersion: string, takenAtMs: number): string { + // Why the version and the timestamp: two activations of one version (a re-deploy after a + // rejected activation) must not overwrite each other's snapshot. + return `pre-${fullVersion}-${takenAtMs}` +} + +/** + * Capture the snapshot, or report why there is nothing to capture. + * + * Prints `CAPTURED`, or `EMPTY` when the data root holds none of the members — a first-ever + * deployment, where there is no state to lose and therefore no snapshot to take. `EMPTY` is + * reported rather than fabricating an empty archive, because a rollback that "restored" an + * empty archive would wipe a root that had filled up in between. + */ +export function captureOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + userDataDir: string, + snapshotDir: string +): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const dir = shellEscape(snapshotDir) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + const memberTests = ORCAD_SNAPSHOT_MEMBERS.map( + // Why the accumulated name is NOT quoted: `$members` is re-split by the shell before it + // reaches tar, so a quoted name arrives as a literal `'profiles'` that tar cannot stat. + // `assertPlainMemberName` is what makes leaving them bare safe. + (member) => + `[ -e ${root}/${shellEscape(member)} ] && members="$members ${assertPlainMemberName(member)}";` + ).join(' ') + return [ + `members=;`, + memberTests, + 'if [ -z "$members" ]; then echo EMPTY; else', + `mkdir -p ${dir} && umask 077 &&`, + // Why a temp name then mv: a deploy killed mid-tar must not leave a truncated archive + // that a later rollback would happily restore. + `tar -C ${root} -cf ${archive}.partial $members && mv ${archive}.partial ${archive} &&`, + 'echo CAPTURED; fi' + ].join(' ') +} + +export type OrcadSnapshotCapture = 'captured' | 'empty' | 'failed' + +export function parseOrcadSnapshotCapture(output: string): OrcadSnapshotCapture { + const value = output.trim().split('\n').pop()?.trim() + if (value === 'CAPTURED') { + return 'captured' + } + return value === 'EMPTY' ? 'empty' : 'failed' +} + +export function probeOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + snapshotDir: string +): string { + assertPosixHost(host) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + return `test -f ${archive} && echo PRESENT || echo ABSENT` +} + +/** + * Restore the snapshot over the data root. + * + * Two things make this safe to run: the members are removed before extraction (so a file the + * new version added is gone rather than half-shadowed), and neither the removal nor the + * extraction can reach `/daemon`, because the member list never names it. + * + * The caller must have stopped orcad first. This does not check — it cannot, from a shell — + * so `orcad-remote-deploy.ts` owns that ordering. + */ +export function restoreOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + userDataDir: string, + snapshotDir: string +): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + const removals = ORCAD_SNAPSHOT_MEMBERS.map( + (member) => `rm -rf ${root}/${shellEscape(member)};` + ).join(' ') + return [ + `test -f ${archive} || { echo MISSING; exit 0; };`, + `test -d ${root} || mkdir -p ${root};`, + removals, + `tar -C ${root} -xf ${archive} && echo RESTORED || echo FAILED` + ].join(' ') +} + +export type OrcadSnapshotRestore = 'restored' | 'missing' | 'failed' + +export function parseOrcadSnapshotRestore(output: string): OrcadSnapshotRestore { + const value = output.trim().split('\n').pop()?.trim() + if (value === 'RESTORED') { + return 'restored' + } + return value === 'MISSING' ? 'missing' : 'failed' +} + +/** + * Has the shared store been written since `activatedAt`? + * + * Prints the newest mtime (epoch seconds) across the snapshot members, or `UNKNOWN`. The + * caller compares; an `UNKNOWN` becomes `null`, which `assessOrcadRollback` treats as "yes, + * assume writes". + */ +export function newestStateMtimeCommand(host: RemoteHostPlatform, userDataDir: string): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const paths = ORCAD_SNAPSHOT_MEMBERS.map((member) => `${root}/${shellEscape(member)}`).join(' ') + return [ + `newest=$(find ${paths} -type f -exec stat -c %Y {} + 2>/dev/null ||`, + `find ${paths} -type f -exec stat -f %m {} + 2>/dev/null);`, + 'if [ -z "$newest" ]; then echo UNKNOWN; else', + `echo "$newest" | sort -n | tail -1; fi` + ].join(' ') +} + +export function parseNewestStateMtimeSeconds(output: string): number | null { + const value = output.trim().split('\n').pop()?.trim() + if (!value || !/^\d+$/.test(value)) { + return null + } + return Number.parseInt(value, 10) +} diff --git a/src/main/ssh/orcad-update-plan.test.ts b/src/main/ssh/orcad-update-plan.test.ts new file mode 100644 index 000000000000..5d90c597e3c2 --- /dev/null +++ b/src/main/ssh/orcad-update-plan.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from 'vitest' + +import { assessOrcadRollback, planOrcadUpdate } from './orcad-update-plan' +import { + emptyOrcadActivationRecord, + type OrcadActivationRecord, + type OrcadStateSnapshot +} from './orcad-activation-record' + +const SNAPSHOT: OrcadStateSnapshot = { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: '0.2.0+bb01', + readableByVersion: '0.1.0+aa01', + takenAt: '2026-01-01T00:00:00.000Z' +} + +function record(overrides: Partial = {}): OrcadActivationRecord { + return { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + activatedAt: '2026-01-01T00:00:01.000Z', + snapshot: SNAPSHOT, + ...overrides + } +} + +describe('planOrcadUpdate', () => { + it('does nothing when the candidate is already active', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.2.0+bb01', + census: { liveSessions: 0, startedSinceActivation: 0 } + }) + expect(plan).toMatchObject({ action: 'noop' }) + }) + + it('defers rather than restarting a host with live terminals', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 3, startedSinceActivation: 1 } + }) + expect(plan).toMatchObject({ action: 'defer', code: 'orcad_update_terminals_running' }) + expect(plan.action === 'defer' && plan.reason).toContain('would not kill them') + }) + + it('defers when the session count cannot be established', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: null, startedSinceActivation: null } + }) + expect(plan).toMatchObject({ + action: 'defer', + code: 'orcad_update_terminal_census_unavailable' + }) + }) + + it('plans a forced update with an unknown census as if terminals were live', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: null, startedSinceActivation: null }, + force: true + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: true }) + }) + + it('carries the daemon across a forced update with live terminals', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 2, startedSinceActivation: 0 }, + force: true + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: true }) + }) + + it('replaces the daemon only when nothing is running under it', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 0, startedSinceActivation: 0 } + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: false }) + }) +}) + +describe('assessOrcadRollback', () => { + it('is clean when the snapshot is intact and nothing happened since activation', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'clean', target: '0.1.0+aa01' }) + }) + + it('is lossy, and names what goes, once the store has been written since activation', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 1, startedSinceActivation: 0 }, + stateWritesSinceActivation: true + }) + expect(safety).toMatchObject({ safety: 'lossy', target: '0.1.0+aa01' }) + expect(safety.safety === 'lossy' && safety.discards[0]).toContain('2026-01-01T00:00:01.000Z') + }) + + it('treats an unreadable store mtime as writes, not as a clean rollback', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: null + }) + expect(safety).toMatchObject({ safety: 'lossy' }) + }) + + // The point past which rollback is unsafe: the first terminal created after activation. + it('refuses once a terminal started after activation, because restoring would orphan it', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 4, startedSinceActivation: 1 }, + stateWritesSinceActivation: true + }) + expect(safety).toMatchObject({ + safety: 'unsafe', + code: 'orcad_rollback_orphans_live_terminals' + }) + expect(safety.safety === 'unsafe' && safety.reason).toContain('nothing would be able to') + }) + + it('refuses when the snapshot the record names is gone from the host', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: false, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_missing' }) + expect(safety.safety === 'unsafe' && safety.reason).toContain('no schema version') + }) + + it('refuses when no snapshot was ever recorded', () => { + const safety = assessOrcadRollback({ + record: record({ snapshot: null }), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_missing' }) + }) + + it('refuses when the post-activation session count is unverifiable', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 2, startedSinceActivation: null }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_census_unavailable' }) + }) + + it('refuses when there is no previous version to go back to', () => { + const safety = assessOrcadRollback({ + record: record({ previous: null }), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_no_target' }) + }) +}) diff --git a/src/main/ssh/orcad-update-plan.ts b/src/main/ssh/orcad-update-plan.ts new file mode 100644 index 000000000000..2653956ebe51 --- /dev/null +++ b/src/main/ssh/orcad-update-plan.ts @@ -0,0 +1,234 @@ +/** + * When an update may restart orcad, and when going back is still sound. + * + * Two constraints shape everything here. + * + * **The daemon must outlive the restart.** orcad forks the terminal daemon and deliberately + * does not kill it on stop (`orcad-daemon-supervision.ts` uses `disconnectDaemon`, never + * `shutdownDaemon`). An update that killed it would destroy every terminal on the host — + * the thing the daemon exists to prevent. After an update the surviving daemon was forked + * from the OUTGOING bundle, so `daemon-init` sees an entry-path/version mismatch and takes + * its `shouldPreserveDaemonWithLiveSessions` branch: with live sessions it preserves the old + * daemon; at exactly zero it replaces it. Both are correct, and both mean the outgoing + * version's directory is still load-bearing. + * + * **The state root is shared across versions.** `~/.orca/` (or `$ORCA_USER_DATA`) is outside + * every version dir, and Orca's persisted state carries no schema version — migrations run + * on load and rewrite in place. So "is the old version able to read what the new one wrote" + * has no answer that can be computed. That is why rollback is defined against a + * pre-activation snapshot rather than against a version comparison. + */ +import type { OrcadActivationRecord } from './orcad-activation-record' + +export type OrcadTerminalCensus = { + /** + * Sessions the live daemon owns right now. `null` means the probe could not answer — + * never treated as zero, because loss of contact is not evidence of process death + * (docs/reference/ssh-execution-boundary.md). + */ + liveSessions: number | null + /** + * Of those, how many started at or after `record.activatedAt`. These are the sessions the + * pre-activation snapshot does not describe. + */ + startedSinceActivation: number | null +} + +export type OrcadUpdateDecision = + | { action: 'noop'; reason: string } + | { + action: 'proceed' + /** True when a live daemon will be carried across the restart rather than replaced. */ + preservesLiveDaemon: boolean + notes: string[] + } + | { action: 'defer'; code: OrcadUpdateDeferCode; reason: string } + +export type OrcadUpdateDeferCode = + | 'orcad_update_terminals_running' + | 'orcad_update_terminal_census_unavailable' + +/** + * Decide whether to restart orcad onto `candidateVersion`. + * + * Deferring on live terminals is a deliberate choice, not caution. The restart itself is + * non-destructive, but it leaves the host running a NEW orcad against an OLD daemon until + * every one of those terminals exits — a mixed pair whose duration the operator, not the + * deploy, should decide. `force` is how they decide it. + */ +export function planOrcadUpdate(input: { + record: OrcadActivationRecord + candidateVersion: string + census: OrcadTerminalCensus + force?: boolean +}): OrcadUpdateDecision { + if (input.record.active === input.candidateVersion) { + return { + action: 'noop', + reason: `${input.candidateVersion} is already the active version; nothing to restart.` + } + } + const { liveSessions } = input.census + if (liveSessions === null) { + if (!input.force) { + return { + action: 'defer', + code: 'orcad_update_terminal_census_unavailable', + reason: + 'The terminal daemon did not answer a session count, so this update cannot tell ' + + 'whether work is running on the host. Retry, or force the update knowing terminals ' + + 'may be mid-flight.' + } + } + return { + action: 'proceed', + // Why true: an unverifiable census must be planned for as if sessions exist. Assuming + // the daemon is replaceable is the assumption that destroys terminals. + preservesLiveDaemon: true, + notes: [ + 'Forced with an unverifiable session count. Planning as if terminals are live: the ' + + 'daemon will be preserved across the restart, and the outgoing version directory ' + + 'stays pinned against GC.' + ] + } + } + if (liveSessions > 0 && !input.force) { + return { + action: 'defer', + code: 'orcad_update_terminals_running', + reason: + `${liveSessions} terminal${liveSessions === 1 ? ' is' : 's are'} running on this host. ` + + 'The restart would not kill them — the daemon is preserved — but the host would run ' + + `orcad ${input.candidateVersion} against a daemon forked from ` + + `${input.record.active ?? 'the previous build'} until they all exit. Update when the ` + + 'host is idle, or force it.' + } + } + if (liveSessions > 0) { + return { + action: 'proceed', + preservesLiveDaemon: true, + notes: [ + `Forced with ${liveSessions} live terminal${liveSessions === 1 ? '' : 's'}. They survive ` + + 'the restart on the existing daemon; the outgoing version directory stays pinned ' + + 'against GC because that daemon was forked from it.' + ] + } + } + return { + action: 'proceed', + // Zero live sessions is the one case where daemon-init's freshness branch replaces the + // daemon, so nothing is carried across and nothing is lost. + preservesLiveDaemon: false, + notes: [ + 'No terminals are running, so the daemon is replaced by one forked from the new bundle.' + ] + } +} + +export type OrcadRollbackSafety = + | { safety: 'clean'; target: string; notes: string[] } + | { safety: 'lossy'; target: string; discards: string[] } + | { safety: 'unsafe'; code: OrcadRollbackUnsafeCode; reason: string } + +export type OrcadRollbackUnsafeCode = + | 'orcad_rollback_no_target' + | 'orcad_rollback_snapshot_missing' + | 'orcad_rollback_orphans_live_terminals' + | 'orcad_rollback_census_unavailable' + +/** + * How safe it is to switch back to `record.previous`. + * + * **The point past which rollback is unsafe is the first terminal created after + * activation.** Not the first state write, and not any schema comparison: + * + * - Rolling back means restoring the pre-activation snapshot, because there is no schema + * version to prove the old build can read what the new one wrote. + * - The snapshot predates activation, so it does not describe sessions created since. + * - The daemon survives the binary swap and still owns those sessions. After the restore, + * a live daemon holds PTYs that the restored store has no rows for: work that is running, + * that no client can reattach to, and that the host will report as neither `live` nor + * `exited` for any session anyone can name. + * + * Settings and UI churn written after activation are merely discarded, which is `lossy`. + * Orphaning running work is not something a deploy gets to do quietly, so it is `unsafe`. + */ +export function assessOrcadRollback(input: { + record: OrcadActivationRecord + /** Whether the snapshot named by the record is actually still on the host. */ + snapshotPresent: boolean + census: OrcadTerminalCensus + /** + * Whether the shared store has been written since activation, from its mtime against + * `record.activatedAt`. `null` means unknown, which is treated as "yes" — claiming a + * lossless rollback we cannot demonstrate is the failure mode, not the caution. + */ + stateWritesSinceActivation: boolean | null +}): OrcadRollbackSafety { + const target = input.record.previous + if (!target) { + return { + safety: 'unsafe', + code: 'orcad_rollback_no_target', + reason: + 'This host has no previous orcad version recorded, so there is nothing to roll back ' + + 'to. Deploy a known-good build instead.' + } + } + if (!input.record.snapshot || !input.snapshotPresent) { + return { + safety: 'unsafe', + code: 'orcad_rollback_snapshot_missing', + reason: + `The pre-activation state snapshot for ${input.record.active ?? 'the active version'} ` + + 'is gone, and Orca state carries no schema version that could prove the older build ' + + 'can read what the newer one migrated. Switching the binary back would hand ' + + `${target} a store it may not understand. Deploy forward instead.` + } + } + const { startedSinceActivation } = input.census + if (startedSinceActivation === null) { + return { + safety: 'unsafe', + code: 'orcad_rollback_census_unavailable', + reason: + 'The daemon did not answer how many of its terminals started after this version was ' + + 'activated, so a snapshot restore might orphan running work. Retry when the host is ' + + 'reachable.' + } + } + if (startedSinceActivation > 0) { + return { + safety: 'unsafe', + code: 'orcad_rollback_orphans_live_terminals', + reason: + `${startedSinceActivation} terminal${startedSinceActivation === 1 ? '' : 's'} started ` + + 'after this version was activated. The daemon survives the rollback and would keep ' + + 'owning them, but the restored snapshot predates them, so nothing would be able to ' + + 'reattach. Close them (or let them exit) and roll back then.' + } + } + if (input.stateWritesSinceActivation === false) { + return { + safety: 'clean', + target, + notes: [ + 'The pre-activation snapshot is intact, no terminals started since activation, and ' + + 'the store has not been written since. Restoring it changes nothing.' + ] + } + } + return { + safety: 'lossy', + target, + discards: [ + input.stateWritesSinceActivation === null + ? 'Any profile, settings and UI change written since activation — the store mtime ' + + 'could not be read, so assume there are some.' + : `Every profile, settings and UI change written since ${ + input.record.activatedAt ?? 'activation' + }, when ${input.record.active ?? 'the active version'} was activated.` + ] + } +} diff --git a/src/main/ssh/remote-install-coexistence.test.ts b/src/main/ssh/remote-install-coexistence.test.ts new file mode 100644 index 000000000000..ce844587d62b --- /dev/null +++ b/src/main/ssh/remote-install-coexistence.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest' + +import { selectRemoteInstallModel } from './remote-install-coexistence' + +const BOTH_INSTALLED = ['relay-0.1.0+aa01', 'orcad-0.2.0+bb01', 'orcad-0.1.0+aa01'] + +describe('what a client does when it finds both models installed', () => { + it('uses the registered model and leaves the other install alone', () => { + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad' }) + expect(selection.outcome === 'use' && selection.coexisting).toEqual(['relay-0.1.0+aa01']) + expect(selection.outcome === 'use' && selection.note).toContain( + 'garbage-collects only its own namespace' + ) + }) + + it('does not switch model just because the other one is also on disk', () => { + const selection = selectRemoteInstallModel({ + registration: 'ssh-target', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'relay' }) + expect(selection.outcome === 'use' && selection.coexisting).toEqual([ + 'orcad-0.2.0+bb01', + 'orcad-0.1.0+aa01' + ]) + }) + + it('picks the registered model even when only the other one is installed', () => { + // On-disk presence is diagnostic, never a vote: an orcad-registered host with only relay + // dirs is a first orcad deploy, not a reason to fall back to the relay. + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: ['relay-0.1.0+aa01'] + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad' }) + }) + + it('refuses a machine registered under both models', () => { + const selection = selectRemoteInstallModel({ + registration: 'both', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ + outcome: 'refuse', + code: 'remote_host_registered_under_both_models' + }) + // The forbidden thing is two registrations, not two directories — say so, or a user will + // "fix" it by deleting an install that is serving someone. + expect(selection.outcome === 'refuse' && selection.reason).toContain( + 'Leaving both install directories on disk is fine' + ) + }) + + it('refuses to infer a model for an unregistered machine', () => { + const selection = selectRemoteInstallModel({ + registration: 'none', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ + outcome: 'refuse', + code: 'remote_host_not_registered' + }) + }) + + it('says nothing when there is nothing coexisting', () => { + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: ['orcad-0.2.0+bb01'] + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad', note: null }) + }) +}) diff --git a/src/main/ssh/remote-install-coexistence.ts b/src/main/ssh/remote-install-coexistence.ts new file mode 100644 index 000000000000..a24ac36cfb1f --- /dev/null +++ b/src/main/ssh/remote-install-coexistence.ts @@ -0,0 +1,83 @@ +/** + * What a client does when it finds both a relay and an orcad installed on one host. + * + * `docs/design/shipping-orcad.html` §06 draws the line the boundary doc actually draws: + * two *directories* on disk are fine and permanent; two *registered targets* for one + * machine are forbidden, because that is what splits a machine's worktrees across two + * identities (`docs/reference/ssh-execution-boundary.md`). + * + * So the model is never inferred from the filesystem. It is decided by how the user + * registered the host, and the on-disk inventory is only ever diagnostic. Inferring it — + * "an orcad dir exists, so prefer orcad" — would let a GC pass, a half-finished install or + * a stale tree silently re-point a live connection at a different execution identity. + */ +import { inventoryRemoteInstallDirs, type RemoteInstallModelId } from './remote-install-model' + +/** + * How this host is registered in the client's own records. + * + * `both` is representable on purpose: it is a state a user can reach by adding an SSH + * target for a machine they have already paired, and the point of this module is to refuse + * it loudly instead of picking one. + */ +export type RemoteHostRegistration = 'ssh-target' | 'orcad-peer' | 'both' | 'none' + +export type RemoteInstallSelection = + | { + outcome: 'use' + model: RemoteInstallModelId + /** Other-model dirs present on this host. Left alone; never GC'd by the chosen model. */ + coexisting: string[] + /** Non-null when there is something an operator should know but nothing to refuse over. */ + note: string | null + } + | { + outcome: 'refuse' + code: 'remote_host_registered_under_both_models' | 'remote_host_not_registered' + reason: string + } + +/** + * Choose the execution model for a connection to a host, given its registration and + * whatever happens to be installed there. + */ +export function selectRemoteInstallModel(input: { + registration: RemoteHostRegistration + /** Raw directory names under `~/.orca-remote/`, as listed on the host. */ + installedDirNames: readonly string[] +}): RemoteInstallSelection { + if (input.registration === 'both') { + return { + outcome: 'refuse', + code: 'remote_host_registered_under_both_models', + reason: + 'This machine is registered both as an SSH target and as a paired orcad peer. One ' + + 'machine must have one execution identity, or its worktrees and terminals split ' + + 'across two owners that cannot see each other. Remove one registration. Leaving ' + + 'both install directories on disk is fine and expected.' + } + } + if (input.registration === 'none') { + return { + outcome: 'refuse', + code: 'remote_host_not_registered', + reason: + 'This machine has no execution model registered. Add it as an SSH target or pair it ' + + 'as an orcad peer; what is already installed on it does not decide which it is.' + } + } + const model: RemoteInstallModelId = input.registration === 'ssh-target' ? 'relay' : 'orcad' + const inventory = inventoryRemoteInstallDirs(input.installedDirNames) + const coexisting = model === 'relay' ? inventory.orcad : inventory.relay + return { + outcome: 'use', + model, + coexisting, + note: + coexisting.length > 0 + ? `This host also has ${coexisting.length} ${model === 'relay' ? 'orcad' : 'relay'} ` + + `install directory(ies) (${coexisting.join(', ')}). They are left untouched: each ` + + 'model garbage-collects only its own namespace.' + : null + } +} diff --git a/src/main/ssh/remote-install-gc.ts b/src/main/ssh/remote-install-gc.ts new file mode 100644 index 000000000000..0d649cd0b039 --- /dev/null +++ b/src/main/ssh/remote-install-gc.ts @@ -0,0 +1,286 @@ +/** + * The version-directory garbage collector, shared by the relay and orcad. + * + * It lives apart from `ssh-relay-versioned-install.ts` because it is the one piece both + * models run, and because the ownership rule below is the whole point of separating them: + * a pass only ever sees, and only ever deletes, directories belonging to `model`. + */ +import type { SshConnection } from './ssh-connection' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { execCommand } from './ssh-relay-deploy-helpers' +import { probeInstallLockExistsCommand } from './ssh-relay-install-lock-commands' +import { isRelayInstallLockStale, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' +import { + RELAY_INSTALL_MODEL, + remoteInstallGcPermits, + remoteInstallVersionDirRegex, + type RemoteInstallModel +} from './remote-install-model' +import { + isRelayGcClaimOwned, + releaseRelayGcClaimWithRetry, + tryAcquireRelayGcClaim +} from './ssh-relay-gc-claim' +import { cleanupRelayGcTombstones } from './ssh-relay-gc-tombstone' +import { + listRemoteInstallBaseDirsCommand, + MAX_RELAY_GC_LISTING_ENTRIES, + moveRemoteTreeCommand, + probeFileExistsCommand, + relayLivenessProbeCommand, + removeRemoteTreeCommand +} from './ssh-remote-commands' +import { + getRemoteHostPlatform, + isWindowsRemoteHost, + joinRemotePath, + remoteBasename, + type RemoteHostPlatform +} from './ssh-remote-platform' +import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' + +// Legacy relay dirs predate `.install-complete`; they need a liveness-only GC check so they +// eventually drain. There is no orcad equivalent — orcad has never shipped without one. +const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/ +const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') + +function execHostCommand( + conn: SshConnection, + host: RemoteHostPlatform, + command: string +): Promise { + return execCommand(conn, command, { wrapCommand: host.commandDialect !== 'powershell' }) +} + +export type RemoteInstallGcOptions = { + windowsNodePath?: string + windowsSockNames?: string[] + /** + * Model-specific "someone is using this directory" probe. It must answer TRUE when + * inconclusive — an unanswered probe is never evidence a tree is idle. + */ + isDirLive: (dir: string) => Promise + /** + * Directories this pass must never remove even when idle and complete, named by directory + * (not absolute path). orcad passes its active and previous versions: the previous one is + * the rollback target, and GC'ing it turns a recoverable bad update into a re-deploy. + */ + pinnedDirNames?: readonly string[] +} + +/** + * Garbage-collect one model's old version directories. + * + * **GC ownership (design §06 falsifier 1):** a pass only ever sees, and only ever deletes, + * directories belonging to `model`. The remote listing is scoped by prefix, and + * `remoteInstallGcPermits` re-checks every candidate locally, so neither a widened glob nor + * a hand-rolled listing can make one model delete the other's live install. + */ +export async function gcOldRemoteInstallVersions( + conn: SshConnection, + model: RemoteInstallModel, + remoteHome: string, + currentDirAbsPath: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options: RemoteInstallGcOptions +): Promise { + const baseDir = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) + const currentDirName = remoteBasename(currentDirAbsPath, host) + let listing: string + try { + listing = await execHostCommand( + conn, + host, + listRemoteInstallBaseDirsCommand(host, baseDir, model) + ) + } catch { + return + } + const entries = listing + .split('\n') + .map((s) => s.trim()) + .filter(Boolean) + .slice(0, MAX_RELAY_GC_LISTING_ENTRIES) + + await cleanupRelayGcTombstones(conn, baseDir, entries, host) + + const versionDirRegex = remoteInstallVersionDirRegex(model) + const pinned = new Set([currentDirName, ...(options.pinnedDirNames ?? [])]) + const candidates = entries + // Why re-check ownership after a prefix-scoped listing: this is the one line that stands + // between a parameterized GC and deleting the sibling model's live install. + .filter((name) => remoteInstallGcPermits(model, name)) + .filter((name) => versionDirRegex.test(name)) + .filter((name) => !pinned.has(name)) + + if (candidates.length === 0) { + return + } + + const removed: string[] = [] + const kept: string[] = [] + for (const name of candidates) { + const dir = joinRemotePath(host, baseDir, name) + try { + const safe = await isCandidateSafeToRemove(conn, model, dir, name, host, options) + if (!safe) { + kept.push(name) + continue + } + // Why: the claim is a sibling, so it survives moving/deleting the candidate and lets installers back out first. + const gcClaimToken = await tryAcquireRelayGcClaim(conn, dir, host) + if (!gcClaimToken) { + kept.push(name) + continue + } + let preserveGcClaim = false + let gcClaimReleaseNeeded = true + try { + // Recheck under the stable claim; installers probe it before and after creating their lock, closing both orders. + if (!(await isCandidateSafeToRemove(conn, model, dir, name, host, options))) { + kept.push(name) + continue + } + if (!(await isRelayGcClaimOwned(conn, dir, gcClaimToken, host))) { + kept.push(name) + continue + } + const tombstone = `${dir}.gc-tombstone.${process.pid}.${Date.now()}` + const moved = await execHostCommand(conn, host, moveRemoteTreeCommand(host, dir, tombstone)) + if (moved.trim() !== 'MOVED') { + kept.push(name) + continue + } + // Once renamed, a fresh install at the original path is isolated from the tombstone's deletion, so release the claim. + const release = await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) + gcClaimReleaseNeeded = release === 'unknown' + await execHostCommand(conn, host, removeRemoteTreeCommand(host, tombstone)) + } catch (err) { + if (isUnconfirmedSshCommandTermination(err)) { + preserveGcClaim = true + } + throw err + } finally { + if (!preserveGcClaim && gcClaimReleaseNeeded) { + await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) + } + } + removed.push(name) + } catch (err) { + console.warn( + `[${model.id}] GC failed for ${dir}: ${err instanceof Error ? err.message : String(err)}` + ) + kept.push(name) + } + } + + if (removed.length > 0) { + const keptSuffix = kept.length > 0 ? ` (kept: ${kept.join(', ')})` : '' + console.log( + `[${model.id}] GC: removed ${removed.length} stale version dir(s): ${removed.join(', ')}${keptSuffix}` + ) + } +} + +async function isCandidateSafeToRemove( + conn: SshConnection, + model: RemoteInstallModel, + dir: string, + name: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options: RemoteInstallGcOptions +): Promise { + const isLegacy = model.id === 'relay' && LEGACY_RELAY_DIR_REGEX.test(name) + + const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME) + let lockProbe: string + try { + lockProbe = await execHostCommand(conn, host, probeInstallLockExistsCommand(host, lockDir)) + } catch { + return false + } + const lockState = lockProbe.trim() + if (lockState !== 'OPEN' && lockState !== 'LOCKED') { + return false + } + const locked = lockState === 'LOCKED' + + if (locked) { + // Why: stale lock = crashed installer; finalize can leave a dir .install-complete yet locked (lock-rm failed), so it's reclaimable. + if (!(await isRelayInstallLockStale(conn, lockDir, host))) { + return false + } + process.stderr.write?.(`[${model.id}] GC: lock at ${lockDir} is stale; treating as recoverable\n`) + } + + // Legacy dirs predate .install-complete; skip the sentinel and rely on the live-socket probe alone. + if (!isLegacy) { + const completePath = joinRemotePath(host, dir, model.installCompleteFilename) + const completeProbe = await execHostCommand( + conn, + host, + probeFileExistsCommand(host, completePath) + ).catch(() => 'PARTIAL') + if (completeProbe.trim() !== 'COMPLETE') { + // Crashed-install partial; leave for the next deploy to recover. + return false + } + } + + return !(await options.isDirLive(dir)) +} + + +/** + * The relay's GC, bound to its own namespace and its own liveness probe (a live unix socket + * or Windows pipe inside the version dir). + */ +export async function gcOldRelayVersions( + conn: SshConnection, + remoteHome: string, + currentDirAbsPath: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: { + windowsNodePath?: string + windowsSockNames?: string[] + } +): Promise { + await gcOldRemoteInstallVersions(conn, RELAY_INSTALL_MODEL, remoteHome, currentDirAbsPath, host, { + ...options, + isDirLive: (dir) => hasLiveRelaySocket(conn, dir, host, options) + }) +} + +async function hasLiveRelaySocket( + conn: SshConnection, + dir: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: { + windowsNodePath?: string + windowsSockNames?: string[] + } +): Promise { + try { + // Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle. + const windowsOptions = + isWindowsRemoteHost(host) && options?.windowsNodePath + ? { + nodePath: options.windowsNodePath, + pipePaths: (options.windowsSockNames ?? []).flatMap((sockName) => + windowsRelayPipePathsForSocketName(host, dir, sockName) + ) + } + : undefined + const out = await execHostCommand( + conn, + host, + relayLivenessProbeCommand(host, dir, windowsOptions) + ) + const state = out.trim() + return state !== 'DEAD' && state !== 'WAITING' + } catch { + // Why: an inconclusive liveness probe must never authorize deletion. + return true + } +} diff --git a/src/main/ssh/remote-install-model.test.ts b/src/main/ssh/remote-install-model.test.ts new file mode 100644 index 000000000000..5a80df8ff4f6 --- /dev/null +++ b/src/main/ssh/remote-install-model.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' + +import { + inventoryRemoteInstallDirs, + ORCAD_INSTALL_MODEL, + RELAY_INSTALL_MODEL, + remoteInstallDirName, + remoteInstallDirOwner, + remoteInstallGcPermits, + remoteInstallListingRegexSource, + remoteInstallVersionDirRegex +} from './remote-install-model' + +const RELAY_DIRS = ['relay-0.1.0+abcdef123456', 'relay-v0.1.0', 'relay-1.2.3'] +const ORCAD_DIRS = ['orcad-0.1.0+abcdef123456', 'orcad-v0.1.0', 'orcad-1.2.3'] + +describe('remote install namespace', () => { + it('names each model its own version dir', () => { + expect(remoteInstallDirName(RELAY_INSTALL_MODEL, '0.1.0+aa')).toBe('relay-0.1.0+aa') + expect(remoteInstallDirName(ORCAD_INSTALL_MODEL, '0.1.0+aa')).toBe('orcad-0.1.0+aa') + }) + + it('keeps the relay listing pattern byte-identical to the one it shipped with', () => { + // The literal that was hardcoded in `listRelayBaseDirsCommand` before it was + // parameterized. A drift here changes what an existing host's GC can see. + expect(remoteInstallListingRegexSource(RELAY_INSTALL_MODEL)).toBe( + String.raw`^relay-(v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?)(\.gc-tombstone\.[0-9]+\.[0-9]+)?$` + ) + }) + + it('refuses a dir prefix that could escape a remote glob or quote', () => { + const injected = { ...RELAY_INSTALL_MODEL, dirPrefix: "relay'; rm -rf ~" } + expect(() => remoteInstallVersionDirRegex(injected)).toThrow('Unsafe remote install dir prefix') + }) +}) + +describe('GC ownership — each model collects only its own namespace', () => { + it.each(ORCAD_DIRS)('the relay never permits GC of %s', (dirName) => { + expect(remoteInstallDirOwner(dirName)).toBe('orcad') + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, dirName)).toBe(false) + }) + + it.each(RELAY_DIRS)('orcad never permits GC of %s', (dirName) => { + expect(remoteInstallDirOwner(dirName)).toBe('relay') + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, dirName)).toBe(false) + }) + + it('permits each model its own dirs and its own tombstones', () => { + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, 'relay-0.1.0+aa')).toBe(true) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'orcad-0.1.0+aa')).toBe(true) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'orcad-0.1.0+aa.gc-tombstone.12.34')).toBe( + true + ) + }) + + it('claims nothing it did not create', () => { + for (const name of ['.orca-remote', 'orcad', 'relayish-0.1.0', 'orcad-notaversion', 'node']) { + expect(remoteInstallDirOwner(name)).toBeNull() + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, name)).toBe(false) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, name)).toBe(false) + } + }) + + it('groups a mixed listing without losing anything to the wrong owner', () => { + const inventory = inventoryRemoteInstallDirs([...RELAY_DIRS, ...ORCAD_DIRS, 'something-else']) + expect(inventory.relay).toEqual(RELAY_DIRS) + expect(inventory.orcad).toEqual(ORCAD_DIRS) + expect(inventory.unknown).toEqual(['something-else']) + }) +}) diff --git a/src/main/ssh/remote-install-model.ts b/src/main/ssh/remote-install-model.ts new file mode 100644 index 000000000000..54be330a55f1 --- /dev/null +++ b/src/main/ssh/remote-install-model.ts @@ -0,0 +1,143 @@ +/** + * The two things Orca installs into `~/.orca-remote/`, and the rules that keep them from + * touching each other. + * + * `docs/design/shipping-orcad.html` §06 settles that on-disk coexistence is permanent: the + * relay is the dumb execution host for SSH-target users, orcad is the peer for paired + * environments, and no plan item retires either. So `relay-/` and `orcad-/` + * sit side by side forever, and the namespace has to be a parameter rather than a literal. + * + * GC ownership is the trap that parameterization creates. Each model garbage-collects ONLY + * its own directories — see `remoteInstallDirOwner`. Relay's regex happened to be narrow + * enough already; making the prefix a parameter is exactly what could have widened it into + * deleting a live orcad tree, so the ownership rule is asserted here rather than left to + * whichever regex a caller passes. + */ +import { + relayArtifactFilenames, + RELAY_INSTALL_COMPLETE_FILENAME, + RELAY_VERSION_FILENAME +} from '../../shared/relay-artifacts' +import { + orcadArtifactFilenames, + ORCAD_INSTALL_COMPLETE_FILENAME, + ORCAD_VERSION_FILENAME +} from '../../shared/orcad-artifacts' + +export type RemoteInstallModelId = 'relay' | 'orcad' + +export type RemoteInstallModel = { + readonly id: RemoteInstallModelId + /** Leading segment of every version dir: `-`. */ + readonly dirPrefix: string + /** npm package name written into the remote `package.json` for native deps. */ + readonly nativeDepsPackageName: string + readonly versionFilename: string + readonly installCompleteFilename: string + /** Files whose absence means a torn install, so the probe forces a re-deploy. */ + requiredArtifacts(isWindows: boolean): string[] +} + +export const RELAY_INSTALL_MODEL: RemoteInstallModel = { + id: 'relay', + dirPrefix: 'relay', + nativeDepsPackageName: 'orca-relay', + versionFilename: RELAY_VERSION_FILENAME, + installCompleteFilename: RELAY_INSTALL_COMPLETE_FILENAME, + requiredArtifacts: (isWindows) => relayArtifactFilenames(isWindows) +} + +export const ORCAD_INSTALL_MODEL: RemoteInstallModel = { + id: 'orcad', + dirPrefix: 'orcad', + nativeDepsPackageName: 'orca-orcad', + versionFilename: ORCAD_VERSION_FILENAME, + installCompleteFilename: ORCAD_INSTALL_COMPLETE_FILENAME, + // Why the parameter is ignored: orcad's forked children are the same three .js files on + // every host. The Windows-only console-list agent patch is a relay/node-pty concern. + requiredArtifacts: () => orcadArtifactFilenames() +} + +export const REMOTE_INSTALL_MODELS: readonly RemoteInstallModel[] = [ + RELAY_INSTALL_MODEL, + ORCAD_INSTALL_MODEL +] + +/** + * The version half of a directory name, shared by both models. + * + * Why `[0-9]` and not `\d`: this exact source string is also embedded in an awk ERE and a + * PowerShell `-match` on the remote host. Those three dialects agree on `[0-9]`, `\.` and + * `\+`; only JavaScript understands `\d`. + */ +const VERSION_PATTERN = String.raw`v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?` + +/** Suffix GC leaves behind mid-delete; the listing must surface these so they can be swept. */ +const TOMBSTONE_PATTERN = String.raw`\.gc-tombstone\.[0-9]+\.[0-9]+` + +/** + * Why validated and not merely typed: the prefix is interpolated into a remote `find -name` + * glob, an awk regex and a single-quoted PowerShell literal. A quote or a metacharacter + * here would be a remote-shell injection on the client's own connection. + */ +function assertSafeDirPrefix(dirPrefix: string): void { + if (!/^[a-z][a-z0-9-]*$/.test(dirPrefix)) { + throw new Error(`Unsafe remote install dir prefix: ${JSON.stringify(dirPrefix)}`) + } +} + +export function remoteInstallDirName(model: RemoteInstallModel, fullVersion: string): string { + assertSafeDirPrefix(model.dirPrefix) + return `${model.dirPrefix}-${fullVersion}` +} + +/** Matches a live version dir for exactly one model — never a tombstone, never a sibling model. */ +export function remoteInstallVersionDirRegex(model: RemoteInstallModel): RegExp { + assertSafeDirPrefix(model.dirPrefix) + return new RegExp(`^${model.dirPrefix}-(${VERSION_PATTERN})$`) +} + +/** What the remote listing is allowed to return: live dirs plus their tombstones. */ +export function remoteInstallListingRegexSource(model: RemoteInstallModel): string { + assertSafeDirPrefix(model.dirPrefix) + return `^${model.dirPrefix}-(${VERSION_PATTERN})(${TOMBSTONE_PATTERN})?$` +} + +/** + * Which model owns a directory found in `~/.orca-remote/`, or null for anything neither + * model created. + * + * This is the answer to §06 falsifier 1's first half: **the model that created a directory + * owns it, and nothing else may delete it.** A relay GC pass that saw `orcad-0.1.0+abc` + * would be looking at the live install of a peer whose lifecycle it has no view into — the + * SSH-execution-boundary collapse in directory form. + */ +export function remoteInstallDirOwner(dirName: string): RemoteInstallModelId | null { + for (const model of REMOTE_INSTALL_MODELS) { + if (new RegExp(remoteInstallListingRegexSource(model)).test(dirName)) { + return model.id + } + } + return null +} + +/** True when `model` is allowed to garbage-collect `dirName`. */ +export function remoteInstallGcPermits(model: RemoteInstallModel, dirName: string): boolean { + return remoteInstallDirOwner(dirName) === model.id +} + +export type RemoteInstallInventory = Record + +/** Group a raw `~/.orca-remote/` listing by owning model, for diagnostics and the client's choice. */ +export function inventoryRemoteInstallDirs(dirNames: readonly string[]): RemoteInstallInventory { + const inventory: RemoteInstallInventory = { relay: [], orcad: [], unknown: [] } + for (const name of dirNames) { + const owner = remoteInstallDirOwner(name) + if (owner) { + inventory[owner].push(name) + } else { + inventory.unknown.push(name) + } + } + return inventory +} diff --git a/src/main/ssh/ssh-relay-install-namespace.ts b/src/main/ssh/ssh-relay-install-namespace.ts index a75b80488416..6fd3c2aa89f2 100644 --- a/src/main/ssh/ssh-relay-install-namespace.ts +++ b/src/main/ssh/ssh-relay-install-namespace.ts @@ -7,6 +7,11 @@ // See: docs/ssh-relay-sftp-namespace.md import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + RELAY_INSTALL_MODEL, + remoteInstallDirName, + type RemoteInstallModel +} from './remote-install-model' import type { SftpNamespacePathMapping } from './sftp-namespace-resolution' import { shellEscape } from './ssh-connection-utils' import { RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' @@ -37,7 +42,19 @@ export function relayRemoteDirSegments( fullVersion: string, pathFlavor: RemotePathFlavor ): string[] { - const segments = [RELAY_REMOTE_DIR, `relay-${fullVersion}`] + return remoteInstallDirSegments(RELAY_INSTALL_MODEL, fullVersion, pathFlavor) +} + +/** + * The model-parameterized form. `relay-` and `orcad-` are permanent siblings under + * one `.orca-remote/` (see remote-install-model.ts), so the prefix is an argument. + */ +export function remoteInstallDirSegments( + model: RemoteInstallModel, + fullVersion: string, + pathFlavor: RemotePathFlavor +): string[] { + const segments = [RELAY_REMOTE_DIR, remoteInstallDirName(model, fullVersion)] for (const segment of segments) { assertSafeRemotePathSegment(segment, pathFlavor) // Why: the version reaches logs and diagnostics, where an embedded CR/LF can forge lines. @@ -49,7 +66,14 @@ export function relayRemoteDirSegments( } export function relayHomeRelativeDir(fullVersion: string): string { - return relayRemoteDirSegments(fullVersion, 'posix').join('/') + return remoteInstallHomeRelativeDir(RELAY_INSTALL_MODEL, fullVersion) +} + +export function remoteInstallHomeRelativeDir( + model: RemoteInstallModel, + fullVersion: string +): string { + return remoteInstallDirSegments(model, fullVersion, 'posix').join('/') } export function createRelayInstallNamespace(homeRelativeRelayDir: string): RelayInstallNamespace { diff --git a/src/main/ssh/ssh-relay-versioned-install.ts b/src/main/ssh/ssh-relay-versioned-install.ts index e4380c319c34..d2ddc62e64bf 100644 --- a/src/main/ssh/ssh-relay-versioned-install.ts +++ b/src/main/ssh/ssh-relay-versioned-install.ts @@ -7,24 +7,12 @@ import { join } from 'node:path' import { existsSync, readFileSync } from 'node:fs' import type { SshConnection } from './ssh-connection' -import { RELAY_REMOTE_DIR } from './relay-protocol' import { execCommand } from './ssh-relay-deploy-helpers' -import { probeInstallLockExistsCommand } from './ssh-relay-install-lock-commands' -import { isRelayInstallLockStale, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' -import { relayRemoteDirSegments } from './ssh-relay-install-namespace' +import { RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' +import { remoteInstallDirSegments } from './ssh-relay-install-namespace' +import { RELAY_INSTALL_MODEL, type RemoteInstallModel } from './remote-install-model' import { - isRelayGcClaimOwned, - releaseRelayGcClaimWithRetry, - tryAcquireRelayGcClaim -} from './ssh-relay-gc-claim' -import { cleanupRelayGcTombstones } from './ssh-relay-gc-tombstone' -import { - listRelayBaseDirsCommand, - MAX_RELAY_GC_LISTING_ENTRIES, - moveRemoteTreeCommand, - probeFileExistsCommand, - probeRelayInstalledCommand, - relayLivenessProbeCommand, + probeRemoteInstallCompleteCommand, removeRemoteTreeCommand, writeRemoteEmptyFileCommand } from './ssh-remote-commands' @@ -32,20 +20,11 @@ import { getRemoteHostPlatform, isWindowsRemoteHost, joinRemotePath, - remoteBasename, type RemoteHostPlatform, type RemotePathFlavor } from './ssh-remote-platform' -import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' -import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' import { isSshSessionLimitError } from './ssh-session-limit-error' -// Single source of truth for GC and the version-dir parser; matches both the new and legacy relay-dir layouts. -const RELAY_VERSION_DIR_REGEX = /^relay-(v?\d+\.\d+\.\d+(\+[0-9a-f]+)?)$/ - -// Legacy dirs predate `.install-complete`; they need a liveness-only GC check so they eventually drain. -const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/ - const INSTALL_COMPLETE_NAME = '.install-complete' const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') @@ -97,13 +76,23 @@ export function computeRemoteRelayDir( remoteHome: string, fullVersion: string, pathFlavor: RemotePathFlavor = 'posix' +): string { + // Why: shell and SFTP-relative builders must derive the same validated segments or the namespaces diverge. + return computeRemoteInstallDir(RELAY_INSTALL_MODEL, remoteHome, fullVersion, pathFlavor) +} + +/** The model-parameterized form of `computeRemoteRelayDir`. */ +export function computeRemoteInstallDir( + model: RemoteInstallModel, + remoteHome: string, + fullVersion: string, + pathFlavor: RemotePathFlavor = 'posix' ): string { const host = pathFlavor === 'windows' ? getRemoteHostPlatform('win32-x64') : getRemoteHostPlatform('linux-x64') - // Why: shell and SFTP-relative builders must derive the same validated segments or the namespaces diverge. - return joinRemotePath(host, remoteHome, ...relayRemoteDirSegments(fullVersion, pathFlavor)) + return joinRemotePath(host, remoteHome, ...remoteInstallDirSegments(model, fullVersion, pathFlavor)) } /** @@ -116,11 +105,26 @@ export async function isRelayAlreadyInstalled( host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, options?: RelayInstalledProbeOptions ): Promise { + return isRemoteInstallComplete(conn, RELAY_INSTALL_MODEL, remoteRelayDir, host, options) +} + +/** The model-parameterized form: each model probes for its own artifact list. */ +export async function isRemoteInstallComplete( + conn: SshConnection, + model: RemoteInstallModel, + remoteInstallDir: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: RelayInstalledProbeOptions +): Promise { + const remoteRelayDir = remoteInstallDir try { const probe = await execHostCommand( conn, host, - probeRelayInstalledCommand(host, remoteRelayDir), + probeRemoteInstallCompleteCommand(host, remoteRelayDir, [ + ...model.requiredArtifacts(isWindowsRemoteHost(host)), + model.installCompleteFilename + ]), { signal: options?.signal } ) return probe.trim() === 'OK' @@ -175,188 +179,10 @@ export async function abandonInstall( * unlocked sibling version dir (never the current one). Best-effort — errors * are swallowed so GC never blocks the user from connecting. */ -export async function gcOldRelayVersions( - conn: SshConnection, - remoteHome: string, - currentDirAbsPath: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - const baseDir = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) - const currentDirName = remoteBasename(currentDirAbsPath, host) - let listing: string - try { - listing = await execHostCommand(conn, host, listRelayBaseDirsCommand(host, baseDir)) - } catch { - return - } - const entries = listing - .split('\n') - .map((s) => s.trim()) - .filter(Boolean) - .slice(0, MAX_RELAY_GC_LISTING_ENTRIES) - - await cleanupRelayGcTombstones(conn, baseDir, entries, host) - - const candidates = entries - .filter((name) => RELAY_VERSION_DIR_REGEX.test(name)) - .filter((name) => name !== currentDirName) - - if (candidates.length === 0) { - return - } - - const removed: string[] = [] - const kept: string[] = [] - for (const name of candidates) { - const dir = joinRemotePath(host, baseDir, name) - try { - const safe = await isCandidateSafeToRemove(conn, dir, name, host, options) - if (!safe) { - kept.push(name) - continue - } - // Why: the claim is a sibling, so it survives moving/deleting the candidate and lets installers back out first. - const gcClaimToken = await tryAcquireRelayGcClaim(conn, dir, host) - if (!gcClaimToken) { - kept.push(name) - continue - } - let preserveGcClaim = false - let gcClaimReleaseNeeded = true - try { - // Recheck under the stable claim; installers probe it before and after creating their lock, closing both orders. - if (!(await isCandidateSafeToRemove(conn, dir, name, host, options))) { - kept.push(name) - continue - } - if (!(await isRelayGcClaimOwned(conn, dir, gcClaimToken, host))) { - kept.push(name) - continue - } - const tombstone = `${dir}.gc-tombstone.${process.pid}.${Date.now()}` - const moved = await execHostCommand(conn, host, moveRemoteTreeCommand(host, dir, tombstone)) - if (moved.trim() !== 'MOVED') { - kept.push(name) - continue - } - // Once renamed, a fresh install at the original path is isolated from the tombstone's deletion, so release the claim. - const release = await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) - gcClaimReleaseNeeded = release === 'unknown' - await execHostCommand(conn, host, removeRemoteTreeCommand(host, tombstone)) - } catch (err) { - if (isUnconfirmedSshCommandTermination(err)) { - preserveGcClaim = true - } - throw err - } finally { - if (!preserveGcClaim && gcClaimReleaseNeeded) { - await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) - } - } - removed.push(name) - } catch (err) { - console.warn( - `[ssh-relay] GC failed for ${dir}: ${err instanceof Error ? err.message : String(err)}` - ) - kept.push(name) - } - } - - if (removed.length > 0) { - const keptSuffix = kept.length > 0 ? ` (kept: ${kept.join(', ')})` : '' - console.log( - `[ssh-relay] GC: removed ${removed.length} stale version dir(s): ${removed.join(', ')}${keptSuffix}` - ) - } -} - -async function isCandidateSafeToRemove( - conn: SshConnection, - dir: string, - name: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - const isLegacy = LEGACY_RELAY_DIR_REGEX.test(name) - - const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME) - let lockProbe: string - try { - lockProbe = await execHostCommand(conn, host, probeInstallLockExistsCommand(host, lockDir)) - } catch { - return false - } - const lockState = lockProbe.trim() - if (lockState !== 'OPEN' && lockState !== 'LOCKED') { - return false - } - const locked = lockState === 'LOCKED' - - if (locked) { - // Why: stale lock = crashed installer; finalize can leave a dir .install-complete yet locked (lock-rm failed), so it's reclaimable. - if (!(await isRelayInstallLockStale(conn, lockDir, host))) { - return false - } - process.stderr.write?.(`[ssh-relay] GC: lock at ${lockDir} is stale; treating as recoverable\n`) - } - - // Legacy dirs predate .install-complete; skip the sentinel and rely on the live-socket probe alone. - if (!isLegacy) { - const completePath = joinRemotePath(host, dir, INSTALL_COMPLETE_NAME) - const completeProbe = await execHostCommand( - conn, - host, - probeFileExistsCommand(host, completePath) - ).catch(() => 'PARTIAL') - if (completeProbe.trim() !== 'COMPLETE') { - // Crashed-install partial; leave for the next deploy to recover. - return false - } - } - - const sockAlive = await hasLiveRelaySocket(conn, dir, host, options) - if (sockAlive) { - return false - } - return true -} - -async function hasLiveRelaySocket( - conn: SshConnection, - dir: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - try { - // Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle. - const windowsOptions = - isWindowsRemoteHost(host) && options?.windowsNodePath - ? { - nodePath: options.windowsNodePath, - pipePaths: (options.windowsSockNames ?? []).flatMap((sockName) => - windowsRelayPipePathsForSocketName(host, dir, sockName) - ) - } - : undefined - const out = await execHostCommand( - conn, - host, - relayLivenessProbeCommand(host, dir, windowsOptions) - ) - const state = out.trim() - return state !== 'DEAD' && state !== 'WAITING' - } catch { - // Why: an inconclusive liveness probe must never authorize deletion. - return true - } -} +// Why re-exported rather than moved outright: deploy and the relay tests import the whole +// versioned-install surface from here, and the split exists for file size, not to redraw an API. +export { + gcOldRelayVersions, + gcOldRemoteInstallVersions, + type RemoteInstallGcOptions +} from './remote-install-gc' diff --git a/src/main/ssh/ssh-remote-commands.ts b/src/main/ssh/ssh-remote-commands.ts index 4fa1445969d2..fc3e965193ed 100644 --- a/src/main/ssh/ssh-remote-commands.ts +++ b/src/main/ssh/ssh-remote-commands.ts @@ -2,6 +2,11 @@ import { RELAY_INSTALL_COMPLETE_FILENAME, relayArtifactFilenames } from '../../shared/relay-artifacts' +import { + RELAY_INSTALL_MODEL, + remoteInstallListingRegexSource, + type RemoteInstallModel +} from './remote-install-model' import type { RemoteHostPlatform } from './ssh-remote-platform' import { isWindowsRemoteHost, joinRemotePath, remoteDirname } from './ssh-remote-platform' import { powerShellCommand, powerShellLiteral, powerShellNativeArg } from './ssh-remote-powershell' @@ -99,10 +104,28 @@ export function probeRelayInstalledCommand( host: RemoteHostPlatform, remoteRelayDir: string ): string { - const required = [ + return probeRemoteInstallCompleteCommand(host, remoteRelayDir, [ ...relayArtifactFilenames(isWindowsRemoteHost(host)), RELAY_INSTALL_COMPLETE_FILENAME - ].map((filename) => joinRemotePath(host, remoteRelayDir, filename)) + ]) +} + +/** + * The model-agnostic form: any install is complete when its directory exists and every + * named artifact is a regular file inside it. + * + * Why the caller passes the list: orcad and the relay ship different artifacts, and a probe + * that checked a shared subset would call a torn install complete. + */ +export function probeRemoteInstallCompleteCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + requiredFilenames: readonly string[] +): string { + const remoteRelayDir = remoteInstallDir + const required = requiredFilenames.map((filename) => + joinRemotePath(host, remoteRelayDir, filename) + ) if (!isWindowsRemoteHost(host)) { const fileTests = required.map((path) => `&& test -f ${shellEscape(path)} `).join('') return `test -d ${shellEscape(remoteRelayDir)} ${fileTests}&& echo OK || echo MISSING` @@ -121,12 +144,28 @@ export function probeRelayInstalledCommand( export const MAX_RELAY_GC_LISTING_ENTRIES = 64 export function listRelayBaseDirsCommand(host: RemoteHostPlatform, baseDir: string): string { + return listRemoteInstallBaseDirsCommand(host, baseDir, RELAY_INSTALL_MODEL) +} + +/** + * List one model's version dirs (and its own tombstones) under `~/.orca-remote/`. + * + * The model scopes BOTH the `find`/`Get-ChildItem` glob and the validating regex. That + * double filter is the on-the-wire half of the GC ownership rule: an orcad GC pass never + * even receives a relay directory name, so it cannot delete one through a later bug. + */ +export function listRemoteInstallBaseDirsCommand( + host: RemoteHostPlatform, + baseDir: string, + model: RemoteInstallModel +): string { + const namePattern = remoteInstallListingRegexSource(model) if (!isWindowsRemoteHost(host)) { const statusPrefix = '__ORCA_RELAY_GC_FIND_STATUS__' return [ `base=${shellEscape(baseDir)}; [ -d "$base" ] || exit 0;`, - `{ find "$base" -mindepth 1 -maxdepth 1 -type d -name 'relay-*' -print; status=$?; printf '\n${statusPrefix}%s\n' "$status"; } |`, - String.raw`awk 'BEGIN { count=0; status=-1 } /^${statusPrefix}[0-9]+$/ { status=substr($0, ${statusPrefix.length + 1}); next } { name=$0; sub(/^.*\//, "", name); if (name ~ /^relay-(v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?)(\.gc-tombstone\.[0-9]+\.[0-9]+)?$/ && count < ${MAX_RELAY_GC_LISTING_ENTRIES}) { entries[count++]=name } } END { if (status != 0) exit 1; for (i=0; i !artifact.optional).map( + (artifact) => artifact.filename + ) +}