diff --git a/config/scripts/build-orcad.mjs b/config/scripts/build-orcad.mjs index 9a6bacd5eddc..69c12f0ecc95 100644 --- a/config/scripts/build-orcad.mjs +++ b/config/scripts/build-orcad.mjs @@ -9,10 +9,25 @@ */ import { fork, spawnSync } from 'node:child_process' import { build } from 'esbuild' -import { chmodSync, copyFileSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' import { arch, platform, tmpdir } from 'node:os' import { join } from 'node:path' import process from 'node:process' +import { + ORCAD_VERSION, + ORCAD_VERSION_FILENAME, + orcadArtifactFilenames +} from '../../src/shared/orcad-artifacts.ts' const ROOT = join(import.meta.dirname, '..', '..') const OUT_DIR = join(ROOT, 'out', 'orcad') @@ -226,9 +241,26 @@ if (graphErrors.length > 0) { } } +// Why a content hash and not ORCAD_VERSION alone: the remote install directory is keyed on +// this string, so two different builds carrying one version would share a directory — and an +// already-`.install-complete` dir is never re-uploaded. The deploy would silently run stale +// bytes while reporting the new version. if (process.exitCode !== 1) { + const hash = createHash('sha256') + for (const filename of orcadArtifactFilenames()) { + const artifactPath = join(OUT_DIR, filename) + if (!existsSync(artifactPath)) { + throw new Error( + `orcad declares ${filename} in ORCAD_ARTIFACTS but never emitted it. Add the build ` + + 'step, or drop it from src/shared/orcad-artifacts.ts.' + ) + } + hash.update(readFileSync(artifactPath)) + } + const fullVersion = `${ORCAD_VERSION}+${hash.digest('hex').slice(0, 12)}` + writeFileSync(join(OUT_DIR, ORCAD_VERSION_FILENAME), fullVersion) console.log( - `[build-orcad] ok — ${(output.bytes / 1024 / 1024).toFixed(2)} MB, ${Object.keys(output.inputs).length} modules, zero electron and node:sqlite imports.` + `[build-orcad] ok — ${fullVersion}, ${(output.bytes / 1024 / 1024).toFixed(2)} MB, ${Object.keys(output.inputs).length} modules, zero electron and node:sqlite imports.` ) } diff --git a/src/main/ssh/orcad-activation-gate.test.ts b/src/main/ssh/orcad-activation-gate.test.ts new file mode 100644 index 000000000000..534d41c08f02 --- /dev/null +++ b/src/main/ssh/orcad-activation-gate.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest' + +import { evaluateOrcadActivation } from './orcad-activation-gate' +import type { ServeReadiness } from '../server/serve-readiness' +import type { OrcadHealth, TerminalDaemonHealth } from '../orcad/orcad-health' + +const EXPECTED = { buildHash: 'abc123def4567890', fullVersion: '0.2.0+bb01' } + +function daemon(overrides: Partial = {}): TerminalDaemonHealth { + return { + state: 'live', + ownsFreshSessions: true, + pid: 4242, + buildVersion: '0.2.0+bb01', + entryPath: '/home/u/.orca-remote/orcad-0.2.0+bb01/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 12 }, + ...overrides + } +} + +function health(overrides: Partial = {}): OrcadHealth { + return { + buildHash: EXPECTED.buildHash, + buildVersion: EXPECTED.fullVersion, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 4200, + terminalDaemon: daemon(), + ...overrides + } +} + +function readiness(overrides: Partial = {}): ServeReadiness { + return { + runtimeId: 'runtime-1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: health(), + ...overrides + } +} + +describe('evaluateOrcadActivation', () => { + it('activates a candidate that proved a real PTY round trip', () => { + const verdict = evaluateOrcadActivation(readiness(), EXPECTED) + expect(verdict).toEqual({ decision: 'activate', coverage: 'pty-spawn', warnings: [] }) + }) + + it('refuses when the candidate never published readiness', () => { + const verdict = evaluateOrcadActivation(null, EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_no_readiness' }) + }) + + it('refuses a readiness payload with no health, rather than reading silence as healthy', () => { + const { health: _dropped, ...withoutHealth } = readiness() + const verdict = evaluateOrcadActivation(withoutHealth as ServeReadiness, EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_no_health' }) + }) + + it('refuses when a different build answered — a stale process holding the port', () => { + const verdict = evaluateOrcadActivation( + readiness({ health: health({ buildHash: '0000000000000000' }) }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_build_mismatch' }) + }) + + it('refuses a listening orcad whose terminal daemon is absent', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + terminalDaemon: daemon({ + state: 'absent', + ownsFreshSessions: false, + selfTest: { ok: false, coverage: 'pty-spawn', verdict: 'no-daemon', durationMs: 1 } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_daemon_absent' }) + }) + + it('refuses a degraded daemon, whose fresh terminals would not survive a restart', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ terminalDaemon: daemon({ state: 'degraded', ownsFreshSessions: false }) }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_daemon_degraded' }) + }) + + it('refuses a live daemon that failed its PTY spawn probe', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + terminalDaemon: daemon({ + selfTest: { + ok: false, + coverage: 'pty-spawn', + verdict: 'pty-spawn-unhealthy', + durationMs: 30 + } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ + decision: 'reject', + code: 'orcad_activation_pty_self_test_failed' + }) + }) + + it('refuses a green daemon that does not own fresh sessions', () => { + const verdict = evaluateOrcadActivation( + readiness({ health: health({ terminalDaemon: daemon({ ownsFreshSessions: false }) }) }), + EXPECTED + ) + expect(verdict).toMatchObject({ + decision: 'reject', + code: 'orcad_activation_no_persistent_terminals' + }) + }) + + it('refuses a candidate that is not listening', () => { + const verdict = evaluateOrcadActivation(readiness({ boundEndpoint: null }), EXPECTED) + expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_not_listening' }) + }) + + it('activates handshake-only coverage but never records it as a proven PTY', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ + platform: 'win32', + terminalDaemon: daemon({ + selfTest: { ok: true, coverage: 'handshake', verdict: 'healthy', durationMs: 5 } + }) + }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ decision: 'activate', coverage: 'handshake' }) + expect(verdict.decision === 'activate' && verdict.warnings[0]).toContain( + 'covered the daemon handshake only' + ) + }) + + it('checks identity before health, so a wrong-build green payload cannot pass', () => { + const verdict = evaluateOrcadActivation( + readiness({ + health: health({ buildHash: 'ffffffffffffffff', terminalDaemon: daemon() }) + }), + EXPECTED + ) + expect(verdict).toMatchObject({ code: 'orcad_activation_build_mismatch' }) + }) +}) diff --git a/src/main/ssh/orcad-activation-gate.ts b/src/main/ssh/orcad-activation-gate.ts new file mode 100644 index 000000000000..c95819191179 --- /dev/null +++ b/src/main/ssh/orcad-activation-gate.ts @@ -0,0 +1,155 @@ +/** + * Whether a freshly launched orcad has earned the right to become the active one. + * + * The failure this exists to prevent is the one `docs/design/shipping-orcad.html` names + * throughout: a deployment that reports success because a port opened. orcad answers RPC + * from its own process, so "listening" stays true while the terminal daemon that owns every + * terminal is dead — a green host that cannot run a single command. Activation therefore + * reads the cross-process health payload the candidate published, not the exit code of the + * command that started it. + * + * A refusal here is not a failure to deploy. The bytes are installed and the previous + * version is still active; nothing was lost. Activating on a bad verdict is what loses + * things. + */ +import type { ServeReadiness } from '../server/serve-readiness' + +export type OrcadActivationRejectCode = + | 'orcad_activation_no_readiness' + | 'orcad_activation_no_health' + | 'orcad_activation_build_mismatch' + | 'orcad_activation_not_listening' + | 'orcad_activation_daemon_absent' + | 'orcad_activation_daemon_degraded' + | 'orcad_activation_pty_self_test_failed' + | 'orcad_activation_no_persistent_terminals' + +export type OrcadActivationVerdict = + | { + decision: 'activate' + /** + * `pty-spawn` means a real PTY was created and torn down inside the daemon. + * `handshake` means the daemon answered but its spawn probe is a no-op on this + * platform (win32). Carried through so an activation is never recorded as proving + * more than it did. + */ + coverage: 'pty-spawn' | 'handshake' + warnings: string[] + } + | { decision: 'reject'; code: OrcadActivationRejectCode; reason: string } + +export type OrcadActivationExpectation = { + /** sha256(orcad.js).slice(0,16) computed from the bytes this client just uploaded. */ + buildHash: string + /** The full content-hashed version this deploy installed. */ + fullVersion: string +} + +/** + * Gate an activation on what the candidate actually reported. + * + * Order matters: identity before health. A health payload from the wrong process is worse + * than no payload, because it is green and about something else. + */ +export function evaluateOrcadActivation( + readiness: ServeReadiness | null, + expected: OrcadActivationExpectation +): OrcadActivationVerdict { + if (!readiness) { + return { + decision: 'reject', + code: 'orcad_activation_no_readiness', + reason: + 'The candidate orcad never published an `orca_server_ready` line. It may have exited, ' + + 'failed to bind, or be wedged before readiness. Nothing was activated.' + } + } + const health = readiness.health + if (!health) { + return { + decision: 'reject', + code: 'orcad_activation_no_health', + reason: + 'The candidate published readiness without a health payload, so its terminal daemon ' + + 'is unverified. Absence of a verdict is not a healthy verdict — treat this build as ' + + 'too old to gate on and do not activate it.' + } + } + // Why identity first: a stale orcad already holding the port would answer readiness and + // report its own (healthy) daemon. Activating on that record points the pointer at bytes + // nobody is running. + if (health.buildHash !== expected.buildHash) { + return { + decision: 'reject', + code: 'orcad_activation_build_mismatch', + reason: + `The process that answered is running build ${health.buildHash}, not the ` + + `${expected.buildHash} this deploy installed. Something else owns that port, or the ` + + 'upload did not land. Nothing was activated.' + } + } + if (!readiness.boundEndpoint) { + return { + decision: 'reject', + code: 'orcad_activation_not_listening', + reason: + 'The candidate reported no bound endpoint, so no client could reach it. Nothing was ' + + 'activated.' + } + } + const daemon = health.terminalDaemon + if (daemon.state === 'absent') { + return { + decision: 'reject', + code: 'orcad_activation_daemon_absent', + reason: + 'The candidate has no terminal daemon. Every terminal on this host would run in the ' + + 'orcad process and die with it, which is the exact regression the daemon exists to ' + + 'prevent. Nothing was activated.' + } + } + if (daemon.state === 'degraded') { + return { + decision: 'reject', + code: 'orcad_activation_daemon_degraded', + reason: + `The candidate's terminal daemon is degraded (self-test: ${daemon.selfTest.verdict}). ` + + 'Existing sessions keep working, but fresh terminals would not survive a restart. ' + + 'Nothing was activated; the previous version is still serving.' + } + } + if (!daemon.selfTest.ok) { + return { + decision: 'reject', + code: 'orcad_activation_pty_self_test_failed', + reason: + `The candidate's PTY self-test failed (${daemon.selfTest.verdict}). The host is ` + + 'listening but cannot create a terminal. Nothing was activated.' + } + } + if (!daemon.ownsFreshSessions) { + return { + decision: 'reject', + code: 'orcad_activation_no_persistent_terminals', + reason: + 'The candidate answered healthy but does not own fresh sessions, so new terminals ' + + 'would not survive its own restart. Nothing was activated.' + } + } + const warnings: string[] = [] + if (daemon.selfTest.coverage === 'handshake') { + warnings.push( + 'The PTY self-test covered the daemon handshake only — this platform does not spawn a ' + + 'probe PTY. Terminal creation is unproven on this host.' + ) + } + if (health.buildVersion !== expected.fullVersion) { + // Not a rejection: the hash already proved identity, and ORCA_VERSION is whatever the + // launch command exported. Worth saying, because a mismatch means the launch env is wrong. + warnings.push( + `The candidate reports version ${health.buildVersion} but was installed as ` + + `${expected.fullVersion}; check ORCA_VERSION in the launch command.` + ) + } + return { decision: 'activate', coverage: daemon.selfTest.coverage, warnings } +} diff --git a/src/main/ssh/orcad-activation-record-store.ts b/src/main/ssh/orcad-activation-record-store.ts new file mode 100644 index 000000000000..d60634ec525d --- /dev/null +++ b/src/main/ssh/orcad-activation-record-store.ts @@ -0,0 +1,48 @@ +/** + * Reading the activation record off a host. + * + * Split out from the deploy driver because the rollback path needs it too, and because an + * unreadable record must fail loudly in both: treating "I cannot parse this" as "nothing is + * activated" would deploy over a live install and lose its rollback target. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_ACTIVATION_FILENAME, + emptyOrcadActivationRecord, + parseOrcadActivationRecord, + type OrcadActivationRecord +} from './orcad-activation-record' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +export function orcadActivationPath(host: RemoteHostPlatform, remoteHome: string): string { + return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_ACTIVATION_FILENAME) +} + +export async function readOrcadActivationRecord(options: { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + signal?: AbortSignal +}): Promise { + const path = orcadActivationPath(options.host, options.remoteHome) + const raw = await execCommand(options.conn, `cat ${shellQuote(path)} 2>/dev/null || true`, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + }).catch(() => '') + const parsed = parseOrcadActivationRecord(raw) + if (parsed.state === 'ok') { + return parsed.record + } + if (parsed.state === 'unreadable') { + // Why throw: an unreadable record is not an empty one. Treating it as empty would + // activate over a live install and orphan its rollback target. + throw new Error(`Cannot read this host's orcad activation record: ${parsed.reason}`) + } + return emptyOrcadActivationRecord() +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} diff --git a/src/main/ssh/orcad-activation-record.test.ts b/src/main/ssh/orcad-activation-record.test.ts new file mode 100644 index 000000000000..dd625238bb12 --- /dev/null +++ b/src/main/ssh/orcad-activation-record.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest' + +import { + emptyOrcadActivationRecord, + orcadGcPinnedDirNames, + parseOrcadActivationRecord, + serializeOrcadActivationRecord, + withActivatedVersion, + withRolledBackVersion, + type OrcadStateSnapshot +} from './orcad-activation-record' + +const SNAPSHOT: OrcadStateSnapshot = { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: '0.2.0+bb01', + readableByVersion: '0.1.0+aa01', + takenAt: '2026-01-01T00:00:00.000Z' +} +const NOW = new Date('2026-01-02T00:00:00.000Z') + +describe('orcad activation record', () => { + it('round-trips through the host', () => { + const record = withActivatedVersion( + { ...emptyOrcadActivationRecord(), active: '0.1.0+aa01' }, + '0.2.0+bb01', + SNAPSHOT, + NOW + ) + const parsed = parseOrcadActivationRecord(serializeOrcadActivationRecord(record)) + expect(parsed).toEqual({ state: 'ok', record }) + }) + + it('reports an absent record as absent', () => { + expect(parseOrcadActivationRecord(null)).toEqual({ state: 'absent' }) + expect(parseOrcadActivationRecord(' ')).toEqual({ state: 'absent' }) + }) + + it('reports a newer schema as unreadable, never as absent', () => { + const parsed = parseOrcadActivationRecord(JSON.stringify({ schemaVersion: 2, active: 'x' })) + expect(parsed.state).toBe('unreadable') + }) + + it('reports corrupt JSON as unreadable, never as absent', () => { + expect(parseOrcadActivationRecord('{not json').state).toBe('unreadable') + }) + + it('names the outgoing version as the rollback target', () => { + const record = withActivatedVersion( + { ...emptyOrcadActivationRecord(), active: '0.1.0+aa01' }, + '0.2.0+bb01', + SNAPSHOT, + NOW + ) + expect(record).toMatchObject({ active: '0.2.0+bb01', previous: '0.1.0+aa01' }) + }) + + it('does not let a re-deploy of the active version erase the rollback target', () => { + const before = { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + snapshot: SNAPSHOT + } + const after = withActivatedVersion(before, '0.2.0+bb01', null, NOW) + expect(after).toMatchObject({ active: '0.2.0+bb01', previous: '0.1.0+aa01' }) + expect(after.snapshot).toEqual(SNAPSHOT) + }) + + it('clears the rollback target after rolling back, so it cannot walk into the bad build', () => { + const before = { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + snapshot: SNAPSHOT + } + expect(withRolledBackVersion(before, NOW)).toMatchObject({ + active: '0.1.0+aa01', + previous: null, + snapshot: null + }) + }) + + it('pins the active version, the rollback target and the live daemon"s bundle against GC', () => { + const pinned = orcadGcPinnedDirNames( + { + ...emptyOrcadActivationRecord(), + active: '0.3.0+cc01', + previous: '0.2.0+bb01' + }, + '0.1.0+aa01' + ) + expect(pinned).toEqual(['orcad-0.3.0+cc01', 'orcad-0.2.0+bb01', 'orcad-0.1.0+aa01']) + }) + + it('deduplicates pins when the live daemon came from the active bundle', () => { + const pinned = orcadGcPinnedDirNames( + { ...emptyOrcadActivationRecord(), active: '0.3.0+cc01', previous: null }, + '0.3.0+cc01' + ) + expect(pinned).toEqual(['orcad-0.3.0+cc01']) + }) +}) diff --git a/src/main/ssh/orcad-activation-record.ts b/src/main/ssh/orcad-activation-record.ts new file mode 100644 index 000000000000..8c32e6c355dc --- /dev/null +++ b/src/main/ssh/orcad-activation-record.ts @@ -0,0 +1,175 @@ +/** + * Which installed orcad is the live one, and which one a rollback goes back to. + * + * A versioned install directory decides where bytes land; it does not decide which version + * runs. Without this record, "roll back" means "deploy the old version again" — which needs + * the client that has those bytes, on a host that may be the only thing still working. The + * record is the host-side half: it names an active version, a rollback target, and the + * pre-activation state snapshot that makes going back to that target sound. + * + * It lives beside the version dirs (`~/.orca-remote/orcad-active.json`), not inside one, + * because it has to outlive whichever version GC removes. + */ +import { remoteInstallDirName, ORCAD_INSTALL_MODEL } from './remote-install-model' + +export const ORCAD_ACTIVATION_FILENAME = 'orcad-active.json' +export const ORCAD_ACTIVATION_SCHEMA_VERSION = 1 + +/** Where a pre-activation copy of the shared data root lives, relative to `.orca-remote/`. */ +export const ORCAD_STATE_SNAPSHOT_DIR = 'orcad-state-snapshots' + +export type OrcadStateSnapshot = { + /** Directory name under `ORCAD_STATE_SNAPSHOT_DIR`. */ + dirName: string + /** The version whose activation this snapshot was taken FOR — i.e. taken before it ran. */ + takenBeforeVersion: string + /** The version that produced the state, i.e. the rollback target it is readable by. */ + readableByVersion: string | null + takenAt: string +} + +export type OrcadActivationRecord = { + schemaVersion: typeof ORCAD_ACTIVATION_SCHEMA_VERSION + /** Full content-hashed version, e.g. `0.1.0+9f2a1c`. Null before the first activation. */ + active: string | null + /** The version `active` replaced. The rollback target, and pinned against GC. */ + previous: string | null + activatedAt: string | null + snapshot: OrcadStateSnapshot | null +} + +export function emptyOrcadActivationRecord(): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: null, + previous: null, + activatedAt: null, + snapshot: null + } +} + +/** + * Parse the record read off the host. + * + * Why a null return and not a throw on a newer schema: a client older than the host must not + * treat "I cannot read this" as "nothing is activated" — that would deploy over a live + * install. Callers distinguish the two through `OrcadActivationReadResult`. + */ +export type OrcadActivationReadResult = + | { state: 'absent' } + | { state: 'ok'; record: OrcadActivationRecord } + | { state: 'unreadable'; reason: string } + +export function parseOrcadActivationRecord(raw: string | null): OrcadActivationReadResult { + if (raw === null || raw.trim() === '') { + return { state: 'absent' } + } + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch (error) { + return { + state: 'unreadable', + reason: `activation record is not JSON: ${error instanceof Error ? error.message : String(error)}` + } + } + if (typeof parsed !== 'object' || parsed === null) { + return { state: 'unreadable', reason: 'activation record is not an object' } + } + const record = parsed as Partial + if (record.schemaVersion !== ORCAD_ACTIVATION_SCHEMA_VERSION) { + return { + state: 'unreadable', + reason: + `activation record schemaVersion ${String(record.schemaVersion)} is not ` + + `${ORCAD_ACTIVATION_SCHEMA_VERSION}; this client cannot safely interpret it` + } + } + return { + state: 'ok', + record: { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: typeof record.active === 'string' ? record.active : null, + previous: typeof record.previous === 'string' ? record.previous : null, + activatedAt: typeof record.activatedAt === 'string' ? record.activatedAt : null, + snapshot: parseSnapshot(record.snapshot) + } + } +} + +function parseSnapshot(value: unknown): OrcadStateSnapshot | null { + if (typeof value !== 'object' || value === null) { + return null + } + const snapshot = value as Partial + if (typeof snapshot.dirName !== 'string' || typeof snapshot.takenBeforeVersion !== 'string') { + return null + } + return { + dirName: snapshot.dirName, + takenBeforeVersion: snapshot.takenBeforeVersion, + readableByVersion: + typeof snapshot.readableByVersion === 'string' ? snapshot.readableByVersion : null, + takenAt: typeof snapshot.takenAt === 'string' ? snapshot.takenAt : '' + } +} + +export function serializeOrcadActivationRecord(record: OrcadActivationRecord): string { + return `${JSON.stringify(record, null, 2)}\n` +} + +/** The record that results from activating `version`, keeping the outgoing one as the target. */ +export function withActivatedVersion( + record: OrcadActivationRecord, + version: string, + snapshot: OrcadStateSnapshot | null, + now: Date +): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: version, + // Why keep the OLD previous when re-activating the same version: a repeated deploy of + // an already-active build is not a version change, so it must not erase the rollback + // target by naming the active version as its own predecessor. + previous: record.active === version ? record.previous : record.active, + activatedAt: now.toISOString(), + snapshot: record.active === version ? record.snapshot : snapshot + } +} + +/** The record that results from rolling `active` back to `previous`. */ +export function withRolledBackVersion( + record: OrcadActivationRecord, + now: Date +): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: record.previous, + // Why null and not the version we just left: it is the build we are rolling back FROM, + // so offering it as the next rollback target would walk straight back into the failure. + previous: null, + activatedAt: now.toISOString(), + // The snapshot was taken before `active` ran; once restored it has been consumed. + snapshot: null + } +} + +/** + * Version dirs GC must not remove, as directory names. + * + * `previous` is here because a rollback target that GC deleted is not a rollback target. + * `daemonEntryVersion` is here because an update preserves a live daemon forked from the + * OUTGOING bundle (see orcad-update-plan.ts) — deleting the tree under a running process is + * how a later respawn finds no entry point. + */ +export function orcadGcPinnedDirNames( + record: OrcadActivationRecord, + daemonEntryVersion?: string | null +): string[] { + const versions = [record.active, record.previous, daemonEntryVersion ?? null].filter( + (v): v is string => typeof v === 'string' && v.length > 0 + ) + return [...new Set(versions)].map((version) => + remoteInstallDirName(ORCAD_INSTALL_MODEL, version) + ) +} diff --git a/src/main/ssh/orcad-local-build-hash.ts b/src/main/ssh/orcad-local-build-hash.ts new file mode 100644 index 000000000000..3740ea6d55c5 --- /dev/null +++ b/src/main/ssh/orcad-local-build-hash.ts @@ -0,0 +1,22 @@ +/** + * The build identity the deploy expects the host to answer with. + * + * It must be computed the same way `computeOrcadBuildHash` computes it on the host — + * sha256 of `orcad.js`, first 16 hex characters — or the activation gate would reject every + * healthy candidate. Keeping the two in one comment is deliberate: they are one contract + * split across a network, and the version string cannot stand in for it, because + * `ORCA_VERSION` is whatever the launch command exported and two builds can carry one value. + */ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +export const ORCAD_BUILD_HASH_LENGTH = 16 + +export function computeLocalOrcadBuildHash(localOrcadDir: string): string { + const entry = join(localOrcadDir, 'orcad.js') + return createHash('sha256') + .update(readFileSync(entry)) + .digest('hex') + .slice(0, ORCAD_BUILD_HASH_LENGTH) +} diff --git a/src/main/ssh/orcad-remote-deploy.test.ts b/src/main/ssh/orcad-remote-deploy.test.ts new file mode 100644 index 000000000000..799193f3384f --- /dev/null +++ b/src/main/ssh/orcad-remote-deploy.test.ts @@ -0,0 +1,264 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', () => ({ + acquireInstallLock: vi.fn().mockResolvedValue(undefined), + RELAY_INSTALL_LOCK_NAME: '.install-lock' +})) +vi.mock('./ssh-relay-install-transfers', () => ({ + uploadRelayDirectory: vi.fn().mockResolvedValue(undefined), + writeRelayFile: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./orcad-local-build-hash', () => ({ + computeLocalOrcadBuildHash: () => 'abc123def4567890' +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy' +import { emptyOrcadActivationRecord, withActivatedVersion } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const mockExec = vi.mocked(execCommand) +const NEW_VERSION = '0.2.0+bb01' +const OLD_VERSION = '0.1.0+aa01' + +vi.mock('./ssh-relay-versioned-install', async (importOriginal) => ({ + ...(await importOriginal>()), + readLocalFullVersion: () => '0.2.0+bb01', + isRemoteInstallComplete: vi.fn().mockResolvedValue(false), + finalizeInstall: vi.fn().mockResolvedValue(undefined), + abandonInstall: vi.fn().mockResolvedValue(undefined) +})) + +function readyLine(overrides: { + buildHash?: string + daemonState?: 'live' | 'degraded' | 'absent' + selfTestOk?: boolean +}): string { + return JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: overrides.buildHash ?? 'abc123def4567890', + buildVersion: NEW_VERSION, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: overrides.daemonState ?? 'live', + ownsFreshSessions: (overrides.daemonState ?? 'live') === 'live', + pid: 2, + buildVersion: NEW_VERSION, + entryPath: '/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { + ok: overrides.selfTestOk ?? true, + coverage: 'pty-spawn', + verdict: (overrides.selfTestOk ?? true) ? 'healthy' : 'pty-spawn-unhealthy', + durationMs: 5 + } + } + } + }) +} + +type HostScript = { + activationRecord: string + /** Readiness content per version dir, keyed by the version in the path. */ + readiness: Record + log: string[] +} + +function scriptHost(script: HostScript): void { + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.startsWith('cat ') && text.includes('orcad-active.json')) { + return script.activationRecord + } + if (text.includes('.orcad-readiness') && text.startsWith('cat ')) { + const version = Object.keys(script.readiness).find((v) => text.includes(v)) + return version ? script.readiness[version] : '' + } + if (text.includes('nohup')) { + script.log.push(`launch:${text.includes(NEW_VERSION) ? NEW_VERSION : OLD_VERSION}`) + return '9999' + } + if (text.includes('kill -TERM')) { + script.log.push(`stop:${text.includes(NEW_VERSION) ? NEW_VERSION : OLD_VERSION}`) + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-cf')) { + script.log.push('snapshot') + return 'CAPTURED' + } + return '' + }) +} + +function options(overrides: Partial = {}): OrcadDeployOptions { + return { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + localOrcadDir: '/local/out/orcad', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + census: { liveSessions: 0, startedSinceActivation: 0 }, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z'), + ...overrides + } +} + +const ACTIVE_OLD = JSON.stringify( + withActivatedVersion(emptyOrcadActivationRecord(), OLD_VERSION, null, new Date(0)) +) + +describe('deployOrcad', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('installs under the orcad namespace, not the relay one', async () => { + const script: HostScript = { + activationRecord: '', + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await deployOrcad(options()) + expect(vi.mocked(acquireInstallLock).mock.calls[0][1]).toBe( + `/home/u/.orca-remote/orcad-${NEW_VERSION}` + ) + expect(vi.mocked(uploadRelayDirectory).mock.calls[0][2]).toContain(`orcad-${NEW_VERSION}`) + }) + + it('activates a healthy candidate and records the outgoing version as the rollback target', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ outcome: 'installed-and-activated', fullVersion: NEW_VERSION }) + const written = vi + .mocked(writeRelayFile) + .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[3]))).toMatchObject({ + active: NEW_VERSION, + previous: OLD_VERSION + }) + }) + + it('snapshots the shared data root before the candidate ever runs', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await deployOrcad(options()) + expect(script.log.indexOf('snapshot')).toBeGreaterThan(-1) + expect(script.log.indexOf('snapshot')).toBeLessThan(script.log.indexOf(`launch:${NEW_VERSION}`)) + }) + + it('installs but does not activate when terminals are running', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad( + options({ census: { liveSessions: 2, startedSinceActivation: 0 } }) + ) + expect(result).toMatchObject({ + outcome: 'installed-not-activated', + code: 'orcad_update_terminals_running' + }) + // The bytes landed; nothing was stopped, launched or snapshotted. + expect(vi.mocked(uploadRelayDirectory)).toHaveBeenCalled() + expect(script.log).toEqual([]) + }) + + it('does not write the activation record when the candidate fails its health gate', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { [NEW_VERSION]: readyLine({ daemonState: 'degraded' }) }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' }) + expect( + vi.mocked(writeRelayFile).mock.calls.some((call) => String(call[2]).endsWith('orcad-active.json')) + ).toBe(false) + }) + + it('puts the previous version back after a rejected candidate, rather than leaving the host down', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { + [NEW_VERSION]: readyLine({ selfTestOk: false }), + [OLD_VERSION]: readyLine({}) + }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ outcome: 'installed-not-activated' }) + expect(script.log).toEqual([ + 'snapshot', + `stop:${OLD_VERSION}`, + `launch:${NEW_VERSION}`, + `stop:${NEW_VERSION}`, + `launch:${OLD_VERSION}` + ]) + expect(result.outcome === 'installed-not-activated' && result.reason).toContain( + `orcad ${OLD_VERSION} was restarted and is serving again` + ) + }) + + it('refuses to activate when a different build answered the port', async () => { + const script: HostScript = { + activationRecord: ACTIVE_OLD, + readiness: { + [NEW_VERSION]: readyLine({ buildHash: 'deadbeefdeadbeef' }), + [OLD_VERSION]: readyLine({}) + }, + log: [] + } + scriptHost(script) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ code: 'orcad_activation_build_mismatch' }) + }) + + it('refuses to treat an unreadable activation record as an empty one', async () => { + const script: HostScript = { + activationRecord: JSON.stringify({ schemaVersion: 99, active: 'x' }), + readiness: { [NEW_VERSION]: readyLine({}) }, + log: [] + } + scriptHost(script) + await expect(deployOrcad(options())).rejects.toThrow('activation record') + expect(vi.mocked(uploadRelayDirectory)).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/orcad-remote-deploy.ts b/src/main/ssh/orcad-remote-deploy.ts new file mode 100644 index 000000000000..1f8a4099a826 --- /dev/null +++ b/src/main/ssh/orcad-remote-deploy.ts @@ -0,0 +1,340 @@ +/** + * Installing orcad on a host and, only if it proves itself, making it the active one. + * + * The install half is the relay's transaction, parameterized: the same per-version lock, + * staged SFTP write, `.install-complete` sentinel and stale-lock recovery, under + * `orcad-/` instead of `relay-/`. That is what §02 marks reusable. + * + * The activation half has no relay equivalent, because the relay has no notion of a version + * being *selected*. Bytes landing in a versioned directory neither picks a version nor rolls + * one back; the activation record does, and it is written only after the candidate publishes + * a health payload that survives `evaluateOrcadActivation`. A rejected candidate leaves the + * previous version running and its own bytes on disk — nothing is lost, and a retry costs no + * upload. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { + abandonInstall, + computeRemoteInstallDir, + finalizeInstall, + isRemoteInstallComplete, + readLocalFullVersion +} from './ssh-relay-versioned-install' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_STATE_SNAPSHOT_DIR, + serializeOrcadActivationRecord, + withActivatedVersion, + type OrcadActivationRecord, + type OrcadStateSnapshot +} from './orcad-activation-record' +import { + orcadActivationPath, + readOrcadActivationRecord +} from './orcad-activation-record-store' +import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' +import { planOrcadUpdate, type OrcadTerminalCensus } from './orcad-update-plan' +import { + ORCAD_LOG_FILENAME, + orcadLaunchCommand, + parseOrcadReadinessOutput, + readOrcadReadinessCommand, + type OrcadLaunchSpec +} from './orcad-remote-launch' +import { + captureOrcadStateSnapshotCommand, + orcadSnapshotDirName, + parseOrcadSnapshotCapture +} from './orcad-state-snapshot' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { computeLocalOrcadBuildHash } from './orcad-local-build-hash' + +export type OrcadDeployOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + /** Local `out/orcad`, containing the artifacts and the `.version` marker. */ + localOrcadDir: string + nodePath: string + userDataDir: string + bindHost: string + port: number + /** + * Live-terminal counts, supplied by the caller from the runtime it is already connected + * to. Not probed here: counting the daemon's sessions needs its protocol, and a deploy + * that guessed zero from silence would be the "loss of contact means death" mistake. + */ + census: OrcadTerminalCensus + force?: boolean + readinessTimeoutMs?: number + now?: () => Date + sleep?: (ms: number) => Promise + signal?: AbortSignal +} + +export type OrcadDeployResult = + | { outcome: 'installed-and-activated'; fullVersion: string; verdict: OrcadActivationVerdict } + | { outcome: 'already-active'; fullVersion: string } + | { outcome: 'installed-not-activated'; fullVersion: string; code: string; reason: string } + +const DEFAULT_READINESS_TIMEOUT_MS = 90_000 +const READINESS_POLL_MS = 500 +const STOP_WAIT_SECONDS = 20 + +function exec( + options: OrcadDeployOptions, + command: string, + signal = options.signal +): Promise { + return execCommand(options.conn, command, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal + }) +} + +function baseDir(options: OrcadDeployOptions): string { + return joinRemotePath(options.host, options.remoteHome, RELAY_REMOTE_DIR) +} + +/** Install the bytes under `orcad-/`, using the relay's install transaction. */ +async function installOrcadBundle( + options: OrcadDeployOptions, + fullVersion: string, + remoteDir: string +): Promise { + if (await isRemoteInstallComplete(options.conn, ORCAD_INSTALL_MODEL, remoteDir, options.host, { + signal: options.signal + })) { + return + } + await acquireInstallLock(options.conn, remoteDir, options.host, { signal: options.signal }) + try { + // Re-probe under the lock: a sibling deploy may have finished while we waited. + if ( + await isRemoteInstallComplete(options.conn, ORCAD_INSTALL_MODEL, remoteDir, options.host, { + signal: options.signal + }) + ) { + return + } + await uploadRelayDirectory(options.conn, options.localOrcadDir, remoteDir, options.host, { + signal: options.signal + }) + await writeRelayFile( + options.conn, + options.host, + joinRemotePath(options.host, remoteDir, ORCAD_INSTALL_MODEL.versionFilename), + fullVersion, + { signal: options.signal } + ) + await finalizeInstall(options.conn, remoteDir, options.host, { signal: options.signal }) + } catch (error) { + // Leave a recoverable partial rather than a dir that probes complete. + await abandonInstall(options.conn, remoteDir, options.host) + throw error + } +} + +async function captureSnapshot( + options: OrcadDeployOptions, + fullVersion: string, + outgoingVersion: string | null, + takenAt: Date +): Promise { + const dirName = orcadSnapshotDirName(fullVersion, takenAt.getTime()) + const snapshotDir = joinRemotePath(options.host, baseDir(options), ORCAD_STATE_SNAPSHOT_DIR, dirName) + const capture = parseOrcadSnapshotCapture( + await exec( + options, + captureOrcadStateSnapshotCommand(options.host, options.userDataDir, snapshotDir) + ) + ) + if (capture === 'failed') { + throw new Error( + `Could not snapshot ${options.userDataDir} before activating ${fullVersion}. Orca's ` + + 'persisted state carries no schema version, so without a snapshot a rollback has no ' + + 'way back. Refusing to activate.' + ) + } + if (capture === 'empty') { + // Nothing on the host to lose: a first deployment. Rollback will correctly report that + // it has no snapshot, rather than restoring an archive of nothing over a populated root. + return null + } + return { + dirName, + takenBeforeVersion: fullVersion, + readableByVersion: outgoingVersion, + takenAt: takenAt.toISOString() + } +} + +async function launchAndAwaitReadiness( + options: OrcadDeployOptions, + spec: OrcadLaunchSpec +): Promise> { + await exec(options, orcadLaunchCommand(options.host, spec)) + const deadline = Date.now() + (options.readinessTimeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS) + const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) + let last = parseOrcadReadinessOutput('') + while (Date.now() < deadline) { + options.signal?.throwIfAborted() + last = parseOrcadReadinessOutput( + await exec(options, readOrcadReadinessCommand(options.host, spec.remoteInstallDir)) + ) + if (last.state !== 'pending') { + return last + } + await sleep(READINESS_POLL_MS) + } + return last +} + +/** + * Put the previous version back after a rejected candidate. + * + * Why this exists at all: activating means swapping which process owns the data root and the + * port, so the incumbent has to stop before the candidate can start. A gate that rejected + * and returned would leave the host with nothing running — a careful deploy causing the + * outage it was being careful about. The returned sentence goes into the caller's reason so + * the operator learns the host's actual state, not just why the candidate failed. + */ +async function restoreIncumbent( + options: OrcadDeployOptions, + record: OrcadActivationRecord, + candidateDir: string +): Promise { + const stopped = parseOrcadStopOutcome( + await exec( + options, + stopOrcadCommand(options.host, candidateDir, { waitSeconds: STOP_WAIT_SECONDS }) + ) + ) + if (!orcadStopFreedTheHost(stopped)) { + return `The candidate itself did not stop (${stopped}); the host may still be serving the rejected build.` + } + if (!record.active) { + return 'No previous version was active, so this host is now serving nothing.' + } + const incumbentDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + record.active + ) + const parsed = await launchAndAwaitReadiness(options, { + remoteInstallDir: incumbentDir, + nodePath: options.nodePath, + fullVersion: record.active, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + return parsed.state === 'ready' + ? `orcad ${record.active} was restarted and is serving again.` + : `orcad ${record.active} was relaunched but has not published readiness; this host may be down.` +} + +/** + * Install, then activate only on a green cross-process health verdict. + * + * Every early return past the install leaves the bytes on disk and the previous version + * serving, which is why they all report `installed-not-activated` rather than throwing: a + * refusal to switch is a successful outcome of a deploy that was asked to be careful. + */ +export async function deployOrcad(options: OrcadDeployOptions): Promise { + const now = options.now ?? ((): Date => new Date()) + const fullVersion = readLocalFullVersion(options.localOrcadDir) + const remoteDir = computeRemoteInstallDir(ORCAD_INSTALL_MODEL, options.remoteHome, fullVersion) + const record = await readOrcadActivationRecord(options) + + await installOrcadBundle(options, fullVersion, remoteDir) + + const plan = planOrcadUpdate({ + record, + candidateVersion: fullVersion, + census: options.census, + ...(options.force !== undefined ? { force: options.force } : {}) + }) + if (plan.action === 'noop') { + return { outcome: 'already-active', fullVersion } + } + if (plan.action === 'defer') { + return { + outcome: 'installed-not-activated', + fullVersion, + code: plan.code, + reason: plan.reason + } + } + + const snapshot = record.active + ? await captureSnapshot(options, fullVersion, record.active, now()) + : null + + if (record.active) { + const outgoingDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + record.active + ) + const stopped = parseOrcadStopOutcome( + await exec(options, stopOrcadCommand(options.host, outgoingDir, { + waitSeconds: STOP_WAIT_SECONDS + })) + ) + if (!orcadStopFreedTheHost(stopped)) { + return { + outcome: 'installed-not-activated', + fullVersion, + code: 'orcad_outgoing_stop_incomplete', + reason: + `orcad ${record.active} did not exit within ${STOP_WAIT_SECONDS}s of SIGTERM ` + + `(${stopped}). It is still holding the data root and the port, so the candidate ` + + 'cannot start. Not escalating to SIGKILL: that skips the shutdown that releases ' + + 'the instance lock, and the successor would then refuse to start.' + } + } + } + + const parsed = await launchAndAwaitReadiness(options, { + remoteInstallDir: remoteDir, + nodePath: options.nodePath, + fullVersion, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { + buildHash: computeLocalOrcadBuildHash(options.localOrcadDir), + fullVersion + }) + if (verdict.decision === 'reject') { + const restored = await restoreIncumbent(options, record, remoteDir) + return { + outcome: 'installed-not-activated', + fullVersion, + code: verdict.code, + reason: + `${verdict.reason} Candidate stderr is at ` + + `${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}` + } + } + + await writeRelayFile( + options.conn, + options.host, + orcadActivationPath(options.host, options.remoteHome), + serializeOrcadActivationRecord(withActivatedVersion(record, fullVersion, snapshot, now())), + { signal: options.signal } + ) + return { outcome: 'installed-and-activated', fullVersion, verdict } +} diff --git a/src/main/ssh/orcad-remote-gc.test.ts b/src/main/ssh/orcad-remote-gc.test.ts new file mode 100644 index 000000000000..3716aaddaf99 --- /dev/null +++ b/src/main/ssh/orcad-remote-gc.test.ts @@ -0,0 +1,153 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-gc-claim', () => ({ + isRelayGcClaimOwned: vi.fn().mockResolvedValue(true), + releaseRelayGcClaimWithRetry: vi.fn().mockResolvedValue('released'), + tryAcquireRelayGcClaim: vi.fn().mockResolvedValue('token') +})) +vi.mock('./ssh-relay-gc-tombstone', () => ({ + cleanupRelayGcTombstones: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./ssh-relay-install-lock', () => ({ + RELAY_INSTALL_LOCK_NAME: '.install-lock', + isRelayInstallLockStale: vi.fn().mockResolvedValue(false) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { gcOldOrcadVersions } from './orcad-remote-gc' +import { emptyOrcadActivationRecord } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const conn = {} as SshConnection +const host = getRemoteHostPlatform('linux-x64') +const mockExec = vi.mocked(execCommand) + +/** + * Drive one GC pass against a scripted host. `listing` is what the remote listing command + * returns; every other command answers from `responses`, defaulting to the happy path + * (unlocked, complete, dead) so a candidate is removed unless a test says otherwise. + */ +function scriptHost(options: { + listing: string[] + liveness?: Record + removed: string[] +}): void { + mockExec.mockImplementation(async (_conn, command: string) => { + if (command.includes('-mindepth 1 -maxdepth 1')) { + return options.listing.join('\n') + } + if (command.includes('.install-lock')) { + return 'OPEN' + } + if (command.includes('.install-complete')) { + return 'COMPLETE' + } + if (command.includes('.orcad-pid')) { + const dir = Object.keys(options.liveness ?? {}).find((name) => command.includes(name)) + return dir ? (options.liveness?.[dir] ?? 'DEAD') : 'DEAD' + } + if (command.startsWith('mv ')) { + const match = command.match(/'([^']*)'/) + if (match) { + options.removed.push(match[1].split('/').pop() ?? '') + } + return 'MOVED' + } + return '' + }) +} + +describe('orcad GC', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('scopes its remote listing to the orcad namespace', async () => { + const removed: string[] = [] + scriptHost({ listing: [], removed }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb', + record: emptyOrcadActivationRecord() + }) + const listCommand = mockExec.mock.calls.map((call) => String(call[1])).find((c) => c.includes('find')) + expect(listCommand).toContain("-name 'orcad-*'") + expect(listCommand).not.toContain("-name 'relay-*'") + }) + + it('leaves relay directories alone even when the host hands them to it', async () => { + // The host is lying — or a future listing bug widened the glob. GC must still refuse. + const removed: string[] = [] + scriptHost({ + listing: ['relay-0.1.0+aa', 'relay-0.0.9+ff', 'orcad-0.1.0+aa'], + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb', + record: emptyOrcadActivationRecord() + }) + expect(removed).toEqual(['orcad-0.1.0+aa']) + }) + + it('never removes the active or the previous version', async () => { + const removed: string[] = [] + scriptHost({ + listing: ['orcad-0.1.0+01d', 'orcad-0.2.0+9ee0', 'orcad-0.3.0+cc0'], + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: { + ...emptyOrcadActivationRecord(), + active: '0.3.0+cc0', + previous: '0.2.0+9ee0' + } + }) + expect(removed).toEqual(['orcad-0.1.0+01d']) + }) + + it('never removes the version a live daemon was forked from', async () => { + const removed: string[] = [] + scriptHost({ listing: ['orcad-0.1.0+01d', 'orcad-0.0.9+01de'], removed }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: { ...emptyOrcadActivationRecord(), active: '0.3.0+cc0' }, + liveDaemonVersion: '0.1.0+01d' + }) + expect(removed).toEqual(['orcad-0.0.9+01de']) + }) + + it('treats an unanswerable liveness probe as in use', async () => { + const removed: string[] = [] + scriptHost({ + listing: ['orcad-0.1.0+bb0', 'orcad-0.0.9+dead'], + liveness: { 'orcad-0.1.0+bb0': 'UNKNOWN', 'orcad-0.0.9+dead': 'DEAD' }, + removed + }) + await gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record: emptyOrcadActivationRecord() + }) + expect(removed).toEqual(['orcad-0.0.9+dead']) + }) +}) diff --git a/src/main/ssh/orcad-remote-gc.ts b/src/main/ssh/orcad-remote-gc.ts new file mode 100644 index 000000000000..a96b386fc1bb --- /dev/null +++ b/src/main/ssh/orcad-remote-gc.ts @@ -0,0 +1,74 @@ +/** + * orcad's garbage collection, and the half of §06 falsifier 1 that says who owns it. + * + * **Each model GCs only its own namespace, permanently.** orcad removes `orcad-/` + * directories; the relay removes `relay-/` directories; neither ever removes the other's, + * and no plan item makes one the winner. That is not a migration compromise — the two models + * serve different users on the same machine (SSH target vs paired peer), so there is no + * moment at which one of them is entitled to clean up after the other. A pass that deleted + * the sibling's tree would be reaching across the execution boundary the whole design exists + * to keep intact. + * + * On top of the ownership rule, orcad pins three directories that are idle-looking but + * load-bearing: the active version, the rollback target, and whichever version the LIVE + * terminal daemon was forked from. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { gcOldRemoteInstallVersions } from './ssh-relay-versioned-install' +import { orcadGcPinnedDirNames, type OrcadActivationRecord } from './orcad-activation-record' +import { + orcadLivenessBlocksGc, + orcadLivenessProbeCommand, + parseOrcadLiveness +} from './orcad-remote-launch' +import type { RemoteHostPlatform } from './ssh-remote-platform' + +export type OrcadGcOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + /** Absolute path of the version dir this client just used; never a candidate. */ + currentDirAbsPath: string + record: OrcadActivationRecord + /** + * The full version the live daemon's PID record names, when it can be read. + * + * An update preserves a daemon forked from the OUTGOING bundle whenever terminals are + * live, so this is routinely a version that is neither active nor previous. Deleting it + * would remove the tree under a running process. + */ + liveDaemonVersion?: string | null + signal?: AbortSignal +} + +export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise { + await gcOldRemoteInstallVersions( + options.conn, + ORCAD_INSTALL_MODEL, + options.remoteHome, + options.currentDirAbsPath, + options.host, + { + pinnedDirNames: orcadGcPinnedDirNames(options.record, options.liveDaemonVersion), + isDirLive: async (dir) => { + try { + const probe = await execCommand( + options.conn, + orcadLivenessProbeCommand(options.host, dir), + { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + } + ) + return orcadLivenessBlocksGc(parseOrcadLiveness(probe)) + } catch { + // Why true: an unanswered probe is not evidence a tree is idle. Same rule the + // relay's socket probe applies, for the same reason. + return true + } + } + } + ) +} diff --git a/src/main/ssh/orcad-remote-host-support.ts b/src/main/ssh/orcad-remote-host-support.ts new file mode 100644 index 000000000000..dfcb0f4b6e4d --- /dev/null +++ b/src/main/ssh/orcad-remote-host-support.ts @@ -0,0 +1,50 @@ +/** + * Which hosts the orcad launch/lifecycle path actually supports, declared rather than + * discovered at runtime. + * + * The install transaction is host-agnostic — it is the relay's, and the relay runs on + * Windows. The launch, liveness and stop path is not: it uses `nohup`, a redirected stdout, + * `kill -0` and `ps`. Emitting a PowerShell-shaped approximation of that would produce a + * deploy that reports success on a host where nothing is running. + */ +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' + +export class OrcadRemoteLaunchUnsupportedError extends Error { + readonly code = 'orcad_remote_launch_unsupported_host' + constructor(hostLabel: string) { + super( + `Deploying orcad to a ${hostLabel} host is not implemented. The install transaction is ` + + 'host-agnostic, but the launch and readiness path is POSIX-only: it uses nohup, a ' + + 'redirected stdout and `kill -0` liveness. Use the relay for this host.' + ) + this.name = 'OrcadRemoteLaunchUnsupportedError' + } +} + +export function assertPosixOrcadHost(host: RemoteHostPlatform): void { + if (isWindowsRemoteHost(host)) { + throw new OrcadRemoteLaunchUnsupportedError('Windows') + } +} + +/** PID of the launched orcad, written into its own version dir at launch. */ +export const ORCAD_PID_FILENAME = '.orcad-pid' + +/** + * A shell function answering whether a PID is a *running* process. + * + * `kill -0` alone is not that question. It succeeds for a zombie — a process that has + * exited but whose parent has not reaped it — so a stop loop built on it reports + * `STILL_RUNNING` for a process that is already gone, and GC reports a dead version dir as + * in use. Verified against a real zombie on macOS; the `ps` state check is what separates + * the two. + * + * A host without `ps` yields an empty state, which falls through to "alive" — the safe + * direction for both callers. + */ +export function posixProcessAliveShellFunction(): string { + return ( + 'orcad_alive() { kill -0 "$1" 2>/dev/null || return 1; ' + + 'case "$(ps -o stat= -p "$1" 2>/dev/null)" in Z*) return 1;; esac; return 0; };' + ) +} diff --git a/src/main/ssh/orcad-remote-launch.test.ts b/src/main/ssh/orcad-remote-launch.test.ts new file mode 100644 index 000000000000..068ae588f657 --- /dev/null +++ b/src/main/ssh/orcad-remote-launch.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest' + +import { + ORCAD_READINESS_FILENAME, + orcadLaunchCommand, + orcadLivenessBlocksGc, + orcadLivenessProbeCommand, + OrcadRemoteLaunchUnsupportedError, + parseOrcadLiveness, + parseOrcadReadinessOutput +} from './orcad-remote-launch' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') + +const SPEC = { + remoteInstallDir: '/home/u/.orca-remote/orcad-0.2.0+bb01', + nodePath: '/usr/bin/node', + fullVersion: '0.2.0+bb01', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777 +} + +const READY_LINE = JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { buildHash: 'abc', terminalDaemon: { state: 'live' } } +}) + +describe('orcadLaunchCommand', () => { + it('states the bind posture rather than inheriting the build default', () => { + expect(orcadLaunchCommand(posix, SPEC)).toContain("--bind '127.0.0.1'") + }) + + it('truncates the readiness file, so a stale line cannot be activated on', () => { + const command = orcadLaunchCommand(posix, SPEC) + const truncate = command.indexOf(`: > '${SPEC.remoteInstallDir}/${ORCAD_READINESS_FILENAME}'`) + const launch = command.indexOf('nohup') + expect(truncate).toBeGreaterThan(-1) + expect(truncate).toBeLessThan(launch) + }) + + it('exports the version and the shared data root the deploy decided on', () => { + const command = orcadLaunchCommand(posix, SPEC) + expect(command).toContain(`ORCA_VERSION '${SPEC.fullVersion}'`.replace(' ', '=')) + expect(command).toContain(`ORCA_USER_DATA='${SPEC.userDataDir}'`) + }) + + it('declares the Windows refusal instead of emitting a command that cannot work', () => { + expect(() => orcadLaunchCommand(windows, SPEC)).toThrow(OrcadRemoteLaunchUnsupportedError) + }) +}) + +describe('readiness parsing', () => { + it('extracts the orca_server_ready payload', () => { + const parsed = parseOrcadReadinessOutput(`${READY_LINE}\n`) + expect(parsed).toMatchObject({ state: 'ready' }) + expect(parsed.state === 'ready' && parsed.readiness.boundEndpoint).toBe('ws://127.0.0.1:7777') + }) + + it('treats an empty or half-written file as pending, not as a failure', () => { + expect(parseOrcadReadinessOutput('')).toEqual({ state: 'pending' }) + expect(parseOrcadReadinessOutput('{"type":"orca_serv')).toEqual({ state: 'pending' }) + }) + + it('reports a complete JSON line that is not a readiness payload as malformed', () => { + expect(parseOrcadReadinessOutput('{"type":"something_else"}')).toMatchObject({ + state: 'malformed' + }) + }) +}) + +describe('liveness', () => { + it('reads the pid recorded in the version dir', () => { + expect(orcadLivenessProbeCommand(posix, SPEC.remoteInstallDir)).toContain('.orcad-pid') + }) + + it.each([ + ['LIVE', 'LIVE', true], + ['DEAD', 'DEAD', false], + ['', 'UNKNOWN', true], + ['garbage', 'UNKNOWN', true] + ])('parses %s and blocks GC = %s', (output, expected, blocks) => { + expect(parseOrcadLiveness(output)).toBe(expected) + expect(orcadLivenessBlocksGc(parseOrcadLiveness(output))).toBe(blocks) + }) +}) + +describe('stopping a running orcad', () => { + it('sends SIGTERM and never SIGKILL', () => { + const command = stopOrcadCommand(posix, SPEC.remoteInstallDir, { waitSeconds: 20 }) + expect(command).toContain('kill -TERM') + for (const kill of ['kill -9', 'kill -KILL', 'kill -SIGKILL', 'pkill']) { + expect(command).not.toContain(kill) + } + }) + + it.each([ + ['STOPPED', 'stopped', true], + ['ALREADY_EXITED', 'already-exited', true], + ['NO_PID', 'no-pid', true], + ['STILL_RUNNING', 'still-running', false], + ['SIGNAL_FAILED', 'signal-failed', false], + ['', 'unknown', false] + ])('parses %s and frees the host = %s', (output, expected, frees) => { + expect(parseOrcadStopOutcome(output)).toBe(expected) + expect(orcadStopFreedTheHost(parseOrcadStopOutcome(output))).toBe(frees) + }) +}) diff --git a/src/main/ssh/orcad-remote-launch.ts b/src/main/ssh/orcad-remote-launch.ts new file mode 100644 index 000000000000..e82be95faec4 --- /dev/null +++ b/src/main/ssh/orcad-remote-launch.ts @@ -0,0 +1,178 @@ +/** + * Starting a candidate orcad on the host and reading back what it says about itself. + * + * This is the piece `docs/design/shipping-orcad.html` §02 marks **fork**, not reuse: the + * relay launches detached and proves itself by printing an `ORCA-RELAY` sentinel, and orcad + * has no such interface. It publishes a single `orca_server_ready` JSON line on stdout, + * carrying the health payload activation is gated on — so the handshake here is "capture + * that line", not "match a marker". + * + * The candidate is launched detached with stdout redirected to a file inside its own version + * directory. Reading readiness off the exec channel would mean holding the channel open for + * the process's whole life; redirecting means the deploy can disconnect and the supervisor + * still owns a running service. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + assertPosixOrcadHost as assertPosixHost, + ORCAD_PID_FILENAME, + posixProcessAliveShellFunction +} from './orcad-remote-host-support' +import type { ServeReadiness } from '../server/serve-readiness' + +/** Stdout of the launched candidate: exactly one `orca_server_ready` line, then nothing. */ +export const ORCAD_READINESS_FILENAME = '.orcad-readiness' +/** Stderr, including the bind-exposure line and every supervision message. */ +export const ORCAD_LOG_FILENAME = 'orcad.log' +export { + ORCAD_PID_FILENAME, + OrcadRemoteLaunchUnsupportedError +} from './orcad-remote-host-support' + +export type OrcadLaunchSpec = { + remoteInstallDir: string + nodePath: string + fullVersion: string + /** Shared across versions, and the reason rollback needs a snapshot. */ + userDataDir: string + /** Loopback by default; the client reaches it through an SSH local port-forward. */ + bindHost: string + port: number +} + +/** + * Launch the candidate detached and echo its PID. + * + * Why `--bind` is always passed explicitly: orcad defaults to loopback, but a default is a + * thing a future version can change. The deploy states the posture it intends rather than + * inheriting whatever the installed build happens to default to. + */ +export function orcadLaunchCommand(host: RemoteHostPlatform, spec: OrcadLaunchSpec): string { + assertPosixHost(host) + const dir = shellEscape(spec.remoteInstallDir) + const readiness = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_READINESS_FILENAME)) + const log = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_LOG_FILENAME)) + const pidFile = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_PID_FILENAME)) + const entry = shellEscape(joinRemotePath(host, spec.remoteInstallDir, 'orcad.js')) + return [ + `cd ${dir} &&`, + // Why truncate: a re-launch into a dir that already holds a previous readiness line would + // otherwise let the deploy activate on the OLD process's health payload. + `: > ${readiness} &&`, + 'umask 077 &&', + `ORCA_VERSION=${shellEscape(spec.fullVersion)}`, + `ORCA_USER_DATA=${shellEscape(spec.userDataDir)}`, + `nohup ${shellEscape(spec.nodePath)} ${entry}`, + `--json --bind ${shellEscape(spec.bindHost)} --port ${String(spec.port)}`, + `> ${readiness} 2>> ${log} < /dev/null &`, + `echo $! > ${pidFile} && cat ${pidFile}` + ].join(' ') +} + +export function readOrcadReadinessCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + assertPosixHost(host) + const readiness = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME)) + return `cat ${readiness} 2>/dev/null || true` +} + +/** + * Is the process recorded in this version dir still running? + * + * Answers `LIVE`, `DEAD`, or `UNKNOWN`. `UNKNOWN` covers a missing or unparseable PID file + * and a `kill -0` that failed for a reason other than "no such process" — a permission + * error means someone else's process holds that PID, which is not evidence of death. + */ +export function orcadLivenessProbeCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + assertPosixHost(host) + const pidFile = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_PID_FILENAME)) + return [ + posixProcessAliveShellFunction(), + `pid=$(cat ${pidFile} 2>/dev/null);`, + 'case "$pid" in', + '"" ) echo UNKNOWN;;', + '*[!0-9]* ) echo UNKNOWN;;', + // Why EPERM is LIVE and not DEAD: a permission error means some process holds that PID, + // and deleting a tree because we could not signal its owner is the wrong direction. + '* ) if orcad_alive "$pid"; then echo LIVE;', + 'elif kill -0 "$pid" 2>&1 | grep -qi "not permitted"; then echo LIVE;', + 'else echo DEAD; fi;;', + 'esac' + ].join(' ') +} + +export type OrcadLiveness = 'LIVE' | 'DEAD' | 'UNKNOWN' + +export function parseOrcadLiveness(output: string): OrcadLiveness { + const value = output.trim().split('\n').pop()?.trim() + return value === 'LIVE' || value === 'DEAD' ? value : 'UNKNOWN' +} + +/** True when GC must leave this directory alone. Inconclusive counts as in use. */ +export function orcadLivenessBlocksGc(liveness: OrcadLiveness): boolean { + return liveness !== 'DEAD' +} + +export type OrcadReadinessParse = + | { state: 'ready'; readiness: ServeReadiness } + | { state: 'pending' } + | { state: 'malformed'; reason: string } + +/** + * Pull the `orca_server_ready` payload out of whatever the candidate has written so far. + * + * Why scan for the type tag rather than parsing the last line: stdout is a file being + * appended to, so a poll can catch a half-written line. A partial JSON line is `pending`, + * not `malformed` — reporting a parse failure for a race would fail deploys that were fine. + */ +export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse { + const lines = raw.split('\n') + let sawCandidate = false + for (const line of lines) { + const trimmed = line.trim() + if (!trimmed.startsWith('{')) { + continue + } + sawCandidate = true + let parsed: unknown + try { + parsed = JSON.parse(trimmed) + } catch { + continue + } + if (typeof parsed !== 'object' || parsed === null) { + continue + } + const payload = parsed as { type?: unknown } + if (payload.type !== 'orca_server_ready') { + return { + state: 'malformed', + reason: `expected an orca_server_ready line, got type=${JSON.stringify(payload.type)}` + } + } + return { state: 'ready', readiness: toServeReadiness(payload as Record) } + } + return sawCandidate ? { state: 'pending' } : { state: 'pending' } +} + +function toServeReadiness(payload: Record): ServeReadiness { + return { + runtimeId: typeof payload.runtimeId === 'string' ? payload.runtimeId : '', + boundEndpoint: typeof payload.boundEndpoint === 'string' ? payload.boundEndpoint : null, + advertisedEndpoint: + typeof payload.advertisedEndpoint === 'string' ? payload.advertisedEndpoint : null, + managedWslCliReconciliation: + payload.managedWslCliReconciliation === 'pending' || + payload.managedWslCliReconciliation === 'failed' + ? payload.managedWslCliReconciliation + : 'settled', + pairing: payload.pairing as ServeReadiness['pairing'], + ...(payload.health ? { health: payload.health as ServeReadiness['health'] } : {}) + } +} diff --git a/src/main/ssh/orcad-remote-process-control.ts b/src/main/ssh/orcad-remote-process-control.ts new file mode 100644 index 000000000000..6a9038ea3d62 --- /dev/null +++ b/src/main/ssh/orcad-remote-process-control.ts @@ -0,0 +1,73 @@ +/** + * Stopping a running orcad on the host without taking its terminals with it. + * + * `SIGKILL` is absent on purpose. orcad's own escalation contract (`orcad-entry.ts`) is + * SIGTERM, then a second SIGTERM meaning "your deadline elapsed, exit now"; a kill skips the + * teardown that releases the instance lock and disconnects — rather than shuts down — the + * terminal daemon. The daemon is detached and would survive a kill, but a stop that leaves + * the lock file behind makes the successor refuse to start with + * `orcad_data_root_shared`, so the update turns into an outage for no gain. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + assertPosixOrcadHost as assertPosixHost, + ORCAD_PID_FILENAME, + posixProcessAliveShellFunction +} from './orcad-remote-host-support' + +/** + * Signal the orcad recorded in a version dir and wait for it to go. + * + * `escalate` sends the second SIGTERM orcad reads as "exit immediately". Callers use it only + * after the first deadline elapses, so the two signals are never in the same command. + */ +export function stopOrcadCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + options: { waitSeconds: number } +): string { + assertPosixHost(host) + const pidFile = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_PID_FILENAME)) + return [ + posixProcessAliveShellFunction(), + `pid=$(cat ${pidFile} 2>/dev/null);`, + 'case "$pid" in "" | *[!0-9]* ) echo NO_PID; exit 0;; esac;', + 'orcad_alive "$pid" || { echo ALREADY_EXITED; exit 0; };', + 'kill -TERM "$pid" 2>/dev/null || { echo SIGNAL_FAILED; exit 0; };', + `i=0; while [ "$i" -lt ${options.waitSeconds} ]; do`, + 'orcad_alive "$pid" || { echo STOPPED; exit 0; };', + 'sleep 1; i=$((i + 1)); done;', + 'echo STILL_RUNNING' + ].join(' ') +} + +export type OrcadStopOutcome = + | 'stopped' + | 'already-exited' + | 'no-pid' + | 'still-running' + | 'signal-failed' + | 'unknown' + +export function parseOrcadStopOutcome(output: string): OrcadStopOutcome { + switch (output.trim().split('\n').pop()?.trim() ?? '') { + case 'STOPPED': + return 'stopped' + case 'ALREADY_EXITED': + return 'already-exited' + case 'NO_PID': + return 'no-pid' + case 'STILL_RUNNING': + return 'still-running' + case 'SIGNAL_FAILED': + return 'signal-failed' + default: + return 'unknown' + } +} + +/** True when the port is free and a successor may bind. */ +export function orcadStopFreedTheHost(outcome: OrcadStopOutcome): boolean { + return outcome === 'stopped' || outcome === 'already-exited' || outcome === 'no-pid' +} diff --git a/src/main/ssh/orcad-remote-rollback.test.ts b/src/main/ssh/orcad-remote-rollback.test.ts new file mode 100644 index 000000000000..59f3d46e330d --- /dev/null +++ b/src/main/ssh/orcad-remote-rollback.test.ts @@ -0,0 +1,206 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-transfers', () => ({ + writeRelayFile: vi.fn().mockResolvedValue(undefined), + uploadRelayDirectory: vi.fn().mockResolvedValue(undefined) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { writeRelayFile } from './ssh-relay-install-transfers' +import { rollbackOrcad, type OrcadRollbackOptions } from './orcad-remote-rollback' +import { + emptyOrcadActivationRecord, + type OrcadActivationRecord +} from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const mockExec = vi.mocked(execCommand) +const ACTIVE = '0.2.0+bb01' +const TARGET = '0.1.0+aa01' +const BUILD_HASH = 'abc123def4567890' + +function record(overrides: Partial = {}): OrcadActivationRecord { + return { + ...emptyOrcadActivationRecord(), + active: ACTIVE, + previous: TARGET, + activatedAt: '2026-01-01T00:00:00.000Z', + snapshot: { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: ACTIVE, + readableByVersion: TARGET, + takenAt: '2026-01-01T00:00:00.000Z' + }, + ...overrides + } +} + +function readyLine(version: string): string { + return JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: BUILD_HASH, + buildVersion: version, + nodeVersion: '20.11.0', + nodeAbi: '115', + platform: 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 2, + buildVersion: version, + entryPath: '/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 5 } + } + } + }) +} + +function scriptHost(log: string[], overrides: { restore?: string } = {}): void { + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { + return 'PRESENT' + } + if (text.includes('find ') && text.includes('stat')) { + return 'UNKNOWN' + } + if (text.includes('kill -TERM')) { + log.push(`stop:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-xf')) { + log.push('restore') + return overrides.restore ?? 'RESTORED' + } + if (text.includes('nohup')) { + log.push(`launch:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) + return '9999' + } + if (text.startsWith('cat ') && text.includes('.orcad-readiness')) { + return readyLine(TARGET) + } + return '' + }) +} + +function options(overrides: Partial = {}): OrcadRollbackOptions { + return { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + record: record(), + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + census: { liveSessions: 0, startedSinceActivation: 0 }, + targetBuildHash: BUILD_HASH, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z'), + ...overrides + } +} + +describe('rollbackOrcad', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('stops, restores state, then starts the target — in that order', async () => { + const log: string[] = [] + scriptHost(log) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'rolled-back', target: TARGET }) + // Restoring under a running orcad would replace the store beneath a process holding it; + // starting first would let the older build migrate the newer build's state. + expect(log).toEqual([`stop:${ACTIVE}`, 'restore', `launch:${TARGET}`]) + }) + + it('refuses before touching anything when terminals started after activation', async () => { + const log: string[] = [] + scriptHost(log) + const result = await rollbackOrcad( + options({ census: { liveSessions: 3, startedSinceActivation: 2 } }) + ) + expect(result).toMatchObject({ + outcome: 'refused', + code: 'orcad_rollback_orphans_live_terminals' + }) + expect(log).toEqual([]) + expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + }) + + it('refuses when the snapshot is gone from the host', async () => { + const log: string[] = [] + mockExec.mockImplementation(async (_conn, command: string) => + String(command).includes('state.tar') ? 'ABSENT' : '' + ) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'refused', code: 'orcad_rollback_snapshot_missing' }) + expect(log).toEqual([]) + }) + + it('does not start the old build when the restore failed', async () => { + const log: string[] = [] + scriptHost(log, { restore: 'FAILED' }) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_rollback_restore_failed' }) + expect(log).toEqual([`stop:${ACTIVE}`, 'restore']) + expect(result.outcome === 'failed' && result.reason).toContain('Do NOT start the older build') + }) + + it('leaves the record naming the newer version when the target fails to come up', async () => { + const log: string[] = [] + scriptHost(log) + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { + return 'PRESENT' + } + if (text.includes('kill -TERM')) { + return 'STOPPED' + } + if (text.includes('tar -C') && text.includes('-xf')) { + return 'RESTORED' + } + // The target never publishes readiness. + return '' + }) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' }) + // Until the target is proven serving, `active` must still name the version an operator + // would have to bring back. + expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + }) + + it('records the rollback only after the target answers healthy', async () => { + const log: string[] = [] + scriptHost(log) + await rollbackOrcad(options()) + const written = vi + .mocked(writeRelayFile) + .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[3]))).toMatchObject({ + active: TARGET, + previous: null, + snapshot: null + }) + }) +}) diff --git a/src/main/ssh/orcad-remote-rollback.ts b/src/main/ssh/orcad-remote-rollback.ts new file mode 100644 index 000000000000..165ccfc037f2 --- /dev/null +++ b/src/main/ssh/orcad-remote-rollback.ts @@ -0,0 +1,244 @@ +/** + * Going back to the previously active orcad. + * + * Rollback is a state operation, not a binary swap. The version dirs are immutable and both + * are still on disk, so pointing at the old one is trivial; what is not trivial is that both + * versions share ONE data root, outside either dir. A newer orcad migrates that root on load + * — and Orca's persisted state carries no schema version to migrate against, so the older + * build cannot be shown to read the result. Rollback therefore restores the pre-activation + * snapshot, and refuses when restoring it would orphan work (`assessOrcadRollback`). + * + * The order below is the whole safety argument: stop, then restore, then start. Restoring + * under a running orcad would replace the store beneath a process holding it open, and + * starting before restoring would let the old build migrate the new build's state — the + * failure this is meant to avoid, arrived at from the other side. + */ +import type { SshConnection } from './ssh-connection' +import { execCommand } from './ssh-relay-deploy-helpers' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { computeRemoteInstallDir } from './ssh-relay-versioned-install' +import { writeRelayFile } from './ssh-relay-install-transfers' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + ORCAD_STATE_SNAPSHOT_DIR, + serializeOrcadActivationRecord, + withRolledBackVersion, + type OrcadActivationRecord +} from './orcad-activation-record' +import { assessOrcadRollback, type OrcadTerminalCensus } from './orcad-update-plan' +import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' +import { + ORCAD_LOG_FILENAME, + orcadLaunchCommand, + parseOrcadReadinessOutput, + readOrcadReadinessCommand +} from './orcad-remote-launch' +import { + newestStateMtimeCommand, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotRestore, + probeOrcadStateSnapshotCommand, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { + orcadStopFreedTheHost, + parseOrcadStopOutcome, + stopOrcadCommand +} from './orcad-remote-process-control' +import { orcadActivationPath } from './orcad-activation-record-store' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +export type OrcadRollbackOptions = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + record: OrcadActivationRecord + nodePath: string + userDataDir: string + bindHost: string + port: number + census: OrcadTerminalCensus + /** Expected build hash of the rollback target, from the client's copy of those bytes. */ + targetBuildHash: string + readinessTimeoutMs?: number + now?: () => Date + sleep?: (ms: number) => Promise + signal?: AbortSignal +} + +export type OrcadRollbackResult = + | { outcome: 'rolled-back'; target: string; discarded: string[]; verdict: OrcadActivationVerdict } + | { outcome: 'refused'; code: string; reason: string } + | { outcome: 'failed'; code: string; reason: string } + +const DEFAULT_READINESS_TIMEOUT_MS = 90_000 +const READINESS_POLL_MS = 500 +const STOP_WAIT_SECONDS = 20 + +function exec(options: OrcadRollbackOptions, command: string): Promise { + return execCommand(options.conn, command, { + wrapCommand: options.host.commandDialect !== 'powershell', + signal: options.signal + }) +} + +function snapshotDirPath(options: OrcadRollbackOptions, dirName: string): string { + return joinRemotePath( + options.host, + options.remoteHome, + RELAY_REMOTE_DIR, + ORCAD_STATE_SNAPSHOT_DIR, + dirName + ) +} + +/** Has the store been written since activation? `null` when it cannot be established. */ +async function readStateWritesSinceActivation( + options: OrcadRollbackOptions +): Promise { + if (!options.record.activatedAt) { + return null + } + const activatedAtSeconds = Math.floor(Date.parse(options.record.activatedAt) / 1000) + if (!Number.isFinite(activatedAtSeconds)) { + return null + } + const newest = parseNewestStateMtimeSeconds( + await exec(options, newestStateMtimeCommand(options.host, options.userDataDir)).catch(() => '') + ) + return newest === null ? null : newest >= activatedAtSeconds +} + +export async function rollbackOrcad( + options: OrcadRollbackOptions +): Promise { + const now = options.now ?? ((): Date => new Date()) + const snapshotPresent = options.record.snapshot + ? ( + await exec( + options, + probeOrcadStateSnapshotCommand( + options.host, + snapshotDirPath(options, options.record.snapshot.dirName) + ) + ).catch(() => 'ABSENT') + ).trim() === 'PRESENT' + : false + + const safety = assessOrcadRollback({ + record: options.record, + snapshotPresent, + census: options.census, + stateWritesSinceActivation: await readStateWritesSinceActivation(options) + }) + if (safety.safety === 'unsafe') { + return { outcome: 'refused', code: safety.code, reason: safety.reason } + } + + if (options.record.active) { + const outgoingDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + options.record.active + ) + const stopped = parseOrcadStopOutcome( + await exec( + options, + stopOrcadCommand(options.host, outgoingDir, { waitSeconds: STOP_WAIT_SECONDS }) + ) + ) + if (!orcadStopFreedTheHost(stopped)) { + return { + outcome: 'failed', + code: 'orcad_rollback_stop_incomplete', + reason: + `orcad ${options.record.active} did not exit within ${STOP_WAIT_SECONDS}s of SIGTERM ` + + `(${stopped}). Nothing was restored — the store is untouched and the host is still ` + + 'serving the version you tried to leave.' + } + } + } + + // Why between stop and start: the store must be replaced while no orcad holds it, and + // before the older build gets a chance to migrate the newer build's state. + const restored = parseOrcadSnapshotRestore( + await exec( + options, + restoreOrcadStateSnapshotCommand( + options.host, + options.userDataDir, + // Guarded by `assessOrcadRollback`: `unsafe` covers a missing snapshot. + snapshotDirPath(options, options.record.snapshot?.dirName ?? '') + ) + ).catch(() => 'FAILED') + ) + if (restored !== 'restored') { + return { + outcome: 'failed', + code: 'orcad_rollback_restore_failed', + reason: + `The pre-activation snapshot could not be restored (${restored}). orcad is stopped and ` + + 'the data root may be partially replaced. Do NOT start the older build against it; ' + + `re-deploy ${options.record.active ?? 'the newer version'}, which can read what is there.` + } + } + + const targetDir = computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + options.remoteHome, + safety.target + ) + await exec( + options, + orcadLaunchCommand(options.host, { + remoteInstallDir: targetDir, + nodePath: options.nodePath, + fullVersion: safety.target, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port + }) + ) + const deadline = Date.now() + (options.readinessTimeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS) + const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) + let parsed = parseOrcadReadinessOutput('') + while (Date.now() < deadline && parsed.state === 'pending') { + options.signal?.throwIfAborted() + parsed = parseOrcadReadinessOutput( + await exec(options, readOrcadReadinessCommand(options.host, targetDir)) + ) + if (parsed.state === 'pending') { + await sleep(READINESS_POLL_MS) + } + } + const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { + buildHash: options.targetBuildHash, + fullVersion: safety.target + }) + if (verdict.decision === 'reject') { + return { + outcome: 'failed', + code: verdict.code, + reason: + `The rollback target ${safety.target} did not come up healthy: ${verdict.reason} The ` + + `store has been restored to its pre-activation state. Its stderr is at ` + + `${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.` + } + } + + // Why the record is written last: until the target is proven serving, `active` still names + // the version an operator would need to bring back, and `previous` still names this target. + await writeRelayFile( + options.conn, + options.host, + orcadActivationPath(options.host, options.remoteHome), + serializeOrcadActivationRecord(withRolledBackVersion(options.record, now())), + { signal: options.signal } + ) + return { + outcome: 'rolled-back', + target: safety.target, + discarded: safety.safety === 'lossy' ? safety.discards : [], + verdict + } +} diff --git a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts new file mode 100644 index 000000000000..d1f8ac0048f7 --- /dev/null +++ b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts @@ -0,0 +1,189 @@ +/** + * Runs the generated POSIX commands through a real `/bin/sh`. + * + * The unit tests assert on command *text*, which is exactly the kind of test that stays + * green while the shell it produces does not work — a quoting slip, a `case` pattern that + * never matches, a `tar` invocation that silently captures nothing. These run the strings. + */ +import { execFileSync, spawn } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + orcadLivenessProbeCommand, + ORCAD_PID_FILENAME, + parseOrcadLiveness +} from './orcad-remote-launch' +import { parseOrcadStopOutcome, stopOrcadCommand } from './orcad-remote-process-control' +import { + captureOrcadStateSnapshotCommand, + newestStateMtimeCommand, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotCapture, + parseOrcadSnapshotRestore, + probeOrcadStateSnapshotCommand, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const host = getRemoteHostPlatform('linux-x64') +let root = '' +let dataDir = '' +let snapshotDir = '' +let versionDir = '' + +function sh(command: string): string { + return execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' }) +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orcad-shell-')) + dataDir = join(root, '.orca') + snapshotDir = join(root, 'snapshots', 'pre-0.2.0+bb01-1000') + versionDir = join(root, '.orca-remote', 'orcad-0.2.0+bb01') + mkdirSync(join(dataDir, 'profiles', 'p1'), { recursive: true }) + mkdirSync(join(dataDir, 'daemon'), { recursive: true }) + mkdirSync(versionDir, { recursive: true }) + writeFileSync(join(dataDir, 'orca-profile-index.json'), '{"v":"before"}') + writeFileSync(join(dataDir, 'profiles', 'p1', 'orca-data.json'), '{"repos":"before"}') + writeFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'live-daemon-token') +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) +}) + +describe('state snapshot commands, run for real', () => { + it('captures, then restores state the newer build overwrote', () => { + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + ).toBe('captured') + expect(sh(probeOrcadStateSnapshotCommand(host, snapshotDir)).trim()).toBe('PRESENT') + + // The new version migrates the store and adds a file of its own. + writeFileSync(join(dataDir, 'orca-profile-index.json'), '{"v":"migrated"}') + writeFileSync(join(dataDir, 'profiles', 'p1', 'new-build-only.json'), '{}') + + expect( + parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + ).toBe('restored') + expect(readFileSync(join(dataDir, 'orca-profile-index.json'), 'utf8')).toBe('{"v":"before"}') + // Removed before extraction, so the older build never sees a file it cannot interpret. + expect(() => readFileSync(join(dataDir, 'profiles', 'p1', 'new-build-only.json'))).toThrow() + }) + + it('leaves the live daemon runtime dir untouched through capture and restore', () => { + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + // The daemon is running across the rollback and rewrites its token; a restore that + // reached /daemon would break the fence that keeps its terminals adoptable. + writeFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'token-after-restart') + sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + expect(readFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'utf8')).toBe( + 'token-after-restart' + ) + }) + + it('reports EMPTY on a data root with nothing to lose, instead of an archive of nothing', () => { + const emptyRoot = join(root, 'fresh') + mkdirSync(emptyRoot) + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, emptyRoot, snapshotDir))) + ).toBe('empty') + expect(sh(probeOrcadStateSnapshotCommand(host, snapshotDir)).trim()).toBe('ABSENT') + }) + + it('reports MISSING rather than claiming a restore it did not perform', () => { + expect( + parseOrcadSnapshotRestore( + sh(restoreOrcadStateSnapshotCommand(host, dataDir, join(root, 'nope'))) + ) + ).toBe('missing') + }) + + it('reads a real mtime for the store', () => { + const seconds = parseNewestStateMtimeSeconds(sh(newestStateMtimeCommand(host, dataDir))) + expect(seconds).toBeGreaterThan(1_600_000_000) + expect(seconds).toBeLessThanOrEqual(Math.floor(Date.now() / 1000) + 5) + }) + + it('survives a data root whose path contains a quote and a space', () => { + const nasty = join(root, `it's a dir`) + mkdirSync(join(nasty, 'profiles'), { recursive: true }) + writeFileSync(join(nasty, 'orca-profile-index.json'), '{"v":"quoted"}') + expect( + parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, nasty, snapshotDir))) + ).toBe('captured') + writeFileSync(join(nasty, 'orca-profile-index.json'), '{"v":"changed"}') + expect( + parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, nasty, snapshotDir))) + ).toBe('restored') + expect(readFileSync(join(nasty, 'orca-profile-index.json'), 'utf8')).toBe('{"v":"quoted"}') + }) +}) + +describe('liveness and stop commands, run for real', () => { + it('reports UNKNOWN with no pid file, and DEAD for a pid that has exited', () => { + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('UNKNOWN') + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), 'not-a-pid') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('UNKNOWN') + // A pid that has certainly exited: our own `sh` child from the line above. + const exited = Number(sh('sh -c "echo $$"').trim()) + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(exited)) + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + }) + + it('reports LIVE for a running process and stops it with SIGTERM', async () => { + const child = spawn('/bin/sh', ['-c', 'sleep 30'], { stdio: 'ignore' }) + try { + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(child.pid)) + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('LIVE') + + const exited = new Promise((resolve) => + child.once('exit', (_code, signal) => resolve(signal)) + ) + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 10 })))).toBe( + 'stopped' + ) + expect(await exited).toBe('SIGTERM') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + } finally { + child.kill('SIGKILL') + } + }) + + // `kill -0` succeeds on a zombie, so a probe built on it alone calls an exited process + // live: the stop loop would time out on a process that is already gone, and GC would keep + // a dead version dir forever. Verified as a real macOS behaviour, not a hypothetical. + it('reports a zombie as DEAD, not as a running process', () => { + const child = spawn('/bin/sh', ['-c', 'exit 0'], { stdio: 'ignore' }) + try { + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(child.pid)) + // Block the event loop so Node never reaps it; the process is now a zombie. + sh('sleep 1') + expect(sh(`ps -o stat= -p ${child.pid} || echo GONE`).trim()).toMatch(/^Z/) + expect(sh(`kill -0 ${child.pid} 2>/dev/null && echo LIVE || echo DEAD`).trim()).toBe('LIVE') + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + expect( + parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 }))) + ).toBe('already-exited') + } finally { + child.unref() + } + }) + + it('reports ALREADY_EXITED for a stale pid file rather than signalling a stranger', () => { + const exited = Number(sh('sh -c "echo $$"').trim()) + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(exited)) + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 })))).toBe( + 'already-exited' + ) + }) + + it('reports NO_PID when the version dir was never launched', () => { + expect(parseOrcadStopOutcome(sh(stopOrcadCommand(host, versionDir, { waitSeconds: 1 })))).toBe( + 'no-pid' + ) + }) +}) diff --git a/src/main/ssh/orcad-state-snapshot.test.ts b/src/main/ssh/orcad-state-snapshot.test.ts new file mode 100644 index 000000000000..0c50dc55fdbe --- /dev/null +++ b/src/main/ssh/orcad-state-snapshot.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest' + +import { + ORCAD_SNAPSHOT_EXCLUDED, + ORCAD_SNAPSHOT_MEMBERS, + captureOrcadStateSnapshotCommand, + newestStateMtimeCommand, + orcadSnapshotDirName, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotCapture, + parseOrcadSnapshotRestore, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') +const ROOT = '/home/u/.orca' +const SNAP = '/home/u/.orca-remote/orcad-state-snapshots/pre-0.2.0+bb01-1000' + +describe('capturing the pre-activation snapshot', () => { + it('captures the profile state a rollback needs', () => { + const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP) + for (const member of ORCAD_SNAPSHOT_MEMBERS) { + expect(command).toContain(`'${member}'`) + } + }) + + // The live daemon owns /daemon and outlives every restart. Restoring a stale copy of + // its socket, PID record and token would break the fence that keeps its terminals adoptable. + it.each(ORCAD_SNAPSHOT_EXCLUDED)('never captures %s', (excluded) => { + expect(captureOrcadStateSnapshotCommand(posix, ROOT, SNAP)).not.toContain(`'${excluded}'`) + }) + + it.each(ORCAD_SNAPSHOT_EXCLUDED)('never removes or restores over %s', (excluded) => { + expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP)).not.toContain(`'${excluded}'`) + }) + + it('writes the archive under a temp name and renames, so a killed deploy leaves no torn tar', () => { + const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP) + expect(command).toContain('.partial') + expect(command.indexOf('tar -C')).toBeLessThan(command.indexOf('mv ')) + }) + + it.each([ + ['CAPTURED', 'captured'], + ['EMPTY', 'empty'], + ['tar: broken', 'failed'], + ['', 'failed'] + ])('parses %s as %s', (output, expected) => { + expect(parseOrcadSnapshotCapture(output)).toBe(expected) + }) + + it('keys the snapshot dir on both version and time, so a retry cannot overwrite one', () => { + expect(orcadSnapshotDirName('0.2.0+bb01', 1000)).not.toBe( + orcadSnapshotDirName('0.2.0+bb01', 2000) + ) + }) +}) + +describe('restoring the snapshot', () => { + it('clears the members before extracting, so files the new build added do not survive', () => { + const command = restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP) + expect(command.indexOf('rm -rf')).toBeLessThan(command.indexOf('tar -C')) + }) + + it('reports a missing archive instead of extracting nothing and claiming success', () => { + expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP)).toContain('echo MISSING') + expect(parseOrcadSnapshotRestore('MISSING')).toBe('missing') + expect(parseOrcadSnapshotRestore('RESTORED')).toBe('restored') + expect(parseOrcadSnapshotRestore('FAILED')).toBe('failed') + }) +}) + +describe('detecting writes since activation', () => { + it.each([ + ['1700000000', 1_700_000_000], + ['UNKNOWN', null], + ['', null] + ])('parses %s', (output, expected) => { + expect(parseNewestStateMtimeSeconds(output)).toBe(expected) + }) + + it('looks at the same members the snapshot covers', () => { + const command = newestStateMtimeCommand(posix, ROOT) + for (const member of ORCAD_SNAPSHOT_MEMBERS) { + expect(command).toContain(`'${member}'`) + } + }) +}) + +describe('Windows hosts', () => { + it.each([ + ['capture', () => captureOrcadStateSnapshotCommand(windows, ROOT, SNAP)], + ['restore', () => restoreOrcadStateSnapshotCommand(windows, ROOT, SNAP)], + ['mtime', () => newestStateMtimeCommand(windows, ROOT)] + ])('refuses %s rather than emitting a POSIX command', (_label, build) => { + expect(build).toThrow('orcad to a Windows host is not implemented') + }) +}) diff --git a/src/main/ssh/orcad-state-snapshot.ts b/src/main/ssh/orcad-state-snapshot.ts new file mode 100644 index 000000000000..78ad8b47d2ac --- /dev/null +++ b/src/main/ssh/orcad-state-snapshot.ts @@ -0,0 +1,173 @@ +/** + * The pre-activation copy of shared profile state that makes rollback sound. + * + * `docs/design/shipping-orcad.html` §04's state-schema row asks for "backward-readable + * migrations or a pre-activation snapshot". Only the second is available here, and not as a + * preference: Orca's persisted state carries **no schema version**. Migrations are cohort + * and shape heuristics that run on load and rewrite in place, and the load path rebuilds + * `settings` and `ui` from known fields — so a newer build's nested additions are silently + * dropped by an older one rather than rejected. There is nothing to compare and nothing that + * fails loudly, which rules out proving backward-readability and leaves the snapshot. + * + * What is snapshotted is deliberately narrow. `/daemon` is EXCLUDED: it holds the live + * daemon's socket, PID record and auth token, and that daemon outlives every orcad restart + * by design. Restoring a stale copy of it over a running daemon would break the endpoint + * fence that keeps its terminals adoptable — turning a rollback into the exact terminal + * massacre the daemon exists to prevent. + */ +import { shellEscape } from './ssh-connection-utils' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { assertPosixOrcadHost as assertPosixHost } from './orcad-remote-host-support' + +/** + * Root-relative paths a rollback needs restored. Everything else under the data root is + * either regenerable, or owned by a process that survives the rollback. + */ +export const ORCAD_SNAPSHOT_MEMBERS = [ + 'orca-profile-index.json', + // Pre-profiles layout; still read as a migration source. + 'orca-data.json', + 'profiles' +] as const + +/** Never captured and never restored — see the module comment. */ +export const ORCAD_SNAPSHOT_EXCLUDED = ['daemon', 'logs'] as const + +/** + * The member names go into the command unquoted (see `captureOrcadStateSnapshotCommand`), so + * they must be inert. They are compile-time constants; this catches the edit that adds one + * with a space or a metacharacter in it. + */ +function assertPlainMemberName(member: string): string { + if (!/^[A-Za-z0-9._-]+$/.test(member)) { + throw new Error(`Unsafe orcad snapshot member name: ${JSON.stringify(member)}`) + } + return member +} + +export function orcadSnapshotDirName(fullVersion: string, takenAtMs: number): string { + // Why the version and the timestamp: two activations of one version (a re-deploy after a + // rejected activation) must not overwrite each other's snapshot. + return `pre-${fullVersion}-${takenAtMs}` +} + +/** + * Capture the snapshot, or report why there is nothing to capture. + * + * Prints `CAPTURED`, or `EMPTY` when the data root holds none of the members — a first-ever + * deployment, where there is no state to lose and therefore no snapshot to take. `EMPTY` is + * reported rather than fabricating an empty archive, because a rollback that "restored" an + * empty archive would wipe a root that had filled up in between. + */ +export function captureOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + userDataDir: string, + snapshotDir: string +): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const dir = shellEscape(snapshotDir) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + const memberTests = ORCAD_SNAPSHOT_MEMBERS.map( + // Why the accumulated name is NOT quoted: `$members` is re-split by the shell before it + // reaches tar, so a quoted name arrives as a literal `'profiles'` that tar cannot stat. + // `assertPlainMemberName` is what makes leaving them bare safe. + (member) => + `[ -e ${root}/${shellEscape(member)} ] && members="$members ${assertPlainMemberName(member)}";` + ).join(' ') + return [ + `members=;`, + memberTests, + 'if [ -z "$members" ]; then echo EMPTY; else', + `mkdir -p ${dir} && umask 077 &&`, + // Why a temp name then mv: a deploy killed mid-tar must not leave a truncated archive + // that a later rollback would happily restore. + `tar -C ${root} -cf ${archive}.partial $members && mv ${archive}.partial ${archive} &&`, + 'echo CAPTURED; fi' + ].join(' ') +} + +export type OrcadSnapshotCapture = 'captured' | 'empty' | 'failed' + +export function parseOrcadSnapshotCapture(output: string): OrcadSnapshotCapture { + const value = output.trim().split('\n').pop()?.trim() + if (value === 'CAPTURED') { + return 'captured' + } + return value === 'EMPTY' ? 'empty' : 'failed' +} + +export function probeOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + snapshotDir: string +): string { + assertPosixHost(host) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + return `test -f ${archive} && echo PRESENT || echo ABSENT` +} + +/** + * Restore the snapshot over the data root. + * + * Two things make this safe to run: the members are removed before extraction (so a file the + * new version added is gone rather than half-shadowed), and neither the removal nor the + * extraction can reach `/daemon`, because the member list never names it. + * + * The caller must have stopped orcad first. This does not check — it cannot, from a shell — + * so `orcad-remote-deploy.ts` owns that ordering. + */ +export function restoreOrcadStateSnapshotCommand( + host: RemoteHostPlatform, + userDataDir: string, + snapshotDir: string +): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) + const removals = ORCAD_SNAPSHOT_MEMBERS.map( + (member) => `rm -rf ${root}/${shellEscape(member)};` + ).join(' ') + return [ + `test -f ${archive} || { echo MISSING; exit 0; };`, + `test -d ${root} || mkdir -p ${root};`, + removals, + `tar -C ${root} -xf ${archive} && echo RESTORED || echo FAILED` + ].join(' ') +} + +export type OrcadSnapshotRestore = 'restored' | 'missing' | 'failed' + +export function parseOrcadSnapshotRestore(output: string): OrcadSnapshotRestore { + const value = output.trim().split('\n').pop()?.trim() + if (value === 'RESTORED') { + return 'restored' + } + return value === 'MISSING' ? 'missing' : 'failed' +} + +/** + * Has the shared store been written since `activatedAt`? + * + * Prints the newest mtime (epoch seconds) across the snapshot members, or `UNKNOWN`. The + * caller compares; an `UNKNOWN` becomes `null`, which `assessOrcadRollback` treats as "yes, + * assume writes". + */ +export function newestStateMtimeCommand(host: RemoteHostPlatform, userDataDir: string): string { + assertPosixHost(host) + const root = shellEscape(userDataDir) + const paths = ORCAD_SNAPSHOT_MEMBERS.map((member) => `${root}/${shellEscape(member)}`).join(' ') + return [ + `newest=$(find ${paths} -type f -exec stat -c %Y {} + 2>/dev/null ||`, + `find ${paths} -type f -exec stat -f %m {} + 2>/dev/null);`, + 'if [ -z "$newest" ]; then echo UNKNOWN; else', + `echo "$newest" | sort -n | tail -1; fi` + ].join(' ') +} + +export function parseNewestStateMtimeSeconds(output: string): number | null { + const value = output.trim().split('\n').pop()?.trim() + if (!value || !/^\d+$/.test(value)) { + return null + } + return Number.parseInt(value, 10) +} diff --git a/src/main/ssh/orcad-update-plan.test.ts b/src/main/ssh/orcad-update-plan.test.ts new file mode 100644 index 000000000000..5d90c597e3c2 --- /dev/null +++ b/src/main/ssh/orcad-update-plan.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from 'vitest' + +import { assessOrcadRollback, planOrcadUpdate } from './orcad-update-plan' +import { + emptyOrcadActivationRecord, + type OrcadActivationRecord, + type OrcadStateSnapshot +} from './orcad-activation-record' + +const SNAPSHOT: OrcadStateSnapshot = { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: '0.2.0+bb01', + readableByVersion: '0.1.0+aa01', + takenAt: '2026-01-01T00:00:00.000Z' +} + +function record(overrides: Partial = {}): OrcadActivationRecord { + return { + ...emptyOrcadActivationRecord(), + active: '0.2.0+bb01', + previous: '0.1.0+aa01', + activatedAt: '2026-01-01T00:00:01.000Z', + snapshot: SNAPSHOT, + ...overrides + } +} + +describe('planOrcadUpdate', () => { + it('does nothing when the candidate is already active', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.2.0+bb01', + census: { liveSessions: 0, startedSinceActivation: 0 } + }) + expect(plan).toMatchObject({ action: 'noop' }) + }) + + it('defers rather than restarting a host with live terminals', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 3, startedSinceActivation: 1 } + }) + expect(plan).toMatchObject({ action: 'defer', code: 'orcad_update_terminals_running' }) + expect(plan.action === 'defer' && plan.reason).toContain('would not kill them') + }) + + it('defers when the session count cannot be established', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: null, startedSinceActivation: null } + }) + expect(plan).toMatchObject({ + action: 'defer', + code: 'orcad_update_terminal_census_unavailable' + }) + }) + + it('plans a forced update with an unknown census as if terminals were live', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: null, startedSinceActivation: null }, + force: true + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: true }) + }) + + it('carries the daemon across a forced update with live terminals', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 2, startedSinceActivation: 0 }, + force: true + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: true }) + }) + + it('replaces the daemon only when nothing is running under it', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 0, startedSinceActivation: 0 } + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: false }) + }) +}) + +describe('assessOrcadRollback', () => { + it('is clean when the snapshot is intact and nothing happened since activation', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'clean', target: '0.1.0+aa01' }) + }) + + it('is lossy, and names what goes, once the store has been written since activation', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 1, startedSinceActivation: 0 }, + stateWritesSinceActivation: true + }) + expect(safety).toMatchObject({ safety: 'lossy', target: '0.1.0+aa01' }) + expect(safety.safety === 'lossy' && safety.discards[0]).toContain('2026-01-01T00:00:01.000Z') + }) + + it('treats an unreadable store mtime as writes, not as a clean rollback', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: null + }) + expect(safety).toMatchObject({ safety: 'lossy' }) + }) + + // The point past which rollback is unsafe: the first terminal created after activation. + it('refuses once a terminal started after activation, because restoring would orphan it', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 4, startedSinceActivation: 1 }, + stateWritesSinceActivation: true + }) + expect(safety).toMatchObject({ + safety: 'unsafe', + code: 'orcad_rollback_orphans_live_terminals' + }) + expect(safety.safety === 'unsafe' && safety.reason).toContain('nothing would be able to') + }) + + it('refuses when the snapshot the record names is gone from the host', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: false, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_missing' }) + expect(safety.safety === 'unsafe' && safety.reason).toContain('no schema version') + }) + + it('refuses when no snapshot was ever recorded', () => { + const safety = assessOrcadRollback({ + record: record({ snapshot: null }), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_missing' }) + }) + + it('refuses when the post-activation session count is unverifiable', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: { liveSessions: 2, startedSinceActivation: null }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_census_unavailable' }) + }) + + it('refuses when there is no previous version to go back to', () => { + const safety = assessOrcadRollback({ + record: record({ previous: null }), + snapshotPresent: true, + census: { liveSessions: 0, startedSinceActivation: 0 }, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_no_target' }) + }) +}) diff --git a/src/main/ssh/orcad-update-plan.ts b/src/main/ssh/orcad-update-plan.ts new file mode 100644 index 000000000000..2653956ebe51 --- /dev/null +++ b/src/main/ssh/orcad-update-plan.ts @@ -0,0 +1,234 @@ +/** + * When an update may restart orcad, and when going back is still sound. + * + * Two constraints shape everything here. + * + * **The daemon must outlive the restart.** orcad forks the terminal daemon and deliberately + * does not kill it on stop (`orcad-daemon-supervision.ts` uses `disconnectDaemon`, never + * `shutdownDaemon`). An update that killed it would destroy every terminal on the host — + * the thing the daemon exists to prevent. After an update the surviving daemon was forked + * from the OUTGOING bundle, so `daemon-init` sees an entry-path/version mismatch and takes + * its `shouldPreserveDaemonWithLiveSessions` branch: with live sessions it preserves the old + * daemon; at exactly zero it replaces it. Both are correct, and both mean the outgoing + * version's directory is still load-bearing. + * + * **The state root is shared across versions.** `~/.orca/` (or `$ORCA_USER_DATA`) is outside + * every version dir, and Orca's persisted state carries no schema version — migrations run + * on load and rewrite in place. So "is the old version able to read what the new one wrote" + * has no answer that can be computed. That is why rollback is defined against a + * pre-activation snapshot rather than against a version comparison. + */ +import type { OrcadActivationRecord } from './orcad-activation-record' + +export type OrcadTerminalCensus = { + /** + * Sessions the live daemon owns right now. `null` means the probe could not answer — + * never treated as zero, because loss of contact is not evidence of process death + * (docs/reference/ssh-execution-boundary.md). + */ + liveSessions: number | null + /** + * Of those, how many started at or after `record.activatedAt`. These are the sessions the + * pre-activation snapshot does not describe. + */ + startedSinceActivation: number | null +} + +export type OrcadUpdateDecision = + | { action: 'noop'; reason: string } + | { + action: 'proceed' + /** True when a live daemon will be carried across the restart rather than replaced. */ + preservesLiveDaemon: boolean + notes: string[] + } + | { action: 'defer'; code: OrcadUpdateDeferCode; reason: string } + +export type OrcadUpdateDeferCode = + | 'orcad_update_terminals_running' + | 'orcad_update_terminal_census_unavailable' + +/** + * Decide whether to restart orcad onto `candidateVersion`. + * + * Deferring on live terminals is a deliberate choice, not caution. The restart itself is + * non-destructive, but it leaves the host running a NEW orcad against an OLD daemon until + * every one of those terminals exits — a mixed pair whose duration the operator, not the + * deploy, should decide. `force` is how they decide it. + */ +export function planOrcadUpdate(input: { + record: OrcadActivationRecord + candidateVersion: string + census: OrcadTerminalCensus + force?: boolean +}): OrcadUpdateDecision { + if (input.record.active === input.candidateVersion) { + return { + action: 'noop', + reason: `${input.candidateVersion} is already the active version; nothing to restart.` + } + } + const { liveSessions } = input.census + if (liveSessions === null) { + if (!input.force) { + return { + action: 'defer', + code: 'orcad_update_terminal_census_unavailable', + reason: + 'The terminal daemon did not answer a session count, so this update cannot tell ' + + 'whether work is running on the host. Retry, or force the update knowing terminals ' + + 'may be mid-flight.' + } + } + return { + action: 'proceed', + // Why true: an unverifiable census must be planned for as if sessions exist. Assuming + // the daemon is replaceable is the assumption that destroys terminals. + preservesLiveDaemon: true, + notes: [ + 'Forced with an unverifiable session count. Planning as if terminals are live: the ' + + 'daemon will be preserved across the restart, and the outgoing version directory ' + + 'stays pinned against GC.' + ] + } + } + if (liveSessions > 0 && !input.force) { + return { + action: 'defer', + code: 'orcad_update_terminals_running', + reason: + `${liveSessions} terminal${liveSessions === 1 ? ' is' : 's are'} running on this host. ` + + 'The restart would not kill them — the daemon is preserved — but the host would run ' + + `orcad ${input.candidateVersion} against a daemon forked from ` + + `${input.record.active ?? 'the previous build'} until they all exit. Update when the ` + + 'host is idle, or force it.' + } + } + if (liveSessions > 0) { + return { + action: 'proceed', + preservesLiveDaemon: true, + notes: [ + `Forced with ${liveSessions} live terminal${liveSessions === 1 ? '' : 's'}. They survive ` + + 'the restart on the existing daemon; the outgoing version directory stays pinned ' + + 'against GC because that daemon was forked from it.' + ] + } + } + return { + action: 'proceed', + // Zero live sessions is the one case where daemon-init's freshness branch replaces the + // daemon, so nothing is carried across and nothing is lost. + preservesLiveDaemon: false, + notes: [ + 'No terminals are running, so the daemon is replaced by one forked from the new bundle.' + ] + } +} + +export type OrcadRollbackSafety = + | { safety: 'clean'; target: string; notes: string[] } + | { safety: 'lossy'; target: string; discards: string[] } + | { safety: 'unsafe'; code: OrcadRollbackUnsafeCode; reason: string } + +export type OrcadRollbackUnsafeCode = + | 'orcad_rollback_no_target' + | 'orcad_rollback_snapshot_missing' + | 'orcad_rollback_orphans_live_terminals' + | 'orcad_rollback_census_unavailable' + +/** + * How safe it is to switch back to `record.previous`. + * + * **The point past which rollback is unsafe is the first terminal created after + * activation.** Not the first state write, and not any schema comparison: + * + * - Rolling back means restoring the pre-activation snapshot, because there is no schema + * version to prove the old build can read what the new one wrote. + * - The snapshot predates activation, so it does not describe sessions created since. + * - The daemon survives the binary swap and still owns those sessions. After the restore, + * a live daemon holds PTYs that the restored store has no rows for: work that is running, + * that no client can reattach to, and that the host will report as neither `live` nor + * `exited` for any session anyone can name. + * + * Settings and UI churn written after activation are merely discarded, which is `lossy`. + * Orphaning running work is not something a deploy gets to do quietly, so it is `unsafe`. + */ +export function assessOrcadRollback(input: { + record: OrcadActivationRecord + /** Whether the snapshot named by the record is actually still on the host. */ + snapshotPresent: boolean + census: OrcadTerminalCensus + /** + * Whether the shared store has been written since activation, from its mtime against + * `record.activatedAt`. `null` means unknown, which is treated as "yes" — claiming a + * lossless rollback we cannot demonstrate is the failure mode, not the caution. + */ + stateWritesSinceActivation: boolean | null +}): OrcadRollbackSafety { + const target = input.record.previous + if (!target) { + return { + safety: 'unsafe', + code: 'orcad_rollback_no_target', + reason: + 'This host has no previous orcad version recorded, so there is nothing to roll back ' + + 'to. Deploy a known-good build instead.' + } + } + if (!input.record.snapshot || !input.snapshotPresent) { + return { + safety: 'unsafe', + code: 'orcad_rollback_snapshot_missing', + reason: + `The pre-activation state snapshot for ${input.record.active ?? 'the active version'} ` + + 'is gone, and Orca state carries no schema version that could prove the older build ' + + 'can read what the newer one migrated. Switching the binary back would hand ' + + `${target} a store it may not understand. Deploy forward instead.` + } + } + const { startedSinceActivation } = input.census + if (startedSinceActivation === null) { + return { + safety: 'unsafe', + code: 'orcad_rollback_census_unavailable', + reason: + 'The daemon did not answer how many of its terminals started after this version was ' + + 'activated, so a snapshot restore might orphan running work. Retry when the host is ' + + 'reachable.' + } + } + if (startedSinceActivation > 0) { + return { + safety: 'unsafe', + code: 'orcad_rollback_orphans_live_terminals', + reason: + `${startedSinceActivation} terminal${startedSinceActivation === 1 ? '' : 's'} started ` + + 'after this version was activated. The daemon survives the rollback and would keep ' + + 'owning them, but the restored snapshot predates them, so nothing would be able to ' + + 'reattach. Close them (or let them exit) and roll back then.' + } + } + if (input.stateWritesSinceActivation === false) { + return { + safety: 'clean', + target, + notes: [ + 'The pre-activation snapshot is intact, no terminals started since activation, and ' + + 'the store has not been written since. Restoring it changes nothing.' + ] + } + } + return { + safety: 'lossy', + target, + discards: [ + input.stateWritesSinceActivation === null + ? 'Any profile, settings and UI change written since activation — the store mtime ' + + 'could not be read, so assume there are some.' + : `Every profile, settings and UI change written since ${ + input.record.activatedAt ?? 'activation' + }, when ${input.record.active ?? 'the active version'} was activated.` + ] + } +} diff --git a/src/main/ssh/remote-install-coexistence.test.ts b/src/main/ssh/remote-install-coexistence.test.ts new file mode 100644 index 000000000000..ce844587d62b --- /dev/null +++ b/src/main/ssh/remote-install-coexistence.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest' + +import { selectRemoteInstallModel } from './remote-install-coexistence' + +const BOTH_INSTALLED = ['relay-0.1.0+aa01', 'orcad-0.2.0+bb01', 'orcad-0.1.0+aa01'] + +describe('what a client does when it finds both models installed', () => { + it('uses the registered model and leaves the other install alone', () => { + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad' }) + expect(selection.outcome === 'use' && selection.coexisting).toEqual(['relay-0.1.0+aa01']) + expect(selection.outcome === 'use' && selection.note).toContain( + 'garbage-collects only its own namespace' + ) + }) + + it('does not switch model just because the other one is also on disk', () => { + const selection = selectRemoteInstallModel({ + registration: 'ssh-target', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'relay' }) + expect(selection.outcome === 'use' && selection.coexisting).toEqual([ + 'orcad-0.2.0+bb01', + 'orcad-0.1.0+aa01' + ]) + }) + + it('picks the registered model even when only the other one is installed', () => { + // On-disk presence is diagnostic, never a vote: an orcad-registered host with only relay + // dirs is a first orcad deploy, not a reason to fall back to the relay. + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: ['relay-0.1.0+aa01'] + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad' }) + }) + + it('refuses a machine registered under both models', () => { + const selection = selectRemoteInstallModel({ + registration: 'both', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ + outcome: 'refuse', + code: 'remote_host_registered_under_both_models' + }) + // The forbidden thing is two registrations, not two directories — say so, or a user will + // "fix" it by deleting an install that is serving someone. + expect(selection.outcome === 'refuse' && selection.reason).toContain( + 'Leaving both install directories on disk is fine' + ) + }) + + it('refuses to infer a model for an unregistered machine', () => { + const selection = selectRemoteInstallModel({ + registration: 'none', + installedDirNames: BOTH_INSTALLED + }) + expect(selection).toMatchObject({ + outcome: 'refuse', + code: 'remote_host_not_registered' + }) + }) + + it('says nothing when there is nothing coexisting', () => { + const selection = selectRemoteInstallModel({ + registration: 'orcad-peer', + installedDirNames: ['orcad-0.2.0+bb01'] + }) + expect(selection).toMatchObject({ outcome: 'use', model: 'orcad', note: null }) + }) +}) diff --git a/src/main/ssh/remote-install-coexistence.ts b/src/main/ssh/remote-install-coexistence.ts new file mode 100644 index 000000000000..a24ac36cfb1f --- /dev/null +++ b/src/main/ssh/remote-install-coexistence.ts @@ -0,0 +1,83 @@ +/** + * What a client does when it finds both a relay and an orcad installed on one host. + * + * `docs/design/shipping-orcad.html` §06 draws the line the boundary doc actually draws: + * two *directories* on disk are fine and permanent; two *registered targets* for one + * machine are forbidden, because that is what splits a machine's worktrees across two + * identities (`docs/reference/ssh-execution-boundary.md`). + * + * So the model is never inferred from the filesystem. It is decided by how the user + * registered the host, and the on-disk inventory is only ever diagnostic. Inferring it — + * "an orcad dir exists, so prefer orcad" — would let a GC pass, a half-finished install or + * a stale tree silently re-point a live connection at a different execution identity. + */ +import { inventoryRemoteInstallDirs, type RemoteInstallModelId } from './remote-install-model' + +/** + * How this host is registered in the client's own records. + * + * `both` is representable on purpose: it is a state a user can reach by adding an SSH + * target for a machine they have already paired, and the point of this module is to refuse + * it loudly instead of picking one. + */ +export type RemoteHostRegistration = 'ssh-target' | 'orcad-peer' | 'both' | 'none' + +export type RemoteInstallSelection = + | { + outcome: 'use' + model: RemoteInstallModelId + /** Other-model dirs present on this host. Left alone; never GC'd by the chosen model. */ + coexisting: string[] + /** Non-null when there is something an operator should know but nothing to refuse over. */ + note: string | null + } + | { + outcome: 'refuse' + code: 'remote_host_registered_under_both_models' | 'remote_host_not_registered' + reason: string + } + +/** + * Choose the execution model for a connection to a host, given its registration and + * whatever happens to be installed there. + */ +export function selectRemoteInstallModel(input: { + registration: RemoteHostRegistration + /** Raw directory names under `~/.orca-remote/`, as listed on the host. */ + installedDirNames: readonly string[] +}): RemoteInstallSelection { + if (input.registration === 'both') { + return { + outcome: 'refuse', + code: 'remote_host_registered_under_both_models', + reason: + 'This machine is registered both as an SSH target and as a paired orcad peer. One ' + + 'machine must have one execution identity, or its worktrees and terminals split ' + + 'across two owners that cannot see each other. Remove one registration. Leaving ' + + 'both install directories on disk is fine and expected.' + } + } + if (input.registration === 'none') { + return { + outcome: 'refuse', + code: 'remote_host_not_registered', + reason: + 'This machine has no execution model registered. Add it as an SSH target or pair it ' + + 'as an orcad peer; what is already installed on it does not decide which it is.' + } + } + const model: RemoteInstallModelId = input.registration === 'ssh-target' ? 'relay' : 'orcad' + const inventory = inventoryRemoteInstallDirs(input.installedDirNames) + const coexisting = model === 'relay' ? inventory.orcad : inventory.relay + return { + outcome: 'use', + model, + coexisting, + note: + coexisting.length > 0 + ? `This host also has ${coexisting.length} ${model === 'relay' ? 'orcad' : 'relay'} ` + + `install directory(ies) (${coexisting.join(', ')}). They are left untouched: each ` + + 'model garbage-collects only its own namespace.' + : null + } +} diff --git a/src/main/ssh/remote-install-gc.ts b/src/main/ssh/remote-install-gc.ts new file mode 100644 index 000000000000..0d649cd0b039 --- /dev/null +++ b/src/main/ssh/remote-install-gc.ts @@ -0,0 +1,286 @@ +/** + * The version-directory garbage collector, shared by the relay and orcad. + * + * It lives apart from `ssh-relay-versioned-install.ts` because it is the one piece both + * models run, and because the ownership rule below is the whole point of separating them: + * a pass only ever sees, and only ever deletes, directories belonging to `model`. + */ +import type { SshConnection } from './ssh-connection' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { execCommand } from './ssh-relay-deploy-helpers' +import { probeInstallLockExistsCommand } from './ssh-relay-install-lock-commands' +import { isRelayInstallLockStale, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' +import { + RELAY_INSTALL_MODEL, + remoteInstallGcPermits, + remoteInstallVersionDirRegex, + type RemoteInstallModel +} from './remote-install-model' +import { + isRelayGcClaimOwned, + releaseRelayGcClaimWithRetry, + tryAcquireRelayGcClaim +} from './ssh-relay-gc-claim' +import { cleanupRelayGcTombstones } from './ssh-relay-gc-tombstone' +import { + listRemoteInstallBaseDirsCommand, + MAX_RELAY_GC_LISTING_ENTRIES, + moveRemoteTreeCommand, + probeFileExistsCommand, + relayLivenessProbeCommand, + removeRemoteTreeCommand +} from './ssh-remote-commands' +import { + getRemoteHostPlatform, + isWindowsRemoteHost, + joinRemotePath, + remoteBasename, + type RemoteHostPlatform +} from './ssh-remote-platform' +import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' + +// Legacy relay dirs predate `.install-complete`; they need a liveness-only GC check so they +// eventually drain. There is no orcad equivalent — orcad has never shipped without one. +const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/ +const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') + +function execHostCommand( + conn: SshConnection, + host: RemoteHostPlatform, + command: string +): Promise { + return execCommand(conn, command, { wrapCommand: host.commandDialect !== 'powershell' }) +} + +export type RemoteInstallGcOptions = { + windowsNodePath?: string + windowsSockNames?: string[] + /** + * Model-specific "someone is using this directory" probe. It must answer TRUE when + * inconclusive — an unanswered probe is never evidence a tree is idle. + */ + isDirLive: (dir: string) => Promise + /** + * Directories this pass must never remove even when idle and complete, named by directory + * (not absolute path). orcad passes its active and previous versions: the previous one is + * the rollback target, and GC'ing it turns a recoverable bad update into a re-deploy. + */ + pinnedDirNames?: readonly string[] +} + +/** + * Garbage-collect one model's old version directories. + * + * **GC ownership (design §06 falsifier 1):** a pass only ever sees, and only ever deletes, + * directories belonging to `model`. The remote listing is scoped by prefix, and + * `remoteInstallGcPermits` re-checks every candidate locally, so neither a widened glob nor + * a hand-rolled listing can make one model delete the other's live install. + */ +export async function gcOldRemoteInstallVersions( + conn: SshConnection, + model: RemoteInstallModel, + remoteHome: string, + currentDirAbsPath: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options: RemoteInstallGcOptions +): Promise { + const baseDir = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) + const currentDirName = remoteBasename(currentDirAbsPath, host) + let listing: string + try { + listing = await execHostCommand( + conn, + host, + listRemoteInstallBaseDirsCommand(host, baseDir, model) + ) + } catch { + return + } + const entries = listing + .split('\n') + .map((s) => s.trim()) + .filter(Boolean) + .slice(0, MAX_RELAY_GC_LISTING_ENTRIES) + + await cleanupRelayGcTombstones(conn, baseDir, entries, host) + + const versionDirRegex = remoteInstallVersionDirRegex(model) + const pinned = new Set([currentDirName, ...(options.pinnedDirNames ?? [])]) + const candidates = entries + // Why re-check ownership after a prefix-scoped listing: this is the one line that stands + // between a parameterized GC and deleting the sibling model's live install. + .filter((name) => remoteInstallGcPermits(model, name)) + .filter((name) => versionDirRegex.test(name)) + .filter((name) => !pinned.has(name)) + + if (candidates.length === 0) { + return + } + + const removed: string[] = [] + const kept: string[] = [] + for (const name of candidates) { + const dir = joinRemotePath(host, baseDir, name) + try { + const safe = await isCandidateSafeToRemove(conn, model, dir, name, host, options) + if (!safe) { + kept.push(name) + continue + } + // Why: the claim is a sibling, so it survives moving/deleting the candidate and lets installers back out first. + const gcClaimToken = await tryAcquireRelayGcClaim(conn, dir, host) + if (!gcClaimToken) { + kept.push(name) + continue + } + let preserveGcClaim = false + let gcClaimReleaseNeeded = true + try { + // Recheck under the stable claim; installers probe it before and after creating their lock, closing both orders. + if (!(await isCandidateSafeToRemove(conn, model, dir, name, host, options))) { + kept.push(name) + continue + } + if (!(await isRelayGcClaimOwned(conn, dir, gcClaimToken, host))) { + kept.push(name) + continue + } + const tombstone = `${dir}.gc-tombstone.${process.pid}.${Date.now()}` + const moved = await execHostCommand(conn, host, moveRemoteTreeCommand(host, dir, tombstone)) + if (moved.trim() !== 'MOVED') { + kept.push(name) + continue + } + // Once renamed, a fresh install at the original path is isolated from the tombstone's deletion, so release the claim. + const release = await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) + gcClaimReleaseNeeded = release === 'unknown' + await execHostCommand(conn, host, removeRemoteTreeCommand(host, tombstone)) + } catch (err) { + if (isUnconfirmedSshCommandTermination(err)) { + preserveGcClaim = true + } + throw err + } finally { + if (!preserveGcClaim && gcClaimReleaseNeeded) { + await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) + } + } + removed.push(name) + } catch (err) { + console.warn( + `[${model.id}] GC failed for ${dir}: ${err instanceof Error ? err.message : String(err)}` + ) + kept.push(name) + } + } + + if (removed.length > 0) { + const keptSuffix = kept.length > 0 ? ` (kept: ${kept.join(', ')})` : '' + console.log( + `[${model.id}] GC: removed ${removed.length} stale version dir(s): ${removed.join(', ')}${keptSuffix}` + ) + } +} + +async function isCandidateSafeToRemove( + conn: SshConnection, + model: RemoteInstallModel, + dir: string, + name: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options: RemoteInstallGcOptions +): Promise { + const isLegacy = model.id === 'relay' && LEGACY_RELAY_DIR_REGEX.test(name) + + const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME) + let lockProbe: string + try { + lockProbe = await execHostCommand(conn, host, probeInstallLockExistsCommand(host, lockDir)) + } catch { + return false + } + const lockState = lockProbe.trim() + if (lockState !== 'OPEN' && lockState !== 'LOCKED') { + return false + } + const locked = lockState === 'LOCKED' + + if (locked) { + // Why: stale lock = crashed installer; finalize can leave a dir .install-complete yet locked (lock-rm failed), so it's reclaimable. + if (!(await isRelayInstallLockStale(conn, lockDir, host))) { + return false + } + process.stderr.write?.(`[${model.id}] GC: lock at ${lockDir} is stale; treating as recoverable\n`) + } + + // Legacy dirs predate .install-complete; skip the sentinel and rely on the live-socket probe alone. + if (!isLegacy) { + const completePath = joinRemotePath(host, dir, model.installCompleteFilename) + const completeProbe = await execHostCommand( + conn, + host, + probeFileExistsCommand(host, completePath) + ).catch(() => 'PARTIAL') + if (completeProbe.trim() !== 'COMPLETE') { + // Crashed-install partial; leave for the next deploy to recover. + return false + } + } + + return !(await options.isDirLive(dir)) +} + + +/** + * The relay's GC, bound to its own namespace and its own liveness probe (a live unix socket + * or Windows pipe inside the version dir). + */ +export async function gcOldRelayVersions( + conn: SshConnection, + remoteHome: string, + currentDirAbsPath: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: { + windowsNodePath?: string + windowsSockNames?: string[] + } +): Promise { + await gcOldRemoteInstallVersions(conn, RELAY_INSTALL_MODEL, remoteHome, currentDirAbsPath, host, { + ...options, + isDirLive: (dir) => hasLiveRelaySocket(conn, dir, host, options) + }) +} + +async function hasLiveRelaySocket( + conn: SshConnection, + dir: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: { + windowsNodePath?: string + windowsSockNames?: string[] + } +): Promise { + try { + // Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle. + const windowsOptions = + isWindowsRemoteHost(host) && options?.windowsNodePath + ? { + nodePath: options.windowsNodePath, + pipePaths: (options.windowsSockNames ?? []).flatMap((sockName) => + windowsRelayPipePathsForSocketName(host, dir, sockName) + ) + } + : undefined + const out = await execHostCommand( + conn, + host, + relayLivenessProbeCommand(host, dir, windowsOptions) + ) + const state = out.trim() + return state !== 'DEAD' && state !== 'WAITING' + } catch { + // Why: an inconclusive liveness probe must never authorize deletion. + return true + } +} diff --git a/src/main/ssh/remote-install-model.test.ts b/src/main/ssh/remote-install-model.test.ts new file mode 100644 index 000000000000..5a80df8ff4f6 --- /dev/null +++ b/src/main/ssh/remote-install-model.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' + +import { + inventoryRemoteInstallDirs, + ORCAD_INSTALL_MODEL, + RELAY_INSTALL_MODEL, + remoteInstallDirName, + remoteInstallDirOwner, + remoteInstallGcPermits, + remoteInstallListingRegexSource, + remoteInstallVersionDirRegex +} from './remote-install-model' + +const RELAY_DIRS = ['relay-0.1.0+abcdef123456', 'relay-v0.1.0', 'relay-1.2.3'] +const ORCAD_DIRS = ['orcad-0.1.0+abcdef123456', 'orcad-v0.1.0', 'orcad-1.2.3'] + +describe('remote install namespace', () => { + it('names each model its own version dir', () => { + expect(remoteInstallDirName(RELAY_INSTALL_MODEL, '0.1.0+aa')).toBe('relay-0.1.0+aa') + expect(remoteInstallDirName(ORCAD_INSTALL_MODEL, '0.1.0+aa')).toBe('orcad-0.1.0+aa') + }) + + it('keeps the relay listing pattern byte-identical to the one it shipped with', () => { + // The literal that was hardcoded in `listRelayBaseDirsCommand` before it was + // parameterized. A drift here changes what an existing host's GC can see. + expect(remoteInstallListingRegexSource(RELAY_INSTALL_MODEL)).toBe( + String.raw`^relay-(v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?)(\.gc-tombstone\.[0-9]+\.[0-9]+)?$` + ) + }) + + it('refuses a dir prefix that could escape a remote glob or quote', () => { + const injected = { ...RELAY_INSTALL_MODEL, dirPrefix: "relay'; rm -rf ~" } + expect(() => remoteInstallVersionDirRegex(injected)).toThrow('Unsafe remote install dir prefix') + }) +}) + +describe('GC ownership — each model collects only its own namespace', () => { + it.each(ORCAD_DIRS)('the relay never permits GC of %s', (dirName) => { + expect(remoteInstallDirOwner(dirName)).toBe('orcad') + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, dirName)).toBe(false) + }) + + it.each(RELAY_DIRS)('orcad never permits GC of %s', (dirName) => { + expect(remoteInstallDirOwner(dirName)).toBe('relay') + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, dirName)).toBe(false) + }) + + it('permits each model its own dirs and its own tombstones', () => { + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, 'relay-0.1.0+aa')).toBe(true) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'orcad-0.1.0+aa')).toBe(true) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'orcad-0.1.0+aa.gc-tombstone.12.34')).toBe( + true + ) + }) + + it('claims nothing it did not create', () => { + for (const name of ['.orca-remote', 'orcad', 'relayish-0.1.0', 'orcad-notaversion', 'node']) { + expect(remoteInstallDirOwner(name)).toBeNull() + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, name)).toBe(false) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, name)).toBe(false) + } + }) + + it('groups a mixed listing without losing anything to the wrong owner', () => { + const inventory = inventoryRemoteInstallDirs([...RELAY_DIRS, ...ORCAD_DIRS, 'something-else']) + expect(inventory.relay).toEqual(RELAY_DIRS) + expect(inventory.orcad).toEqual(ORCAD_DIRS) + expect(inventory.unknown).toEqual(['something-else']) + }) +}) diff --git a/src/main/ssh/remote-install-model.ts b/src/main/ssh/remote-install-model.ts new file mode 100644 index 000000000000..54be330a55f1 --- /dev/null +++ b/src/main/ssh/remote-install-model.ts @@ -0,0 +1,143 @@ +/** + * The two things Orca installs into `~/.orca-remote/`, and the rules that keep them from + * touching each other. + * + * `docs/design/shipping-orcad.html` §06 settles that on-disk coexistence is permanent: the + * relay is the dumb execution host for SSH-target users, orcad is the peer for paired + * environments, and no plan item retires either. So `relay-/` and `orcad-/` + * sit side by side forever, and the namespace has to be a parameter rather than a literal. + * + * GC ownership is the trap that parameterization creates. Each model garbage-collects ONLY + * its own directories — see `remoteInstallDirOwner`. Relay's regex happened to be narrow + * enough already; making the prefix a parameter is exactly what could have widened it into + * deleting a live orcad tree, so the ownership rule is asserted here rather than left to + * whichever regex a caller passes. + */ +import { + relayArtifactFilenames, + RELAY_INSTALL_COMPLETE_FILENAME, + RELAY_VERSION_FILENAME +} from '../../shared/relay-artifacts' +import { + orcadArtifactFilenames, + ORCAD_INSTALL_COMPLETE_FILENAME, + ORCAD_VERSION_FILENAME +} from '../../shared/orcad-artifacts' + +export type RemoteInstallModelId = 'relay' | 'orcad' + +export type RemoteInstallModel = { + readonly id: RemoteInstallModelId + /** Leading segment of every version dir: `-`. */ + readonly dirPrefix: string + /** npm package name written into the remote `package.json` for native deps. */ + readonly nativeDepsPackageName: string + readonly versionFilename: string + readonly installCompleteFilename: string + /** Files whose absence means a torn install, so the probe forces a re-deploy. */ + requiredArtifacts(isWindows: boolean): string[] +} + +export const RELAY_INSTALL_MODEL: RemoteInstallModel = { + id: 'relay', + dirPrefix: 'relay', + nativeDepsPackageName: 'orca-relay', + versionFilename: RELAY_VERSION_FILENAME, + installCompleteFilename: RELAY_INSTALL_COMPLETE_FILENAME, + requiredArtifacts: (isWindows) => relayArtifactFilenames(isWindows) +} + +export const ORCAD_INSTALL_MODEL: RemoteInstallModel = { + id: 'orcad', + dirPrefix: 'orcad', + nativeDepsPackageName: 'orca-orcad', + versionFilename: ORCAD_VERSION_FILENAME, + installCompleteFilename: ORCAD_INSTALL_COMPLETE_FILENAME, + // Why the parameter is ignored: orcad's forked children are the same three .js files on + // every host. The Windows-only console-list agent patch is a relay/node-pty concern. + requiredArtifacts: () => orcadArtifactFilenames() +} + +export const REMOTE_INSTALL_MODELS: readonly RemoteInstallModel[] = [ + RELAY_INSTALL_MODEL, + ORCAD_INSTALL_MODEL +] + +/** + * The version half of a directory name, shared by both models. + * + * Why `[0-9]` and not `\d`: this exact source string is also embedded in an awk ERE and a + * PowerShell `-match` on the remote host. Those three dialects agree on `[0-9]`, `\.` and + * `\+`; only JavaScript understands `\d`. + */ +const VERSION_PATTERN = String.raw`v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?` + +/** Suffix GC leaves behind mid-delete; the listing must surface these so they can be swept. */ +const TOMBSTONE_PATTERN = String.raw`\.gc-tombstone\.[0-9]+\.[0-9]+` + +/** + * Why validated and not merely typed: the prefix is interpolated into a remote `find -name` + * glob, an awk regex and a single-quoted PowerShell literal. A quote or a metacharacter + * here would be a remote-shell injection on the client's own connection. + */ +function assertSafeDirPrefix(dirPrefix: string): void { + if (!/^[a-z][a-z0-9-]*$/.test(dirPrefix)) { + throw new Error(`Unsafe remote install dir prefix: ${JSON.stringify(dirPrefix)}`) + } +} + +export function remoteInstallDirName(model: RemoteInstallModel, fullVersion: string): string { + assertSafeDirPrefix(model.dirPrefix) + return `${model.dirPrefix}-${fullVersion}` +} + +/** Matches a live version dir for exactly one model — never a tombstone, never a sibling model. */ +export function remoteInstallVersionDirRegex(model: RemoteInstallModel): RegExp { + assertSafeDirPrefix(model.dirPrefix) + return new RegExp(`^${model.dirPrefix}-(${VERSION_PATTERN})$`) +} + +/** What the remote listing is allowed to return: live dirs plus their tombstones. */ +export function remoteInstallListingRegexSource(model: RemoteInstallModel): string { + assertSafeDirPrefix(model.dirPrefix) + return `^${model.dirPrefix}-(${VERSION_PATTERN})(${TOMBSTONE_PATTERN})?$` +} + +/** + * Which model owns a directory found in `~/.orca-remote/`, or null for anything neither + * model created. + * + * This is the answer to §06 falsifier 1's first half: **the model that created a directory + * owns it, and nothing else may delete it.** A relay GC pass that saw `orcad-0.1.0+abc` + * would be looking at the live install of a peer whose lifecycle it has no view into — the + * SSH-execution-boundary collapse in directory form. + */ +export function remoteInstallDirOwner(dirName: string): RemoteInstallModelId | null { + for (const model of REMOTE_INSTALL_MODELS) { + if (new RegExp(remoteInstallListingRegexSource(model)).test(dirName)) { + return model.id + } + } + return null +} + +/** True when `model` is allowed to garbage-collect `dirName`. */ +export function remoteInstallGcPermits(model: RemoteInstallModel, dirName: string): boolean { + return remoteInstallDirOwner(dirName) === model.id +} + +export type RemoteInstallInventory = Record + +/** Group a raw `~/.orca-remote/` listing by owning model, for diagnostics and the client's choice. */ +export function inventoryRemoteInstallDirs(dirNames: readonly string[]): RemoteInstallInventory { + const inventory: RemoteInstallInventory = { relay: [], orcad: [], unknown: [] } + for (const name of dirNames) { + const owner = remoteInstallDirOwner(name) + if (owner) { + inventory[owner].push(name) + } else { + inventory.unknown.push(name) + } + } + return inventory +} diff --git a/src/main/ssh/ssh-relay-install-namespace.ts b/src/main/ssh/ssh-relay-install-namespace.ts index a75b80488416..6fd3c2aa89f2 100644 --- a/src/main/ssh/ssh-relay-install-namespace.ts +++ b/src/main/ssh/ssh-relay-install-namespace.ts @@ -7,6 +7,11 @@ // See: docs/ssh-relay-sftp-namespace.md import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + RELAY_INSTALL_MODEL, + remoteInstallDirName, + type RemoteInstallModel +} from './remote-install-model' import type { SftpNamespacePathMapping } from './sftp-namespace-resolution' import { shellEscape } from './ssh-connection-utils' import { RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' @@ -37,7 +42,19 @@ export function relayRemoteDirSegments( fullVersion: string, pathFlavor: RemotePathFlavor ): string[] { - const segments = [RELAY_REMOTE_DIR, `relay-${fullVersion}`] + return remoteInstallDirSegments(RELAY_INSTALL_MODEL, fullVersion, pathFlavor) +} + +/** + * The model-parameterized form. `relay-` and `orcad-` are permanent siblings under + * one `.orca-remote/` (see remote-install-model.ts), so the prefix is an argument. + */ +export function remoteInstallDirSegments( + model: RemoteInstallModel, + fullVersion: string, + pathFlavor: RemotePathFlavor +): string[] { + const segments = [RELAY_REMOTE_DIR, remoteInstallDirName(model, fullVersion)] for (const segment of segments) { assertSafeRemotePathSegment(segment, pathFlavor) // Why: the version reaches logs and diagnostics, where an embedded CR/LF can forge lines. @@ -49,7 +66,14 @@ export function relayRemoteDirSegments( } export function relayHomeRelativeDir(fullVersion: string): string { - return relayRemoteDirSegments(fullVersion, 'posix').join('/') + return remoteInstallHomeRelativeDir(RELAY_INSTALL_MODEL, fullVersion) +} + +export function remoteInstallHomeRelativeDir( + model: RemoteInstallModel, + fullVersion: string +): string { + return remoteInstallDirSegments(model, fullVersion, 'posix').join('/') } export function createRelayInstallNamespace(homeRelativeRelayDir: string): RelayInstallNamespace { diff --git a/src/main/ssh/ssh-relay-versioned-install.ts b/src/main/ssh/ssh-relay-versioned-install.ts index e4380c319c34..d2ddc62e64bf 100644 --- a/src/main/ssh/ssh-relay-versioned-install.ts +++ b/src/main/ssh/ssh-relay-versioned-install.ts @@ -7,24 +7,12 @@ import { join } from 'node:path' import { existsSync, readFileSync } from 'node:fs' import type { SshConnection } from './ssh-connection' -import { RELAY_REMOTE_DIR } from './relay-protocol' import { execCommand } from './ssh-relay-deploy-helpers' -import { probeInstallLockExistsCommand } from './ssh-relay-install-lock-commands' -import { isRelayInstallLockStale, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' -import { relayRemoteDirSegments } from './ssh-relay-install-namespace' +import { RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' +import { remoteInstallDirSegments } from './ssh-relay-install-namespace' +import { RELAY_INSTALL_MODEL, type RemoteInstallModel } from './remote-install-model' import { - isRelayGcClaimOwned, - releaseRelayGcClaimWithRetry, - tryAcquireRelayGcClaim -} from './ssh-relay-gc-claim' -import { cleanupRelayGcTombstones } from './ssh-relay-gc-tombstone' -import { - listRelayBaseDirsCommand, - MAX_RELAY_GC_LISTING_ENTRIES, - moveRemoteTreeCommand, - probeFileExistsCommand, - probeRelayInstalledCommand, - relayLivenessProbeCommand, + probeRemoteInstallCompleteCommand, removeRemoteTreeCommand, writeRemoteEmptyFileCommand } from './ssh-remote-commands' @@ -32,20 +20,11 @@ import { getRemoteHostPlatform, isWindowsRemoteHost, joinRemotePath, - remoteBasename, type RemoteHostPlatform, type RemotePathFlavor } from './ssh-remote-platform' -import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' -import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' import { isSshSessionLimitError } from './ssh-session-limit-error' -// Single source of truth for GC and the version-dir parser; matches both the new and legacy relay-dir layouts. -const RELAY_VERSION_DIR_REGEX = /^relay-(v?\d+\.\d+\.\d+(\+[0-9a-f]+)?)$/ - -// Legacy dirs predate `.install-complete`; they need a liveness-only GC check so they eventually drain. -const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/ - const INSTALL_COMPLETE_NAME = '.install-complete' const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') @@ -97,13 +76,23 @@ export function computeRemoteRelayDir( remoteHome: string, fullVersion: string, pathFlavor: RemotePathFlavor = 'posix' +): string { + // Why: shell and SFTP-relative builders must derive the same validated segments or the namespaces diverge. + return computeRemoteInstallDir(RELAY_INSTALL_MODEL, remoteHome, fullVersion, pathFlavor) +} + +/** The model-parameterized form of `computeRemoteRelayDir`. */ +export function computeRemoteInstallDir( + model: RemoteInstallModel, + remoteHome: string, + fullVersion: string, + pathFlavor: RemotePathFlavor = 'posix' ): string { const host = pathFlavor === 'windows' ? getRemoteHostPlatform('win32-x64') : getRemoteHostPlatform('linux-x64') - // Why: shell and SFTP-relative builders must derive the same validated segments or the namespaces diverge. - return joinRemotePath(host, remoteHome, ...relayRemoteDirSegments(fullVersion, pathFlavor)) + return joinRemotePath(host, remoteHome, ...remoteInstallDirSegments(model, fullVersion, pathFlavor)) } /** @@ -116,11 +105,26 @@ export async function isRelayAlreadyInstalled( host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, options?: RelayInstalledProbeOptions ): Promise { + return isRemoteInstallComplete(conn, RELAY_INSTALL_MODEL, remoteRelayDir, host, options) +} + +/** The model-parameterized form: each model probes for its own artifact list. */ +export async function isRemoteInstallComplete( + conn: SshConnection, + model: RemoteInstallModel, + remoteInstallDir: string, + host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, + options?: RelayInstalledProbeOptions +): Promise { + const remoteRelayDir = remoteInstallDir try { const probe = await execHostCommand( conn, host, - probeRelayInstalledCommand(host, remoteRelayDir), + probeRemoteInstallCompleteCommand(host, remoteRelayDir, [ + ...model.requiredArtifacts(isWindowsRemoteHost(host)), + model.installCompleteFilename + ]), { signal: options?.signal } ) return probe.trim() === 'OK' @@ -175,188 +179,10 @@ export async function abandonInstall( * unlocked sibling version dir (never the current one). Best-effort — errors * are swallowed so GC never blocks the user from connecting. */ -export async function gcOldRelayVersions( - conn: SshConnection, - remoteHome: string, - currentDirAbsPath: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - const baseDir = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) - const currentDirName = remoteBasename(currentDirAbsPath, host) - let listing: string - try { - listing = await execHostCommand(conn, host, listRelayBaseDirsCommand(host, baseDir)) - } catch { - return - } - const entries = listing - .split('\n') - .map((s) => s.trim()) - .filter(Boolean) - .slice(0, MAX_RELAY_GC_LISTING_ENTRIES) - - await cleanupRelayGcTombstones(conn, baseDir, entries, host) - - const candidates = entries - .filter((name) => RELAY_VERSION_DIR_REGEX.test(name)) - .filter((name) => name !== currentDirName) - - if (candidates.length === 0) { - return - } - - const removed: string[] = [] - const kept: string[] = [] - for (const name of candidates) { - const dir = joinRemotePath(host, baseDir, name) - try { - const safe = await isCandidateSafeToRemove(conn, dir, name, host, options) - if (!safe) { - kept.push(name) - continue - } - // Why: the claim is a sibling, so it survives moving/deleting the candidate and lets installers back out first. - const gcClaimToken = await tryAcquireRelayGcClaim(conn, dir, host) - if (!gcClaimToken) { - kept.push(name) - continue - } - let preserveGcClaim = false - let gcClaimReleaseNeeded = true - try { - // Recheck under the stable claim; installers probe it before and after creating their lock, closing both orders. - if (!(await isCandidateSafeToRemove(conn, dir, name, host, options))) { - kept.push(name) - continue - } - if (!(await isRelayGcClaimOwned(conn, dir, gcClaimToken, host))) { - kept.push(name) - continue - } - const tombstone = `${dir}.gc-tombstone.${process.pid}.${Date.now()}` - const moved = await execHostCommand(conn, host, moveRemoteTreeCommand(host, dir, tombstone)) - if (moved.trim() !== 'MOVED') { - kept.push(name) - continue - } - // Once renamed, a fresh install at the original path is isolated from the tombstone's deletion, so release the claim. - const release = await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) - gcClaimReleaseNeeded = release === 'unknown' - await execHostCommand(conn, host, removeRemoteTreeCommand(host, tombstone)) - } catch (err) { - if (isUnconfirmedSshCommandTermination(err)) { - preserveGcClaim = true - } - throw err - } finally { - if (!preserveGcClaim && gcClaimReleaseNeeded) { - await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host) - } - } - removed.push(name) - } catch (err) { - console.warn( - `[ssh-relay] GC failed for ${dir}: ${err instanceof Error ? err.message : String(err)}` - ) - kept.push(name) - } - } - - if (removed.length > 0) { - const keptSuffix = kept.length > 0 ? ` (kept: ${kept.join(', ')})` : '' - console.log( - `[ssh-relay] GC: removed ${removed.length} stale version dir(s): ${removed.join(', ')}${keptSuffix}` - ) - } -} - -async function isCandidateSafeToRemove( - conn: SshConnection, - dir: string, - name: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - const isLegacy = LEGACY_RELAY_DIR_REGEX.test(name) - - const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME) - let lockProbe: string - try { - lockProbe = await execHostCommand(conn, host, probeInstallLockExistsCommand(host, lockDir)) - } catch { - return false - } - const lockState = lockProbe.trim() - if (lockState !== 'OPEN' && lockState !== 'LOCKED') { - return false - } - const locked = lockState === 'LOCKED' - - if (locked) { - // Why: stale lock = crashed installer; finalize can leave a dir .install-complete yet locked (lock-rm failed), so it's reclaimable. - if (!(await isRelayInstallLockStale(conn, lockDir, host))) { - return false - } - process.stderr.write?.(`[ssh-relay] GC: lock at ${lockDir} is stale; treating as recoverable\n`) - } - - // Legacy dirs predate .install-complete; skip the sentinel and rely on the live-socket probe alone. - if (!isLegacy) { - const completePath = joinRemotePath(host, dir, INSTALL_COMPLETE_NAME) - const completeProbe = await execHostCommand( - conn, - host, - probeFileExistsCommand(host, completePath) - ).catch(() => 'PARTIAL') - if (completeProbe.trim() !== 'COMPLETE') { - // Crashed-install partial; leave for the next deploy to recover. - return false - } - } - - const sockAlive = await hasLiveRelaySocket(conn, dir, host, options) - if (sockAlive) { - return false - } - return true -} - -async function hasLiveRelaySocket( - conn: SshConnection, - dir: string, - host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, - options?: { - windowsNodePath?: string - windowsSockNames?: string[] - } -): Promise { - try { - // Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle. - const windowsOptions = - isWindowsRemoteHost(host) && options?.windowsNodePath - ? { - nodePath: options.windowsNodePath, - pipePaths: (options.windowsSockNames ?? []).flatMap((sockName) => - windowsRelayPipePathsForSocketName(host, dir, sockName) - ) - } - : undefined - const out = await execHostCommand( - conn, - host, - relayLivenessProbeCommand(host, dir, windowsOptions) - ) - const state = out.trim() - return state !== 'DEAD' && state !== 'WAITING' - } catch { - // Why: an inconclusive liveness probe must never authorize deletion. - return true - } -} +// Why re-exported rather than moved outright: deploy and the relay tests import the whole +// versioned-install surface from here, and the split exists for file size, not to redraw an API. +export { + gcOldRelayVersions, + gcOldRemoteInstallVersions, + type RemoteInstallGcOptions +} from './remote-install-gc' diff --git a/src/main/ssh/ssh-remote-commands.ts b/src/main/ssh/ssh-remote-commands.ts index 4fa1445969d2..fc3e965193ed 100644 --- a/src/main/ssh/ssh-remote-commands.ts +++ b/src/main/ssh/ssh-remote-commands.ts @@ -2,6 +2,11 @@ import { RELAY_INSTALL_COMPLETE_FILENAME, relayArtifactFilenames } from '../../shared/relay-artifacts' +import { + RELAY_INSTALL_MODEL, + remoteInstallListingRegexSource, + type RemoteInstallModel +} from './remote-install-model' import type { RemoteHostPlatform } from './ssh-remote-platform' import { isWindowsRemoteHost, joinRemotePath, remoteDirname } from './ssh-remote-platform' import { powerShellCommand, powerShellLiteral, powerShellNativeArg } from './ssh-remote-powershell' @@ -99,10 +104,28 @@ export function probeRelayInstalledCommand( host: RemoteHostPlatform, remoteRelayDir: string ): string { - const required = [ + return probeRemoteInstallCompleteCommand(host, remoteRelayDir, [ ...relayArtifactFilenames(isWindowsRemoteHost(host)), RELAY_INSTALL_COMPLETE_FILENAME - ].map((filename) => joinRemotePath(host, remoteRelayDir, filename)) + ]) +} + +/** + * The model-agnostic form: any install is complete when its directory exists and every + * named artifact is a regular file inside it. + * + * Why the caller passes the list: orcad and the relay ship different artifacts, and a probe + * that checked a shared subset would call a torn install complete. + */ +export function probeRemoteInstallCompleteCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + requiredFilenames: readonly string[] +): string { + const remoteRelayDir = remoteInstallDir + const required = requiredFilenames.map((filename) => + joinRemotePath(host, remoteRelayDir, filename) + ) if (!isWindowsRemoteHost(host)) { const fileTests = required.map((path) => `&& test -f ${shellEscape(path)} `).join('') return `test -d ${shellEscape(remoteRelayDir)} ${fileTests}&& echo OK || echo MISSING` @@ -121,12 +144,28 @@ export function probeRelayInstalledCommand( export const MAX_RELAY_GC_LISTING_ENTRIES = 64 export function listRelayBaseDirsCommand(host: RemoteHostPlatform, baseDir: string): string { + return listRemoteInstallBaseDirsCommand(host, baseDir, RELAY_INSTALL_MODEL) +} + +/** + * List one model's version dirs (and its own tombstones) under `~/.orca-remote/`. + * + * The model scopes BOTH the `find`/`Get-ChildItem` glob and the validating regex. That + * double filter is the on-the-wire half of the GC ownership rule: an orcad GC pass never + * even receives a relay directory name, so it cannot delete one through a later bug. + */ +export function listRemoteInstallBaseDirsCommand( + host: RemoteHostPlatform, + baseDir: string, + model: RemoteInstallModel +): string { + const namePattern = remoteInstallListingRegexSource(model) if (!isWindowsRemoteHost(host)) { const statusPrefix = '__ORCA_RELAY_GC_FIND_STATUS__' return [ `base=${shellEscape(baseDir)}; [ -d "$base" ] || exit 0;`, - `{ find "$base" -mindepth 1 -maxdepth 1 -type d -name 'relay-*' -print; status=$?; printf '\n${statusPrefix}%s\n' "$status"; } |`, - String.raw`awk 'BEGIN { count=0; status=-1 } /^${statusPrefix}[0-9]+$/ { status=substr($0, ${statusPrefix.length + 1}); next } { name=$0; sub(/^.*\//, "", name); if (name ~ /^relay-(v?[0-9]+\.[0-9]+\.[0-9]+(\+[0-9a-f]+)?)(\.gc-tombstone\.[0-9]+\.[0-9]+)?$/ && count < ${MAX_RELAY_GC_LISTING_ENTRIES}) { entries[count++]=name } } END { if (status != 0) exit 1; for (i=0; i !artifact.optional).map( + (artifact) => artifact.filename + ) +}