diff --git a/packages/runtime-host/src/__tests__/owned-candidate.test.ts b/packages/runtime-host/src/__tests__/owned-candidate.test.ts index 94e3a6b41e..7687bbe733 100644 --- a/packages/runtime-host/src/__tests__/owned-candidate.test.ts +++ b/packages/runtime-host/src/__tests__/owned-candidate.test.ts @@ -7,9 +7,12 @@ import { INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, RUNTIME_HOST_PROTOCOL_VERSION, } from '../protocol/index.js'; -import { connectOwnedRuntimeHostWithDependencies } from '../client/connect-or-spawn.js'; +import { + connectOrSpawnRuntimeHostWithDependencies, + connectOwnedRuntimeHostWithDependencies, +} from '../client/connect-or-spawn.js'; import { runHostedExecution } from '../client/hosted-execution.js'; -import { launchOwnedRuntimeHostCandidate } from '../client/launcher.js'; +import { launchOwnedRuntimeHostCandidate, type OwnedCandidateAttempt } from '../client/launcher.js'; test('owned connection keeps a fresh Host alive for its full election window', async () => { const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-first-connection-')); @@ -47,6 +50,173 @@ test('owned connection keeps a fresh Host alive for its full election window', a } }); +test('owned connect returns a missed election without waiting for a late candidate', { + timeout: 15_000, +}, async () => { + const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-startup-timeout-')); + let launched = 0; + let released = 0; + let settled = 0; + let resolveSpawned!: (host: OwnedCandidateAttempt) => void; + let rejectSpawned!: (error: Error) => void; + let spawnedSettled = false; + const spawned = new Promise((resolve, reject) => { + resolveSpawned = resolve; + rejectSpawned = reject; + }); + void spawned.then( + () => { + spawnedSettled = true; + }, + () => { + spawnedSettled = true; + }, + ); + + try { + const result = await connectOwnedRuntimeHostWithDependencies( + { + rootPath, + surface: 'run', + protocol: { + min: RUNTIME_HOST_PROTOCOL_VERSION, + max: RUNTIME_HOST_PROTOCOL_VERSION, + }, + compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, + electionDeadlineMs: 250, + }, + { + launchCandidate: () => { + launched += 1; + return { spawned }; + }, + }, + ); + + assert.ok(launched >= 1, 'launchCandidate must run before the election ends'); + assert.equal(result.kind, 'failed'); + assert.equal(connectFailure(result), 'failed:startup_timeout'); + assert.equal(spawnedSettled, false); + + resolveSpawned({ + pid: 4_242, + releaseToEnvironment() { + released += 1; + }, + async settle() { + settled += 1; + return true; + }, + }); + await spawned; + assert.equal(released, 1); + assert.equal(settled, 0); + } finally { + if (!spawnedSettled) rejectSpawned(new Error('abandoned after election')); + } +}); + +test('a late owned candidate stays alive after another client adopts it', { + timeout: 25_000, +}, async () => { + const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-late-adopt-')); + let launchedHost: OwnedCandidateAttempt | undefined; + let resolveLate!: (host: OwnedCandidateAttempt) => void; + let rejectLate!: (error: Error) => void; + let lateDelivered = false; + const lateSpawned = new Promise((resolve, reject) => { + resolveLate = resolve; + rejectLate = reject; + }); + + try { + const missed = await connectOwnedRuntimeHostWithDependencies( + { + rootPath, + surface: 'run', + protocol: { + min: RUNTIME_HOST_PROTOCOL_VERSION, + max: RUNTIME_HOST_PROTOCOL_VERSION, + }, + compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, + electionDeadlineMs: 500, + }, + { + launchCandidate(input) { + const launch = launchOwnedRuntimeHostCandidate({ + ...input, + initialConnectionTimeoutMs: 10_000, + }); + void launch.spawned.then((host) => { + launchedHost = host; + }); + return { spawned: lateSpawned }; + }, + }, + ); + + assert.equal(missed.kind, 'failed'); + assert.equal(connectFailure(missed), 'failed:startup_timeout'); + + const host = await waitForDefined( + () => launchedHost, + 8_000, + 'late owned candidate did not spawn', + ); + const adopted = await connectOrSpawnRuntimeHostWithDependencies( + { + rootPath, + surface: 'run', + protocol: { + min: RUNTIME_HOST_PROTOCOL_VERSION, + max: RUNTIME_HOST_PROTOCOL_VERSION, + }, + compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, + candidateEntrypoint: new URL('../execution-candidate-main.js', import.meta.url), + electionDeadlineMs: 8_000, + }, + { + random: Math.random, + launchCandidate() { + throw new Error('adopter must use the late owned candidate'); + }, + }, + ); + + try { + assert.equal(adopted.kind, 'connected', connectFailure(adopted)); + if (adopted.kind !== 'connected') return; + + let released = 0; + let settled = 0; + lateDelivered = true; + resolveLate({ + ...host, + releaseToEnvironment() { + released += 1; + host.releaseToEnvironment(); + }, + async settle(timeoutMs) { + settled += 1; + return host.settle(timeoutMs); + }, + }); + await lateSpawned; + assert.equal(released, 1); + assert.equal(settled, 0); + + const diagnostics = await adopted.connection.queryHostDiagnostics(); + assert.equal(diagnostics.pid, host.pid); + assert.doesNotThrow(() => process.kill(host.pid, 0)); + } finally { + if (adopted.kind === 'connected') await adopted.connection.close(); + } + } finally { + if (!lateDelivered) rejectLate(new Error('abandoned after election')); + if (launchedHost) await launchedHost.settle(5_000); + } +}); + test('owned Host exits promptly after its first connection closes', async () => { const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-first-disconnect-')); const result = await connectOwnedRuntimeHostWithDependencies( @@ -58,12 +228,15 @@ test('owned Host exits promptly after its first connection closes', async () => max: RUNTIME_HOST_PROTOCOL_VERSION, }, compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, - electionDeadlineMs: 2_000, + // Connection is not the promptness claim. Two seconds is enough on an + // idle machine and too little under a full CI suite, where spawn plus + // handshake routinely miss the window and surface as `failed`. + electionDeadlineMs: 8_000, }, { launchCandidate: launchOwnedRuntimeHostCandidate }, ); - assert.equal(result.kind, 'connected'); + assert.equal(result.kind, 'connected', connectFailure(result)); if (result.kind !== 'connected') return; await result.connection.close(); assert.equal(await result.host.settle(500), true); @@ -138,3 +311,30 @@ test('pre-cancelled hosted execution does not start a Runtime Host', async () => assert.equal(result.kind, 'indeterminate'); assert.deepEqual(await readdir(rootPath), []); }); + +function connectFailure( + result: + | Awaited> + | Awaited>, +): string { + if (result.kind === 'connected') return 'connected'; + if (result.kind === 'failed') return `failed:${result.reason}`; + if (result.kind === 'upgrade_required') return 'upgrade_required'; + return `incompatible:${result.handshake.replacement}`; +} + +async function waitForDefined( + read: () => T | undefined, + timeoutMs: number, + message: string, +): Promise { + const deadline = performance.now() + timeoutMs; + for (;;) { + const value = read(); + if (value !== undefined) return value; + if (performance.now() >= deadline) throw new Error(message); + await new Promise((resolve) => { + setTimeout(resolve, 20); + }); + } +} diff --git a/packages/runtime-host/src/client/connect-or-spawn.ts b/packages/runtime-host/src/client/connect-or-spawn.ts index 962292945c..62cbb7b9d2 100644 --- a/packages/runtime-host/src/client/connect-or-spawn.ts +++ b/packages/runtime-host/src/client/connect-or-spawn.ts @@ -127,11 +127,17 @@ export async function connectOwnedRuntimeHostWithDependencies( random: Math.random, }, ); - const host = await launch?.spawned; - if (result.kind !== 'connected' || !host) { - if (result.kind === 'connected') await result.connection.close(); - await host?.settle(1_000); - return result.kind === 'connected' ? { kind: 'failed', reason: 'existing_host' } : result; + if (result.kind !== 'connected') { + releaseOwnedLaunch(launch); + return result; + } + const host = await launch?.spawned.catch(() => undefined); + if (!host) { + await result.connection.close(); + return { + kind: 'failed', + reason: launch ? 'host_unresponsive' : 'existing_host', + }; } const ownedConnection = result.connection; connection = ownedConnection; @@ -145,12 +151,22 @@ export async function connectOwnedRuntimeHostWithDependencies( return { kind: 'connected', connection: ownedConnection, host }; } catch { await connection?.close().catch(() => undefined); - const host = await launch?.spawned.catch(() => undefined); - await host?.settle(1_000); + releaseOwnedLaunch(launch); return { kind: 'failed', reason: 'host_unresponsive' }; } } +function releaseOwnedLaunch( + launch: ReturnType | undefined, +): void { + if (!launch) return; + void launch.spawned + .then((host) => { + host.releaseToEnvironment(); + }) + .catch(() => undefined); +} + export async function connectOrSpawnRuntimeHostWithDependencies( input: ConnectOrSpawnRuntimeHostInput, dependencies: ConnectOrSpawnRuntimeHostDependencies,