diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 2e330f305a..b3c6ff9d85 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -1268,7 +1268,7 @@ describe('non-serving Runtime Host kernel', () => { }); }); - test('process-exit retention closes admission before requiring termination without releasing ownership', async () => { + test('process-exit retention neither stalls the graceful close nor retains ownership', async () => { await withHostPaths(async (paths) => { const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); const owner = await tryAcquireInteractiveRootOwner(capability); @@ -1276,7 +1276,7 @@ describe('non-serving Runtime Host kernel', () => { const host = await RuntimeHostKernel.start({ owner, idleGraceMs: 10_000, - shutdownGraceMs: 50, + shutdownGraceMs: 10_000, composition: defineInteractiveRuntimeHostComposition(async (context) => { context.retainUntilProcessExit(); context.retainUntilProcessExit(); @@ -1286,11 +1286,12 @@ describe('non-serving Runtime Host kernel', () => { }); try { - await assert.rejects( + // The anti-idle marker is not work: the drain it accompanies closes + // gracefully, long before the shutdown deadline. + await withTimeout( host.closed, - (error: unknown) => - error instanceof RuntimeHostProcessTerminationRequiredError && - error.code === 'process_termination_required', + 2_000, + 'retained Host waited out its shutdown deadline instead of closing gracefully', ); await assert.rejects( () => openSocket(host.endpoint), @@ -1300,7 +1301,9 @@ describe('non-serving Runtime Host kernel', () => { ((error as NodeJS.ErrnoException).code === 'ENOENT' || (error as NodeJS.ErrnoException).code === 'ECONNREFUSED'), ); - assert.equal(await tryAcquireInteractiveRootOwner(capability), undefined); + const successor = await tryAcquireInteractiveRootOwner(capability); + assert.ok(successor, 'graceful close must release the State Root writer lease'); + await successor?.close(); } finally { await owner.close(); } @@ -1405,6 +1408,75 @@ describe('non-serving Runtime Host kernel', () => { }); }); + test('an in-flight handshake keeps an ephemeral Host alive past the idle deadline', async () => { + await withHostPaths(async (paths) => { + const candidate = await startTestRuntimeHostCandidate(paths, { + rootPath: paths.root, + idleGraceMs: 250, + initialConnectionTimeoutMs: 5_000, + handshakeTimeoutMs: 5_000, + }); + assert.equal(candidate.kind, 'winner'); + if (candidate.kind !== 'winner') return; + const host = candidate.host; + + // The first accepted connection leaves and the idle timer arms; a + // handshake that begins now is the phase the idle timer used to be + // blind to. + const first = await retryConnect(paths, CURRENT_PROTOCOL); + assert.equal(first.kind, 'connected'); + if (first.kind !== 'connected') return; + await first.connection.close(); + + const silent = await openSocket(host.endpoint); + await new Promise((resolve) => setTimeout(resolve, 50)); + try { + // Past the idle deadline with the handshake in flight: the Host must + // not drain under a connecting Client. + await new Promise((resolve) => setTimeout(resolve, 500)); + assert.equal(host.state, 'ready'); + } finally { + silent.destroy(); + } + // Once the handshake settles, the idle timer re-arms and the Host exits. + await withTimeout( + host.closed, + 5_000, + 'ephemeral Host never idle-exited after the handshake settled', + ); + }); + }); + + test('a poisoned Host closes gracefully without waiting out the shutdown deadline', async () => { + await withHostPaths(async (paths) => { + const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + const host = await RuntimeHostKernel.start({ + owner, + lifecycleMode: 'service', + shutdownGraceMs: 10_000, + composition: defineInteractiveRuntimeHostComposition(async (context) => { + // Mirror the poison/fatal path: the anti-idle marker must not stall + // the drain it accompanies. + context.retainUntilProcessExit(); + context.requestDrain(); + return { + handlers: createUnavailableDomainOperationHandlers(), + beginDrain() {}, + async recover() {}, + async close() {}, + }; + }), + }); + await withTimeout( + host.closed, + 2_000, + 'poisoned Host waited out its shutdown deadline instead of closing gracefully', + ); + }); + }); + test('drain requested before factory completion begins drain before recovery exactly once', async () => { await withHostPaths(async (paths) => { const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); @@ -1591,18 +1663,26 @@ describe('non-serving Runtime Host kernel', () => { staleWhileResident.abort(); await staleWhileResident.closed; - const blocked = await connectOrSpawnRuntimeHost({ - ...paths, - rootPath: paths.root, - protocol: LEGACY_PROTOCOL, - compositionId: KERNEL_COMPOSITION.descriptor.id, - candidateEntrypoint: KERNEL_CANDIDATE_ENTRYPOINT, - electionDeadlineMs: 2_000, - }); - assert.equal(blocked.kind, 'incompatible'); - if (blocked.kind === 'incompatible') { - assert.equal(blocked.handshake.replacement, 'blocked_by_residency'); + const blockedWhileResident = new FramedTransport(await openSocket(candidate.host.endpoint)); + await writeRawLocalIpc( + blockedWhileResident, + encodeLegacyProtocolFrame({ + kind: 'hello', + clientInstanceId: 'blocked-legacy-resident', + protocolMin: LEGACY_PROTOCOL.min, + protocolMax: LEGACY_PROTOCOL.max, + }), + ); + const blockedResponse = decodeHostFrame(await blockedWhileResident.read(1_000)); + assert.ok('kind' in blockedResponse && blockedResponse.kind === 'incompatible'); + if ('kind' in blockedResponse && blockedResponse.kind === 'incompatible') { + assert.equal(blockedResponse.replacement, 'blocked_by_residency'); } + blockedWhileResident.abort(); + await blockedWhileResident.closed; + // The rejected handshake's teardown is asynchronous Host-side; let it + // settle so only the next probe's own handshake remains in flight. + await sleep(50); await resident.connection.close(); const staleAtIdle = new FramedTransport(await openSocket(candidate.host.endpoint)); diff --git a/packages/runtime-host/src/__tests__/host-residency-registry.test.ts b/packages/runtime-host/src/__tests__/host-residency-registry.test.ts index 09130ed308..bb914b9381 100644 --- a/packages/runtime-host/src/__tests__/host-residency-registry.test.ts +++ b/packages/runtime-host/src/__tests__/host-residency-registry.test.ts @@ -46,3 +46,55 @@ test('Host residency registry explains liveness and drains on exact release', as assert.equal(registry.activeCount, 0); assert.deepEqual(registry.snapshot(), []); }); + +test('idle-kind residencies block liveness but never the drain', async () => { + const registry = new HostResidencyRegistry(); + const marker = registry.acquire('process-retention', 'idle'); + const work = registry.acquire('hosted-execution'); + + assert.equal(registry.activeCount, 2); + assert.equal(registry.drainCount, 1); + assert.deepEqual(registry.snapshot(), [ + { label: 'hosted-execution', count: 1 }, + { label: 'process-retention', count: 1 }, + ]); + + let drained = false; + const drain = registry.waitForEmpty().then(() => { + drained = true; + }); + await Promise.resolve(); + assert.equal(drained, false); + work.release(); + await drain; + assert.equal(drained, true); + assert.equal(registry.activeCount, 1); + assert.equal(registry.drainCount, 0); + + const resource = registry.acquire('runtime-resource'); + let exceptResolved = false; + const except = registry.waitForEmptyExcept('runtime-resource').then(() => { + exceptResolved = true; + }); + await Promise.resolve(); + assert.equal(exceptResolved, true); + resource.release(); + await except; + marker.release(); + assert.equal(registry.activeCount, 0); + assert.deepEqual(registry.snapshot(), []); +}); + +test('idle-kind residency release resolves only drain waiters when nothing drains', async () => { + const registry = new HostResidencyRegistry(); + const marker = registry.acquire('process-retention', 'idle'); + let drained = false; + const drain = registry.waitForEmpty().then(() => { + drained = true; + }); + await drain; + assert.equal(drained, true); + assert.equal(registry.activeCount, 1); + marker.release(); + assert.equal(registry.activeCount, 0); +}); diff --git a/packages/runtime-host/src/server/host-kernel.ts b/packages/runtime-host/src/server/host-kernel.ts index 9655398dda..81de1c498e 100644 --- a/packages/runtime-host/src/server/host-kernel.ts +++ b/packages/runtime-host/src/server/host-kernel.ts @@ -407,6 +407,9 @@ export class RuntimeHostKernel { void this.#serveConnection(connection).finally(() => { this.#handshakingTransports.delete(transport); this.#transportAuthorities.delete(transport); + // A handshake that never completes keeps the Host visible to the idle + // timer while it is in flight; once it settles, idle must re-evaluate. + this.#scheduleIdleIfNeeded(); }); } @@ -516,7 +519,7 @@ export class RuntimeHostKernel { hello.generation !== undefined && hello.generation !== this.#options.generation; if (generationMismatch && hello.takeover?.expectedHostEpoch === this.hostEpoch) { - if (authority.principalKind === 'local_owner' && this.#isTrueIdle()) { + if (authority.principalKind === 'local_owner' && this.#isTrueIdle(transport)) { this.#requestDrain(); return { kind: 'draining', @@ -543,7 +546,7 @@ export class RuntimeHostKernel { ...(this.#options.generation === undefined ? {} : { generation: this.#options.generation }), state: admittedState, replacement: - this.#lifecycle.kind === 'ephemeral' && this.#isTrueIdle() + this.#lifecycle.kind === 'ephemeral' && this.#isSettledForReplacementAdvice() ? 'wait_for_idle_exit' : 'blocked_by_residency', ...(generationMismatch && authority.principalKind === 'local_owner' @@ -670,7 +673,9 @@ export class RuntimeHostKernel { #retainUntilProcessExit(): void { if (this.#retainedUntilProcessExit) return; this.#retainedUntilProcessExit = true; - this.#residencies.acquire('process-retention'); + // Not work in flight: the marker only blocks idle exit, so it must not + // stall the drain it accompanies. + this.#residencies.acquire('process-retention', 'idle'); this.#cancelIdle(); } @@ -866,7 +871,7 @@ export class RuntimeHostKernel { // requires explicit interruption authority before retirement. if (this.#acceptedTransports.size > 1) return true; if (this.#activeCommandOperations > 1) return true; - return this.#residencies.snapshot().some(({ label }) => label !== 'process-retention'); + return this.#residencies.drainCount > 0; } #beginCompositionDrain(): void { @@ -893,8 +898,9 @@ export class RuntimeHostKernel { if (this.#shutdownRequested) return; // One timer authority per lifecycle phase: until the first connection is // accepted, only #initialConnectionDeadline governs (it defers under an - // in-flight handshake, which #isTrueIdle() cannot see); afterwards the - // idle timer owns the idleGraceMs exit. + // in-flight handshake up to a bounded number of times); afterwards the + // idle timer owns the idleGraceMs exit, with in-flight handshakes visible + // to #isTrueIdle(). if (!this.#hasAcceptedConnection) return; if (!this.#isTrueIdle() || this.#idleTimer) return; this.#idleTimer = setTimeout(() => { @@ -904,7 +910,28 @@ export class RuntimeHostKernel { }, this.#lifecycle.idleGraceMs); } - #isTrueIdle(): boolean { + #isTrueIdle(exceptHandshaking?: RuntimeHostMessageTransport): boolean { + // A transport mid-handshake keeps the Host busy, except the one whose + // admission is being decided right now: counting it would make every + // true-idle takeover observe itself as activity. + const handshaking = + exceptHandshaking !== undefined && this.#handshakingTransports.has(exceptHandshaking) + ? this.#handshakingTransports.size - 1 + : this.#handshakingTransports.size; + return ( + this.#state === 'ready' && + this.#acceptedTransports.size === 0 && + handshaking === 0 && + this.#activeOperations === 0 && + this.#residencies.activeCount === 0 + ); + } + + // The replacement advice in a rejection is what a stale Client acts on. + // In-flight handshakes resolve within milliseconds and must not flip that + // advice, so unlike the idle timer and the takeover decision it ignores + // the handshaking set entirely. + #isSettledForReplacementAdvice(): boolean { return ( this.#state === 'ready' && this.#acceptedTransports.size === 0 && diff --git a/packages/runtime-host/src/server/host-residency-registry.ts b/packages/runtime-host/src/server/host-residency-registry.ts index cbb94fc4fe..069635b7c4 100644 --- a/packages/runtime-host/src/server/host-residency-registry.ts +++ b/packages/runtime-host/src/server/host-residency-registry.ts @@ -21,33 +21,61 @@ import type { OperationResidency } from './operation-dispatcher.js'; const RESIDENCY_LABEL_PATTERN = /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/; +/** + * Residencies answer two different questions and the kinds must not be + * conflated: + * + * - `drain` residencies mark real work in flight. They block idle exit and + * must settle before a graceful close completes. + * - `idle` residencies only block idle exit. A marker such as + * process-retention is not work: it must not stall a drain or count as + * activity that blocks a maintenance probe. + */ +export type HostResidencyKind = 'drain' | 'idle'; + export interface HostResidencySnapshot { readonly label: string; readonly count: number; } +interface HostResidencyCounts { + total: number; + drain: number; +} + export class HostResidencyRegistry { - readonly #counts = new Map(); + readonly #counts = new Map(); readonly #drainWaiters = new Set<{ readonly excludedLabel: string | undefined; readonly resolve: () => void; }>(); #activeCount = 0; + #drainCount = 0; + /** Residencies of either kind keep the process alive against idle exit. */ get activeCount(): number { return this.#activeCount; } - acquire(label: string): OperationResidency { + /** Only drain-kind residencies block maintenance probes and graceful close. */ + get drainCount(): number { + return this.#drainCount; + } + + acquire(label: string, kind: HostResidencyKind = 'drain'): OperationResidency { requireResidencyLabel(label); this.#activeCount += 1; - this.#counts.set(label, (this.#counts.get(label) ?? 0) + 1); + if (kind === 'drain') this.#drainCount += 1; + const counts = this.#counts.get(label) ?? { total: 0, drain: 0 }; + counts.total += 1; + if (kind === 'drain') counts.drain += 1; + this.#counts.set(label, counts); let active = true; return { release: () => { if (!active) return; active = false; - this.#release(label); + this.#release(label, kind); }, }; } @@ -55,41 +83,45 @@ export class HostResidencyRegistry { snapshot(): readonly HostResidencySnapshot[] { return [...this.#counts] .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([label, count]) => Object.freeze({ label, count })); + .map(([label, counts]) => Object.freeze({ label, count: counts.total })); } waitForEmpty(): Promise { - return this.#waitForEmptyExcept(undefined); + return this.#waitForDrainEmptyExcept(undefined); } waitForEmptyExcept(excludedLabel: string): Promise { requireResidencyLabel(excludedLabel); - return this.#waitForEmptyExcept(excludedLabel); + return this.#waitForDrainEmptyExcept(excludedLabel); } - #release(label: string): void { - const count = this.#counts.get(label); - if (count === undefined || count === 0 || this.#activeCount === 0) { + #release(label: string, kind: HostResidencyKind): void { + const counts = this.#counts.get(label); + if (counts === undefined || counts.total === 0 || this.#activeCount === 0) { throw new Error('Runtime Host residency underflow'); } - if (count === 1) this.#counts.delete(label); - else this.#counts.set(label, count - 1); + counts.total -= 1; + if (kind === 'drain') { + counts.drain -= 1; + this.#drainCount -= 1; + } this.#activeCount -= 1; + if (counts.total === 0) this.#counts.delete(label); for (const waiter of this.#drainWaiters) { - if (!this.#isEmptyExcept(waiter.excludedLabel)) continue; + if (!this.#isDrainEmptyExcept(waiter.excludedLabel)) continue; this.#drainWaiters.delete(waiter); waiter.resolve(); } } - #waitForEmptyExcept(excludedLabel: string | undefined): Promise { - if (this.#isEmptyExcept(excludedLabel)) return Promise.resolve(); + #waitForDrainEmptyExcept(excludedLabel: string | undefined): Promise { + if (this.#isDrainEmptyExcept(excludedLabel)) return Promise.resolve(); return new Promise((resolve) => this.#drainWaiters.add({ excludedLabel, resolve })); } - #isEmptyExcept(excludedLabel: string | undefined): boolean { - for (const [label, count] of this.#counts) { - if (count > 0 && label !== excludedLabel) return false; + #isDrainEmptyExcept(excludedLabel: string | undefined): boolean { + for (const [label, counts] of this.#counts) { + if (counts.drain > 0 && label !== excludedLabel) return false; } return true; }