diff --git a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts index 6e67dcd363..a490a9ca5d 100644 --- a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts +++ b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts @@ -20,10 +20,11 @@ import { openSync } from 'node:fs'; import { launchOwnedRuntimeHostCandidate } from '../../client/launcher.js'; -const [rootPath, expectedRootId, leasePath, clientInstanceId] = process.argv.slice(2); +const [rootPath, expectedRootId, leasePath, clientInstanceId, entrypointOverride] = + process.argv.slice(2); if (!rootPath || !expectedRootId || !leasePath || !clientInstanceId) { throw new Error( - 'usage: owned-authority-launcher ', + 'usage: owned-authority-launcher [entrypoint]', ); } @@ -31,8 +32,16 @@ const leaseFd = openSync(leasePath, 'a+'); const attempt = await launchOwnedRuntimeHostCandidate({ rootPath, expectedRootId, - entrypoint: new URL('../../execution-candidate-main.js', import.meta.url), - idleGraceMs: 10_000, + // Tests that bound an owner-loss exit pass a test-only entry whose + // startup window they control; the production entry stays the default. + entrypoint: new URL(entrypointOverride ?? '../../execution-candidate-main.js', import.meta.url), + // The idle grace only has to outlast the test, and it has to stay clear of + // any bound a test puts on an owner-loss exit: a Candidate that exits + // because it went idle must never be mistaken for one that exited because + // its launch owner died. The first-connection deadline stays short so a + // Candidate no Client ever reaches still exits on its own. + idleGraceMs: 60_000, + initialConnectionTimeoutMs: 10_000, inheritableAuthorityLeaseFd: leaseFd, launchOwnerClientInstanceId: clientInstanceId, }).spawned; diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index f3c007aa05..f6b5307f58 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -21,7 +21,7 @@ import { deferred, withTimeout } from '@maka/core/test-only/async-primitives'; import { RuntimeHostProtocolError } from '../protocol/errors.js'; import { defineInteractiveRuntimeHostComposition } from '../server/host-composition.js'; import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { execFile, fork, type ChildProcess } from 'node:child_process'; import { chmod, @@ -2204,11 +2204,22 @@ describe('non-serving Runtime Host kernel', () => { capability.rootId, join(paths.base, 'authority-lease-probe'), launchOwnerClientInstanceId, + // The owner-loss exit bound below covers the gated recovery + // window, so this run pins startup behind the gated-recovery + // entry instead of leaving both the window and the kill's + // ordering to however scheduling resolves them. + '../../test-only/owned-candidate-gated-recovery-main.js', ], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, ), ); const launchedPid = paths.resources.trackPid(await waitForLaunch(launcher)); + // The gated-recovery entry parks composition creation behind these + // markers under the same base directory; the stall marker proves the + // Candidate reached the gated window, and the release marker lets the + // test unblock it through a channel that survives the launcher. + const stallMarker = join(paths.base, 'authority-lease-probe.stalled'); + const releaseMarker = join(paths.base, 'authority-lease-probe.release'); const connected = await retryConnect(paths, CURRENT_PROTOCOL, { clientInstanceId: launchOwnerClientInstanceId, }); @@ -2224,14 +2235,67 @@ describe('non-serving Runtime Host kernel', () => { }); assert.equal(ordinary.kind, 'draining'); + // The owner-loss contract under test is recorded pre-bind: a + // launch-owner Client is admitted while the Host is still recovering, + // and the guard that closes the Host on owner loss binds only after + // startup returns. Kill timing alone cannot prove the loss was + // recorded pre-bind — a test-side pause longer than the candidate's + // startup would silently turn this into the post-bind scenario — so + // the gated-recovery entry holds startup behind a release file and + // marks the stall; waiting for that marker makes the kill land inside + // the gated window by construction rather than by luck. + const stallDeadline = Date.now() + 10_000; + while (!existsSync(stallMarker) && Date.now() < stallDeadline) { + await sleep(20); + } + assert.ok(existsSync(stallMarker), 'gated-recovery entry never reached its stall window'); launcher.kill('SIGKILL'); await waitForExit(launcher); + // The process is the only thing that reports the claim. A Client's + // `connection.closed` does not: it is that Client's own transport, and + // the Client aborts it after its liveness probe goes unanswered for two + // seconds. A Host that is merely busy therefore resolves it while still + // running, so it is used only as a post-exit consistency check below. + // + // Startup — composition creation and recovery included — runs after the + // release and is not bounded by the kernel's shutdown grace, so the exit + // budget must not start at the release. The entry's `onWon` marker is + // the explicit guard-bound boundary that starts it instead: the + // launch-owner guard has bound and the pre-bind recorded loss is being + // acted on, so everything the 20-second deadline covers (the + // `shutdownGraceMs` close plus margin — which sits below the launcher's + // 60 s idle grace, so it cannot be satisfied by a Candidate that merely + // went idle) happens after the marker. + // + // The race below keeps that boundary honest without breaking local + // Windows runs: there the Candidate can be terminated abruptly the + // moment its launcher dies — no JS exit event, so no bind and no marker + // — and `isProcessAlive` releasing the wait only records that platform + // limitation, while a Candidate still alive without a marker past the + // deadline is a failure. The assertion observes the real + // operating-system PID: the kernel resolving its `closed` promise does + // not by itself mean the OS process has exited, so only CI verdicts + // count as cross-platform evidence here. + writeFileSync(releaseMarker, String(Date.now())); + const boundMarker = join(paths.base, 'authority-lease-probe.bound'); + const boundDeadline = Date.now() + 10_000; + while ( + !existsSync(boundMarker) && + isProcessAlive(launchedPid) && + Date.now() < boundDeadline + ) { + await sleep(20); + } + assert.ok( + existsSync(boundMarker) || !isProcessAlive(launchedPid), + 'gated-recovery entry never reached its guard bind', + ); + await waitForProcessExit(launchedPid, 20_000); await withTimeout( connected.connection.closed, 5_000, - 'authority-supervised Candidate survived its launch owner', + 'authority-supervised Candidate exited without closing its Client connection', ); - await waitForProcessExit(launchedPid); paths.resources.forgetPid(launchedPid); }); }); diff --git a/packages/runtime-host/src/__tests__/owned-candidate.test.ts b/packages/runtime-host/src/__tests__/owned-candidate.test.ts index 3c264f145d..bc25e6a05e 100644 --- a/packages/runtime-host/src/__tests__/owned-candidate.test.ts +++ b/packages/runtime-host/src/__tests__/owned-candidate.test.ts @@ -37,6 +37,11 @@ import { type CandidateExitDetails, type OwnedCandidateAttempt, } from '../client/launcher.js'; +import { + resolveExistingStorageRoot, + resolveExistingStorageRootControlDirectory, +} from '@maka/storage/root-authority'; +import { readHostRegistration } from '../control/registration.js'; test('owned connection keeps a fresh Host alive for its full election window', async () => { const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-first-connection-')); @@ -257,12 +262,20 @@ test('owned Host exits promptly after its first connection closes', async () => assert.equal(result.kind, 'connected', connectFailure(result)); if (result.kind !== 'connected') return; + const controlDirectory = await resolveHostControlDirectory(rootPath, result.connection.rootId); await result.connection.close(); - // Prompt means the owned launch's idleGraceMs of 0, as opposed to the 30 s - // default grace, so the bound only has to sit well below that. Shutdown takes - // about 30 ms on an idle machine and stretches past 500 ms under a full CI - // suite while still exiting cleanly: the Host is starved, not stuck. - assert.equal(await result.host.settle(5_000), true); + // Promptness is when the Host starts shutting down, not how long shutting + // down takes: the owned launch's idleGraceMs is 0 against a 30 s default. + // The kernel publishes its draining registration as the first step of + // shutdown, so the registration reports the idle grace directly. Reading it + // from `settle` alone could not separate the two, which is why a loaded + // machine that only made the shutdown itself slow failed this assertion. + await waitForHostShutdownStart(controlDirectory, 10_000); + // The exit is a second claim with a bound of its own, and the kernel sets + // it: shutdown gets `shutdownGraceMs` (10 s) to close every resource before + // the kernel force-terminates the process. Anything below that fails a Host + // that is starved rather than stuck. + assert.equal(await result.host.settle(15_000), true); }); test('an exited owned Candidate permits one real successor in the same election', { @@ -436,6 +449,38 @@ test('pre-cancelled hosted execution does not start a Runtime Host', async () => assert.deepEqual(await readdir(rootPath), []); }); +async function resolveHostControlDirectory(rootPath: string, rootId: string): Promise { + const capability = await resolveExistingStorageRoot({ + path: rootPath, + kind: 'interactive', + expectedRootId: rootId, + }); + const { controlDirectory } = await resolveExistingStorageRootControlDirectory(capability); + return controlDirectory; +} + +/** + * Resolves once the Host has begun shutting down. `draining` is the state the + * kernel publishes before it does any shutdown work, and the registration is + * removed near the end of that work, so either observation proves shutdown + * started; the Host was serving this Client, so its registration existed. + */ +async function waitForHostShutdownStart( + controlDirectory: string, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + // readHostRegistration already maps a missing file to `undefined`; the + // unguarded await lets real I/O or decode errors fail this wait loudly + // instead of masquerading as "shutdown started". + const registration = await readHostRegistration(controlDirectory); + if (!registration || registration.state === 'draining') return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error('owned Host did not begin shutting down after its first connection closed'); +} + function connectFailure( result: | Awaited> diff --git a/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts b/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts new file mode 100644 index 0000000000..6afcbf9fc4 --- /dev/null +++ b/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts @@ -0,0 +1,67 @@ +#!/usr/bin/env node +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Test-only Candidate entry for the owner-loss lifecycle tests. A + * launch-owner Client is admitted while the Host is still recovering, and + * the guard that closes the Host on owner loss binds only after startup + * returns — so whether the test's kill lands before or after the bind is a + * scheduling race a fixed sleep cannot decide. This entry instead gates + * composition creation behind a release file: writing the stall marker + * proves the Candidate is parked before the bind, the test kills the + * launcher at that point, and releasing the gate afterwards lets startup + * return promptly so the recorded loss closes the Host under the kernel's + * `shutdownGraceMs`. The run still goes through the real Runtime Host + * composition — only its start is held at the gate. The `onWon` hook adds + * the second boundary the test needs: a marker written right after the + * guard binds, so the exit budget starts at the bind rather than at the + * release, which only unblocks startup. + */ +import { existsSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { runExecutionCandidateEntry } from '../candidate-entry.js'; +import { createExecutionRuntimeHostComposition } from '../server/execution-composition.js'; + +const rootArgumentIndex = process.argv.indexOf('--root') + 1; +const rootPath = process.argv[rootArgumentIndex]; +if (!rootPath) throw new Error('gated-recovery entry requires --root'); +const stallMarker = join(dirname(rootPath), 'authority-lease-probe.stalled'); +const releaseMarker = join(dirname(rootPath), 'authority-lease-probe.release'); +const boundMarker = join(dirname(rootPath), 'authority-lease-probe.bound'); + +await runExecutionCandidateEntry(process.argv.slice(2), import.meta.url, { + // `onWon` fires right after the launch-owner guard binds (candidate-entry + // binds before invoking it), so this marker is the test's explicit + // guard-bound boundary: the pre-bind recorded loss starts acting only past + // it, which is where the exit budget under test actually begins. + onWon: () => { + writeFileSync(boundMarker, String(Date.now())); + return () => undefined; + }, + dependencies: { + createComposition: async (context, compositionOptions) => { + writeFileSync(stallMarker, String(Date.now())); + while (!existsSync(releaseMarker)) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return createExecutionRuntimeHostComposition(context, compositionOptions); + }, + }, +});