Skip to content

Commit bbd769c

Browse files
committed
fix(runtime-host): recover approved guest regeneration
Admit durable Guest authorization on regenerate roots, and let explicit managed-Host repair safely replace a Host that dies while preparing retirement. Generated-by: OpenAI Codex
1 parent b1ec289 commit bbd769c

7 files changed

Lines changed: 229 additions & 20 deletions

File tree

apps/desktop/src/main/__tests__/runtime-host-local-operator.test.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ test('local update runs the selected package against the exact managed deploymen
185185
deploymentId,
186186
},
187187
expectedHost: { hostEpoch: 'older-host', pid: 42 },
188+
allowManualUpdate: true,
188189
},
189190
(phase) => phases.push(phase),
190191
);
@@ -200,12 +201,48 @@ test('local update runs the selected package against the exact managed deploymen
200201
'--expected-root-path', '/tmp/maka/root',
201202
'--expected-root-id', 'a'.repeat(64),
202203
'--expected-deployment-id', deploymentId,
204+
'--allow-manual-update',
203205
]);
204206
assert.equal(
205207
environment?.[RUNTIME_HOST_OPERATOR_PROJECT_DIRECTORY_CONFIGURATION_REQUEST_ENV],
206208
'1',
207209
);
208210
assert.deepEqual(phases, ['staging']);
211+
212+
const developmentIntegrity = `sha512-${Buffer.alloc(64, 9).toString('base64')}`;
213+
await operator.runUpdate(
214+
{
215+
setupPackage: {
216+
kind: 'development_archive',
217+
path: '/tmp/maka-agent-development.tgz',
218+
integrity: developmentIntegrity,
219+
},
220+
target: {
221+
serviceId: 'a'.repeat(64),
222+
rootPath: '/tmp/maka/root',
223+
rootId: 'a'.repeat(64),
224+
deploymentId,
225+
},
226+
allowManualUpdate: true,
227+
allowInterruptActiveTasks: true,
228+
},
229+
() => undefined,
230+
);
231+
232+
assert.deepEqual(args, [
233+
'exec', '--yes', '--package', '/tmp/maka-agent-development.tgz', '--',
234+
'maka', 'runtime-host', 'service', 'update', '--framed',
235+
'--managed-root-id', 'a'.repeat(64),
236+
'--expected-service-id', 'a'.repeat(64),
237+
'--expected-root-path', '/tmp/maka/root',
238+
'--expected-root-id', 'a'.repeat(64),
239+
'--expected-deployment-id', deploymentId,
240+
'--allow-interrupt-active-tasks',
241+
]);
242+
assert.equal(
243+
environment?.[RUNTIME_HOST_SETUP_SOURCE_PACKAGE_INTEGRITY_ENV],
244+
developmentIntegrity,
245+
);
209246
});
210247

211248
test('local Peer Mesh join keeps invitations off argv and accepts bounded large results', async (t) => {

apps/desktop/src/main/runtime-host-local-operator.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,7 @@ export function createDesktopRuntimeHostLocalOperator(input: {
425425
? ['--expected-host-json', JSON.stringify(command.expectedHost)]
426426
: []),
427427
...managedTargetArgs(command.target),
428-
...(command.allowManualUpdate ? ['--allow-manual-update'] : []),
428+
...(command.allowManualUpdate && targetVersion ? ['--allow-manual-update'] : []),
429429
...(command.allowInterruptActiveTasks ? ['--allow-interrupt-active-tasks'] : []),
430430
],
431431
},

packages/cli/src/__tests__/runtime-host-lifecycle-transaction.test.ts

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,10 @@ import {
3131
type RuntimeHostManagedDeploymentConfig,
3232
type RuntimeHostSupervisorProvider,
3333
} from '@maka/runtime-host/operator';
34-
import type { connectExistingRuntimeHost } from '@maka/runtime-host/client';
34+
import {
35+
RuntimeHostOperationError,
36+
type connectExistingRuntimeHost,
37+
} from '@maka/runtime-host/client';
3538
import { resolveStorageRoot, tryAcquireStateRootOwner } from '@maka/storage/root-authority';
3639
import type {
3740
RuntimeHostLifecycleProvider,
@@ -533,6 +536,80 @@ test('does not consume replacement consent after the supervised Host exits', asy
533536
assert.equal(retired, false);
534537
});
535538

539+
test('requires explicit interruption authority when a supervised Host drains before diagnostics', async (t) => {
540+
const stateRoot = await mkdtemp(join(tmpdir(), 'maka-lifecycle-diagnostics-drain-'));
541+
t.after(() => rm(stateRoot, { recursive: true, force: true }));
542+
const capability = await resolveStorageRoot({ path: stateRoot, kind: 'interactive' });
543+
let owner: Awaited<ReturnType<typeof tryAcquireStateRootOwner>> =
544+
await tryAcquireStateRootOwner(capability);
545+
assert.ok(owner);
546+
t.after(async () => owner?.close());
547+
const events: string[] = [];
548+
const connectExisting = (async () => {
549+
events.push('connect');
550+
return {
551+
kind: 'connected',
552+
registration: { hostEpoch: 'host-a', pid: 42 },
553+
connection: {
554+
request: async (operation: string) => {
555+
events.push(operation);
556+
throw new RuntimeHostOperationError(
557+
'host.diagnostics.query',
558+
'host_draining',
559+
'Runtime Host is draining',
560+
);
561+
},
562+
close: async () => {
563+
events.push('close');
564+
},
565+
},
566+
} as unknown as Awaited<ReturnType<typeof connectExistingRuntimeHost>>;
567+
}) as typeof connectExistingRuntimeHost;
568+
const supervisor = {
569+
status: async () => {
570+
events.push('status');
571+
return { active: true, pid: 42 };
572+
},
573+
retire: async () => {
574+
events.push('retire');
575+
await owner?.close();
576+
owner = undefined;
577+
},
578+
};
579+
580+
assert.deepEqual(
581+
await retireRuntimeHostLifecycleOwner({
582+
rootPath: capability.canonicalPath,
583+
rootId: capability.rootId,
584+
connectExisting,
585+
expectedOwner: { hostEpoch: 'host-a', pid: 42 },
586+
supervisor,
587+
}),
588+
{ kind: 'active_tasks' },
589+
);
590+
const retired = await retireRuntimeHostLifecycleOwner({
591+
rootPath: capability.canonicalPath,
592+
rootId: capability.rootId,
593+
connectExisting,
594+
expectedOwner: { hostEpoch: 'host-a', pid: 42 },
595+
supervisor,
596+
allowInterruptActiveTasks: true,
597+
});
598+
assert.equal(retired.kind, 'retired');
599+
if (retired.kind === 'retired') await retired.owner.close();
600+
assert.deepEqual(events, [
601+
'connect',
602+
'status',
603+
'host.diagnostics.query',
604+
'close',
605+
'connect',
606+
'status',
607+
'host.diagnostics.query',
608+
'retire',
609+
'close',
610+
]);
611+
});
612+
536613
test('requires explicit interruption authority to recover an unreachable supervised transition', async (t) => {
537614
const stateRoot = await mkdtemp(join(tmpdir(), 'maka-lifecycle-recovery-consent-'));
538615
const capability = await resolveStorageRoot({ path: stateRoot, kind: 'interactive' });

packages/cli/src/runtime-host-lifecycle-transaction.ts

Lines changed: 40 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ import {
2828
import {
2929
connectExistingRuntimeHost,
3030
prepareConnectedRuntimeHostRetirement,
31+
RuntimeHostOperationError,
32+
RuntimeHostRequestInterruptedError,
3133
waitForRuntimeHostReady,
3234
} from '@maka/runtime-host/client';
3335
import { RUNTIME_HOST_PROTOCOL_VERSION } from '@maka/runtime-host/protocol';
@@ -316,6 +318,7 @@ function assertRecoveryTarget(
316318
export async function retireRuntimeHostLifecycleOwner(input: {
317319
readonly rootPath: string;
318320
readonly rootId: string;
321+
readonly connectExisting?: typeof connectExistingRuntimeHost;
319322
readonly allowInterruptActiveTasks?: boolean;
320323
/**
321324
* Freshness fence evaluated before a canonical supervised-deployment retirement is admitted.
@@ -357,7 +360,7 @@ export async function retireRuntimeHostLifecycleOwner(input: {
357360
throw error;
358361
}
359362
}
360-
const connected = await connectExistingRuntimeHost({
363+
const connected = await (input.connectExisting ?? connectExistingRuntimeHost)({
361364
rootPath: capability.canonicalPath,
362365
protocol: {
363366
min: RUNTIME_HOST_PROTOCOL_VERSION,
@@ -384,31 +387,44 @@ export async function retireRuntimeHostLifecycleOwner(input: {
384387
);
385388
}
386389
try {
387-
const diagnostics = await connected.connection.request('host.diagnostics.query', {});
388390
const supervisorStatus = await input.supervisor?.status();
389391
if (supervisorStatus) assertExpectedSupervisorOwner(input.expectedOwner, supervisorStatus);
390392
if (
391393
supervisorStatus &&
392-
(!supervisorStatus.active || supervisorStatus.pid !== diagnostics.pid)
394+
(!supervisorStatus.active || supervisorStatus.pid !== connected.registration.pid)
393395
) {
394396
throw new RuntimeHostLifecycleTransactionError(
395397
'transition_failed',
396-
'The supervisor and State Root report different Runtime Host processes',
398+
'The supervisor and Runtime Host registration report different processes',
397399
);
398400
}
399-
// The exact Root owner and canonical supervisor now agree while the deployment lock is held.
400-
// This admits retirement of that deployment; a later same-deployment restart is not a new
401-
// authority, but it must not acquire the Root before the supervisor is retired.
402-
const prepared = await prepareConnectedRuntimeHostRetirement(
403-
connected.connection,
404-
input.allowInterruptActiveTasks ? 'interrupt_active_work' : 'refuse_active_work',
405-
);
406-
if (prepared.kind === 'active_tasks') return prepared;
407-
if (prepared.pid !== diagnostics.pid) {
408-
throw new RuntimeHostLifecycleTransactionError(
409-
'transition_failed',
410-
'The Runtime Host process changed while retirement was prepared',
401+
try {
402+
const diagnostics = await connected.connection.request('host.diagnostics.query', {});
403+
if (diagnostics.pid !== connected.registration.pid) {
404+
throw new RuntimeHostLifecycleTransactionError(
405+
'transition_failed',
406+
'The Runtime Host registration and diagnostics report different processes',
407+
);
408+
}
409+
// The exact Root owner and canonical supervisor now agree while the deployment lock is held.
410+
// This admits retirement of that deployment; a later same-deployment restart is not a new
411+
// authority, but it must not acquire the Root before the supervisor is retired.
412+
const prepared = await prepareConnectedRuntimeHostRetirement(
413+
connected.connection,
414+
input.allowInterruptActiveTasks ? 'interrupt_active_work' : 'refuse_active_work',
411415
);
416+
if (prepared.kind === 'active_tasks') return prepared;
417+
if (prepared.pid !== diagnostics.pid) {
418+
throw new RuntimeHostLifecycleTransactionError(
419+
'transition_failed',
420+
'The Runtime Host process changed while retirement was prepared',
421+
);
422+
}
423+
} catch (error) {
424+
if (!isRuntimeHostRetirementUnavailable(error) || !input.supervisor) throw error;
425+
if (!input.allowInterruptActiveTasks) return { kind: 'active_tasks' };
426+
await input.supervisor.retire();
427+
return waitForRuntimeHostLifecycleOwner(capability, input.timeoutMs ?? 45_000);
412428
}
413429
const retirement = await waitForRuntimeHostLifecycleOwner(
414430
capability,
@@ -426,6 +442,14 @@ export async function retireRuntimeHostLifecycleOwner(input: {
426442
}
427443
}
428444

445+
function isRuntimeHostRetirementUnavailable(error: unknown): boolean {
446+
return (
447+
error instanceof RuntimeHostRequestInterruptedError ||
448+
(error instanceof RuntimeHostOperationError &&
449+
(error.code === 'host_not_ready' || error.code === 'host_draining'))
450+
);
451+
}
452+
429453
function assertExpectedRuntimeHostOwner(
430454
expected: { readonly hostEpoch: string; readonly pid: number } | undefined,
431455
observed: { readonly hostEpoch: string; readonly pid: number } | undefined,

packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -875,6 +875,64 @@ test('turn.start durably binds a Guest request approval to the admitted Turn', a
875875
}
876876
});
877877

878+
test('turn.regenerate durably binds a Guest request approval to the admitted Turn', async () => {
879+
const fixture = await createFailureFixture({
880+
registerBackend: (backends) =>
881+
backends.register('ai-sdk', (context) => new FakeBackend(context)),
882+
});
883+
const authorization = {
884+
kind: 'session_turn_access_request' as const,
885+
requestId: 'request-regenerate-1',
886+
principalId: 'session_guest:guest-1',
887+
grantId: 'grant-1',
888+
approvedAt: 1_788_000_000_000,
889+
approvedBy: 'local_owner',
890+
};
891+
const input = {
892+
sessionId: fixture.sessionId,
893+
sourceTurnId: 'turn-regenerate-source',
894+
turnId: 'turn-regenerate-approved',
895+
};
896+
try {
897+
assertStartedTurn(
898+
await fixture.interactiveTurns.handlers['turn.start'](
899+
{
900+
sessionId: fixture.sessionId,
901+
turnId: input.sourceTurnId,
902+
content: { text: 'Regenerate this approved request.' },
903+
},
904+
operationContext(fixture.hostEpoch, fixture.acquireResidency),
905+
),
906+
);
907+
await fixture.coordinator.whenIdle(fixture.sessionId);
908+
909+
const regenerated = await fixture.interactiveTurns.handlers['turn.regenerate'](input, {
910+
...operationContext(fixture.hostEpoch, fixture.acquireResidency),
911+
principal: authorization.principalId,
912+
turnAdmissionAuthorization: authorization,
913+
});
914+
assert.equal(regenerated.ok, true, JSON.stringify(regenerated));
915+
const admission = await fixture.stores.agentRunStore.readRootTurnAdmission(
916+
fixture.sessionId,
917+
input.turnId,
918+
);
919+
assert.deepEqual(admission?.execution, {
920+
kind: 'regenerate',
921+
sourceTurnId: input.sourceTurnId,
922+
});
923+
assert.deepEqual(admission?.authorization, authorization);
924+
925+
const conflictingRetry = await fixture.interactiveTurns.handlers['turn.regenerate'](
926+
input,
927+
operationContext(fixture.hostEpoch, fixture.acquireResidency),
928+
);
929+
assert.equal(conflictingRetry.ok, false);
930+
if (!conflictingRetry.ok) assert.equal(conflictingRetry.error.code, 'operation_conflict');
931+
} finally {
932+
await fixture.dispose();
933+
}
934+
});
935+
878936
test('turn.start resolves explicit Skills once before durable admission and replays the result', async () => {
879937
let preparationCount = 0;
880938
let blocked = false;

packages/storage/src/__tests__/regenerate-root-admission.test.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ test('regenerate admission durably binds the immutable source Turn', async () =>
3636
kind: 'regenerate',
3737
sourceTurnId: 'source-turn',
3838
});
39+
assert.deepEqual(admitted.admission.authorization, input.authorization);
3940
store.close?.();
4041

4142
const reopened = createSqliteAgentRunStore(root);
@@ -116,6 +117,14 @@ function admissionInput(overrides: Partial<AdmitRootTurnInput> = {}): AdmitRootT
116117
previousRootTurnId: null,
117118
normalizedInput: { text: 'Original request' },
118119
sourceMessages: [],
120+
authorization: {
121+
kind: 'session_turn_access_request',
122+
requestId: 'request-regenerate',
123+
principalId: 'session_guest:guest-1',
124+
grantId: 'grant-1',
125+
approvedAt: 45,
126+
approvedBy: 'local_owner',
127+
},
119128
admittedAt: 50,
120129
...overrides,
121130
};

packages/storage/src/agent-run-store.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1897,9 +1897,13 @@ function assertRootTurnAdmissionContract(admission: RootTurnAdmission): void {
18971897
'Invalid root turn admission contract: Skill invocation requires external message execution',
18981898
);
18991899
}
1900-
if (admission.authorization && execution.kind !== 'external_message') {
1900+
if (
1901+
admission.authorization &&
1902+
execution.kind !== 'external_message' &&
1903+
execution.kind !== 'regenerate'
1904+
) {
19011905
throw new Error(
1902-
'Invalid root turn admission contract: authorization proof requires external message execution',
1906+
'Invalid root turn admission contract: authorization proof requires external message or regenerate execution',
19031907
);
19041908
}
19051909
if (execution.kind === 'claimed_agent_graph_intent') {

0 commit comments

Comments
 (0)