From ce45c1ea9db700993c5def838686fbd743938f5e Mon Sep 17 00:00:00 2001 From: Christopher Nelson Date: Sat, 5 Sep 2026 23:54:01 -0400 Subject: [PATCH] feat: enforce experiment provider attempt budgets --- README.md | 2 +- apps/game-api/src/app.test.ts | 64 ++++++ apps/game-api/src/attempt-accounting.test.ts | 76 +++++++ apps/game-api/src/attempt-accounting.ts | 91 ++++++++ apps/game-api/src/experiment-export.ts | 2 +- apps/game-api/src/simulation-service.test.ts | 195 +++++++++++++++++- apps/game-api/src/simulation-service.ts | 132 +++++++++++- apps/world-lab/src/app/styles.css | 3 +- .../src/components/world-lab.test.tsx | 93 +++++++++ apps/world-lab/src/components/world-lab.tsx | 105 +++++++++- docs/ARCHITECTURE.md | 5 + docs/SECURITY.md | 4 +- docs/TESTING.md | 5 + ...5-server-owned-provider-attempt-budgets.md | 34 +++ .../agent-runtime/src/tick-dispatcher.test.ts | 66 ++++++ packages/agent-runtime/src/tick-dispatcher.ts | 20 ++ packages/shared/src/index.test.ts | 6 + packages/shared/src/index.ts | 89 ++++++++ packages/shared/src/limits.ts | 1 + packages/shared/src/scenario.test.ts | 36 ++++ packages/world-engine/src/index.ts | 5 + 21 files changed, 1012 insertions(+), 22 deletions(-) create mode 100644 apps/game-api/src/attempt-accounting.test.ts create mode 100644 apps/game-api/src/attempt-accounting.ts create mode 100644 docs/adr/0025-server-owned-provider-attempt-budgets.md diff --git a/README.md b/README.md index cea61e4..515b002 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ repair or transient-retry charge per agent. All agents observe the same frozen pre-tick world; valid decisions resolve together while an individual provider failure is retained as that agent's final lost tick. Start is deliberately disabled when the server has no key. This development API has no -authentication or cost controls and is not suitable for an unauthenticated +authentication or monetary cost controls and is not suitable for an unauthenticated public deployment. State is held only in the Game API process. The API captures one active safe diff --git a/apps/game-api/src/app.test.ts b/apps/game-api/src/app.test.ts index 6babde2..8e58840 100644 --- a/apps/game-api/src/app.test.ts +++ b/apps/game-api/src/app.test.ts @@ -73,6 +73,70 @@ describe('provider environment compatibility', () => { }); describe('game API simulation boundary', () => { + it('returns a specific 409 before dispatch when a full tick cannot be reserved', async () => { + let calls = 0; + const app = createApp({ + provider: { + mode: 'scripted-test', + model: 'deterministic-script', + configured: true, + async decide(): Promise { + calls += 1; + return { + decision: { worldAction: { type: 'wait' }, summary: 'Wait.' }, + metadata: { + provider: 'scripted-test', + model: 'deterministic-script', + latencyMs: 0, + costCredits: 0, + }, + }; + }, + }, + }); + const defaults = defaultWorldSetupResponseSchema.parse( + await ( + await app.request('/api/simulation/experiment/setup/default') + ).json(), + ).request; + await app.request('/api/simulation/experiment/setup', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + ...defaults, + modelConfiguration: { + ...defaults.modelConfiguration, + globalModelId: 'deterministic-script', + }, + executionLimits: { + version: 'execution-limits-v1', + providerAttemptLimit: 1, + }, + }), + }); + const response = await app.request('/api/simulation/tick', { + method: 'POST', + }); + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: { + code: 'experiment_budget_exhausted', + message: + 'The experiment does not have enough provider attempts remaining for a complete tick.', + }, + }); + expect(calls).toBe(0); + expect( + simulationSnapshotSchema.parse( + await (await app.request('/api/simulation')).json(), + ), + ).toMatchObject({ + tickNumber: 0, + status: 'budget-exhausted', + experiment: { attemptAccounting: { attemptsStarted: 0 } }, + }); + }); + it('requires a known Patient Zero through the public setup boundary', async () => { const app = createApp({ provider: new ScriptedAgentProvider([ diff --git a/apps/game-api/src/attempt-accounting.test.ts b/apps/game-api/src/attempt-accounting.test.ts new file mode 100644 index 0000000..fb32509 --- /dev/null +++ b/apps/game-api/src/attempt-accounting.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { AttemptAccounting } from './attempt-accounting'; + +describe('AttemptAccounting', () => { + it('reserves atomically and finalizes known and unknown cost exactly once', () => { + const accounting = new AttemptAccounting(3); + expect(accounting.reserve(3)).toBe(true); + expect(accounting.reserve(1)).toBe(false); + const first = accounting.startReserved()!; + const second = accounting.startReserved()!; + const third = accounting.startReserved()!; + accounting.finalize(first, { + provider: 'scripted-test', + model: 'deterministic-script' as never, + latencyMs: 0, + costCredits: 0.1, + }); + accounting.finalize(first); + accounting.finalize(second, { + provider: 'scripted-test', + model: 'deterministic-script' as never, + latencyMs: 0, + costCredits: 0.2, + }); + accounting.finalize(third); + expect(accounting.snapshot()).toEqual({ + providerAttemptLimit: 3, + reservedPermits: 0, + attemptsStarted: 3, + attemptsFinalized: 3, + attemptsInFlight: 0, + remainingAttempts: 0, + knownCostCredits: 0.3, + attemptsWithUnknownCost: 1, + exhausted: true, + exhaustionReason: 'provider-attempt-limit', + }); + }); + + it('releases unused reservations without charging them', () => { + const accounting = new AttemptAccounting(3); + expect(accounting.reserve(3)).toBe(true); + const permit = accounting.startReserved()!; + accounting.finalize(permit); + accounting.releaseReservations(); + expect(accounting.snapshot()).toMatchObject({ + attemptsStarted: 1, + attemptsFinalized: 1, + remainingAttempts: 2, + attemptsWithUnknownCost: 1, + exhausted: false, + }); + }); + + it('reports explicit unlimited capacity', () => { + const accounting = new AttemptAccounting(null); + expect(accounting.startAdditional()).not.toBeNull(); + expect(accounting.snapshot()).toMatchObject({ + providerAttemptLimit: null, + remainingAttempts: null, + exhausted: false, + exhaustionReason: null, + }); + }); + + it.each([0, -1, 1.5, Number.NaN])( + 'rejects invalid internal reservation count %s', + (count) => { + const accounting = new AttemptAccounting(10); + expect(() => accounting.reserve(count)).toThrow( + 'Attempt reservation count must be a positive integer.', + ); + expect(accounting.snapshot().attemptsStarted).toBe(0); + }, + ); +}); diff --git a/apps/game-api/src/attempt-accounting.ts b/apps/game-api/src/attempt-accounting.ts new file mode 100644 index 0000000..a7b2308 --- /dev/null +++ b/apps/game-api/src/attempt-accounting.ts @@ -0,0 +1,91 @@ +import type { ProviderMetadata } from '@hexzero/shared'; +import { addDecimalValue } from './experiment-export'; + +export interface AttemptAccountingSnapshot { + providerAttemptLimit: number | null; + reservedPermits: number; + attemptsStarted: number; + attemptsFinalized: number; + attemptsInFlight: number; + remainingAttempts: number | null; + knownCostCredits: number; + attemptsWithUnknownCost: number; + exhausted: boolean; + exhaustionReason: 'provider-attempt-limit' | null; +} + +/** Billing accounting is independent of transactional world/tick state. */ +export class AttemptAccounting { + #reserved = 0; + #started = 0; + #finalized = 0; + #knownCost = '0'; + #unknownCost = 0; + #nextPermitId = 1; + readonly #inFlight = new Set(); + #exhausted = false; + + constructor(readonly limit: number | null) {} + + reserve(count: number): boolean { + if (!Number.isInteger(count) || count < 1) + throw new Error('Attempt reservation count must be a positive integer.'); + if ( + this.limit !== null && + this.#started + this.#reserved + count > this.limit + ) { + this.#exhausted = true; + return false; + } + this.#reserved += count; + return true; + } + + startReserved(): number | null { + if (this.#reserved < 1) return null; + this.#reserved -= 1; + this.#started += 1; + const permitId = this.#nextPermitId++; + this.#inFlight.add(permitId); + return permitId; + } + + startAdditional(): number | null { + if (!this.reserve(1)) return null; + return this.startReserved(); + } + + releaseReservations(): void { + this.#reserved = 0; + } + + finalize(permitId: number, metadata?: ProviderMetadata): void { + if (!this.#inFlight.delete(permitId)) return; + this.#finalized += 1; + if (metadata?.costCredits === undefined) this.#unknownCost += 1; + else + this.#knownCost = addDecimalValue(this.#knownCost, metadata.costCredits); + } + + snapshot(): AttemptAccountingSnapshot { + const remaining = + this.limit === null + ? null + : Math.max(0, this.limit - this.#started - this.#reserved); + const exhausted = + this.limit !== null && + (this.#exhausted || (remaining === 0 && this.#reserved === 0)); + return { + providerAttemptLimit: this.limit, + reservedPermits: this.#reserved, + attemptsStarted: this.#started, + attemptsFinalized: this.#finalized, + attemptsInFlight: this.#inFlight.size, + remainingAttempts: remaining, + knownCostCredits: Number(this.#knownCost), + attemptsWithUnknownCost: this.#unknownCost, + exhausted, + exhaustionReason: exhausted ? 'provider-attempt-limit' : null, + }; + } +} diff --git a/apps/game-api/src/experiment-export.ts b/apps/game-api/src/experiment-export.ts index d3cac68..75e8a8d 100644 --- a/apps/game-api/src/experiment-export.ts +++ b/apps/game-api/src/experiment-export.ts @@ -2134,7 +2134,7 @@ export function serializeExperimentExport( : JSON.stringify(document); } -function addDecimalValue(left: string, right: number): string { +export function addDecimalValue(left: string, right: number): string { const leftParts = decimalParts(left); const rightParts = decimalParts(right); const scale = Math.max(leftParts.scale, rightParts.scale); diff --git a/apps/game-api/src/simulation-service.test.ts b/apps/game-api/src/simulation-service.test.ts index 359b832..bb0fdfd 100644 --- a/apps/game-api/src/simulation-service.test.ts +++ b/apps/game-api/src/simulation-service.test.ts @@ -1225,14 +1225,26 @@ describe('SimulationService', () => { }, }); const setup = defaultWorldSetupRequest(); + const roster = generateDeterministicRoster(12, 'cancel-roster'); simulation.applyWorldSetup({ ...setup, + roster, + patientZeroAgentId: roster[0]!.id, objectiveVersion: 'durable-influence-v3', modelConfiguration: { ...setup.modelConfiguration, globalModelId: 'deterministic-script', }, capabilities: { ...setup.capabilities, simulatedPlayerPressure: true }, + behaviorConfiguration: { + ...setup.behaviorConfiguration, + seed: 'cancel-behavior', + assignments: assignBehavior( + roster.map(({ id }) => id), + 'cancel-behavior', + 'balanced-random', + ), + }, simulatedPlayer: { enabled: true, profile: 'casual-cleaner', @@ -1252,6 +1264,59 @@ describe('SimulationService', () => { world: before.world, agentGoals: before.agentGoals, agentMemories: before.agentMemories, + experiment: { + attemptAccounting: { + reservedPermits: 0, + attemptsInFlight: 0, + attemptsStarted: expect.any(Number), + attemptsWithUnknownCost: expect.any(Number), + }, + }, + }); + const accounting = simulation.getSnapshot().experiment.attemptAccounting; + expect(accounting.attemptsStarted).toBeGreaterThan(0); + expect(accounting.attemptsFinalized).toBe(accounting.attemptsStarted); + expect(accounting.attemptsWithUnknownCost).toBe(accounting.attemptsStarted); + expect(accounting.attemptsStarted).toBe(8); + }); + + it('retains safe provider cost when legacy cancellation is observed after the response', async () => { + const simulation = service({ + mode: 'scripted-test', + model: 'deterministic-script', + configured: true, + async decide(): Promise { + simulation.cancelCurrentRequest(); + return { + decision: { + worldAction: { type: 'wait' }, + goalRevision: { operation: 'keep' }, + memoryOperation: { operation: 'keep' }, + summary: 'The response completed as cancellation arrived.', + }, + metadata: { + provider: 'scripted-test', + model: 'deterministic-script', + latencyMs: 2, + costCredits: 0.125, + }, + }; + }, + }); + await expect(simulation.executeNextTurn()).rejects.toBeInstanceOf( + SimulationTurnCancelledError, + ); + expect(simulation.getSnapshot()).toMatchObject({ + turnNumber: 0, + experiment: { + attemptAccounting: { + attemptsStarted: 1, + attemptsFinalized: 1, + attemptsInFlight: 0, + knownCostCredits: 0.125, + attemptsWithUnknownCost: 0, + }, + }, }); }); @@ -4065,6 +4130,15 @@ describe('SimulationService', () => { await simulation.executeNextTurn(); await simulation.retryFailedTurn(); await simulation.retryFailedTurn(); + const beforeSkipAccounting = + simulation.getSnapshot().experiment.attemptAccounting; + expect(beforeSkipAccounting).toMatchObject({ + attemptsStarted: 4, + attemptsFinalized: 4, + attemptsInFlight: 0, + knownCostCredits: 0.04, + attemptsWithUnknownCost: 0, + }); const skipped = simulation.skipFailedTurn(); expect(skipped).toMatchObject({ turnNumber: 1, @@ -4084,6 +4158,9 @@ describe('SimulationService', () => { status: 'paused', pendingFailedTurn: null, }); + expect(simulation.getSnapshot().experiment.attemptAccounting).toEqual( + beforeSkipAccounting, + ); const exported = simulation.generateExperimentExport({ ...exportRequest('minimal'), outcomes: ['operator-skipped'], @@ -4195,7 +4272,11 @@ describe('SimulationService', () => { metadata: { provider: 'scripted-test', model: '', - latencyMs: 0, + latencyMs: 4, + promptTokens: 3, + completionTokens: 2, + totalTokens: 5, + costCredits: 0.125, }, } as ProviderDecision; } @@ -4225,7 +4306,29 @@ describe('SimulationService', () => { expect(afterFailure).toMatchObject({ activeAgentId: null, status: 'provider-error', - pendingFailedTurn: { turnNumber: 1 }, + pendingFailedTurn: { + turnNumber: 1, + attempts: [ + { + provider: { + model: 'invalid-metadata-test', + latencyMs: 4, + promptTokens: 3, + completionTokens: 2, + totalTokens: 5, + costCredits: 0.125, + }, + }, + ], + }, + experiment: { + attemptAccounting: { + attemptsStarted: 1, + attemptsFinalized: 1, + knownCostCredits: 0.125, + attemptsWithUnknownCost: 0, + }, + }, }); const recovered = await simulation.retryFailedTurn(); @@ -4566,4 +4669,92 @@ describe('SimulationService', () => { expect(legacy.legacy).toBe(true); expect(legacy.snapshot.modelConfiguration.globalModelId).toBeNull(); }); + + it('admits a complete roster atomically and stops at the provider-attempt cap', async () => { + let calls = 0; + const provider: AgentProvider = { + mode: 'scripted-test', + model: 'deterministic-script', + configured: true, + async decide() { + calls += 1; + return { + decision: { + worldAction: { type: 'wait' }, + goalRevision: { operation: 'keep' }, + memoryOperation: { operation: 'keep' }, + summary: 'Wait within the bounded experiment.', + }, + metadata: { + provider: 'scripted-test', + model: 'deterministic-script', + latencyMs: 0, + costCredits: 0, + }, + }; + }, + }; + const simulation = service(provider); + const setup = defaultWorldSetupRequest(); + simulation.applyWorldSetup({ + ...setup, + modelConfiguration: { + ...setup.modelConfiguration, + globalModelId: 'deterministic-script', + }, + executionLimits: { + version: 'execution-limits-v1', + providerAttemptLimit: setup.roster.length, + }, + }); + await simulation.executeNextTick(); + const afterFirst = simulation.getSnapshot(); + expect(afterFirst.experiment.attemptAccounting).toMatchObject({ + attemptsStarted: setup.roster.length, + attemptsFinalized: setup.roster.length, + attemptsInFlight: 0, + knownCostCredits: 0, + attemptsWithUnknownCost: 0, + remainingAttempts: 0, + exhausted: true, + }); + simulation.updateAgentPersonality( + afterFirst.world.agents[0]!.id, + 'Preserve the attempt ledger while changing this personality.', + ); + expect( + simulation.getSnapshot().experiment.attemptAccounting.attemptsStarted, + ).toBe(setup.roster.length); + const world = structuredClone(afterFirst.world); + await expect(simulation.executeNextTick()).rejects.toMatchObject({ + code: 'experiment_budget_exhausted', + }); + expect(calls).toBe(setup.roster.length); + expect(simulation.getSnapshot().world.hexes).toEqual(world.hexes); + expect(simulation.reset().experiment.attemptAccounting).toMatchObject({ + attemptsStarted: 0, + attemptsFinalized: 0, + knownCostCredits: 0, + attemptsWithUnknownCost: 0, + exhausted: false, + }); + await simulation.executeNextTick(); + expect( + simulation.getSnapshot().experiment.attemptAccounting.attemptsStarted, + ).toBeGreaterThan(0); + const applied = simulation.applyWorldSetup({ + ...setup, + modelConfiguration: { + ...setup.modelConfiguration, + globalModelId: 'deterministic-script', + }, + }); + expect(applied.experiment.attemptAccounting).toMatchObject({ + attemptsStarted: 0, + attemptsFinalized: 0, + knownCostCredits: 0, + attemptsWithUnknownCost: 0, + exhausted: false, + }); + }); }); diff --git a/apps/game-api/src/simulation-service.ts b/apps/game-api/src/simulation-service.ts index 09be8e8..5a2e4d3 100644 --- a/apps/game-api/src/simulation-service.ts +++ b/apps/game-api/src/simulation-service.ts @@ -34,6 +34,7 @@ import { PATIENT_ZERO_PRESSURE_WINDOW_TICKS, MEMORY_ENTRY_LIMIT, personalitySchema, + providerMetadataSchema, simulationSnapshotSchema, type Agent, type AgentId, @@ -100,6 +101,7 @@ import { } from './experiment-export'; import { geographicDirectionBetweenCells } from './geographic-direction'; import { ObservationHistory } from './observation-history'; +import { AttemptAccounting } from './attempt-accounting'; const RESET_GENERATED_AT = '2026-08-13T12:00:00.000Z'; const MAX_TURN_HISTORY = 120; @@ -219,6 +221,7 @@ export type SimulationValidationCode = | 'invalid_personality' | 'invalid_model_configuration' | 'models_unavailable' + | 'experiment_budget_exhausted' | 'invalid_behavior_configuration'; export class SimulationValidationError extends Error { @@ -280,6 +283,7 @@ export class SimulationService { #agentMemories = new Map(); #simulatedPlayerEvents: SimulatedPlayerEvent[] = []; #observationHistory: ObservationHistory; + #attemptAccounting: AttemptAccounting; constructor({ provider, @@ -344,6 +348,9 @@ export class SimulationService { modelConfiguration: structuredClone(this.#modelConfiguration), behaviorConfiguration: structuredClone(this.#behaviorConfiguration), }; + this.#attemptAccounting = new AttemptAccounting( + this.#scenario.executionLimits.providerAttemptLimit, + ); } getSnapshot(): SimulationSnapshot { @@ -395,6 +402,7 @@ export class SimulationService { lastRetainedTurn: this.#experimentTurns.at(-1)?.turnNumber, droppedRecords, complete: droppedRecords === 0, + attemptAccounting: this.#attemptAccounting.snapshot(), metrics: this.#experimentMetrics.snapshot(agents.map(({ id }) => id)), currentTerritory: this.#territoryScoreboard(), currentAlliances: this.#allianceTerritorySummaries(), @@ -443,6 +451,9 @@ export class SimulationService { this.#experimentMetrics = new ExperimentMetricAccumulator([ ...this.#state.agents.keys(), ]); + this.#attemptAccounting = new AttemptAccounting( + this.#scenario.executionLimits.providerAttemptLimit, + ); this.#modelConfiguration = { ...structuredClone(this.#scenario.modelConfiguration), locked: false, @@ -584,6 +595,9 @@ export class SimulationService { this.#experimentMetrics = new ExperimentMetricAccumulator([ ...this.#state.agents.keys(), ]); + this.#attemptAccounting = new AttemptAccounting( + this.#scenario.executionLimits.providerAttemptLimit, + ); this.#status = this.#provider.configured ? 'paused' : 'configuration-error'; return this.getSnapshot(); } @@ -1036,6 +1050,13 @@ export class SimulationService { 'models_unavailable', 'Every agent requires an available compatible model before the experiment can run.', ); + if (!this.#attemptAccounting.reserve(agents.length)) { + this.#status = 'budget-exhausted'; + throw new SimulationValidationError( + 'experiment_budget_exhausted', + 'The experiment does not have enough provider attempts remaining for a complete tick.', + ); + } const tickNumber = this.#completedTickCount + 1; const preTickState = this.#state; @@ -1071,6 +1092,9 @@ export class SimulationService { structuredClone(this.#buildObservation(id, playerAdvance.events)), ]), ); + } catch (error) { + this.#attemptAccounting.releaseReservations(); + throw error; } finally { this.#state = preTickState; } @@ -1098,6 +1122,16 @@ export class SimulationService { deadlineAtMs, signal: controller.signal, now: this.#now, + beginAttempt: (_job, kind) => { + const permitId = + kind === 'initial' + ? this.#attemptAccounting.startReserved() + : this.#attemptAccounting.startAdditional(); + return permitId === null + ? null + : (metadata) => + this.#attemptAccounting.finalize(permitId, metadata); + }, }, ); if (controller.signal.aborted) throw new SimulationTurnCancelledError(); @@ -1277,7 +1311,9 @@ export class SimulationService { playerAdvance.events, committedObservationEvents, ); - this.#status = 'paused'; + this.#status = this.#attemptAccounting.snapshot().exhausted + ? 'budget-exhausted' + : 'paused'; return records; } catch (error) { if ( @@ -1293,11 +1329,17 @@ export class SimulationService { } throw error; } finally { + this.#attemptAccounting.releaseReservations(); this.#busy = false; this.#activeRequestController = null; this.#activeAgentId = null; this.#cancellationRequested = false; - if (this.#status === 'waiting-for-model') this.#status = 'paused'; + if (this.#status === 'waiting-for-model') + this.#status = this.#attemptAccounting.snapshot().exhausted + ? 'budget-exhausted' + : 'paused'; + if (this.#attemptAccounting.snapshot().exhausted) + this.#status = 'budget-exhausted'; } } @@ -1393,7 +1435,9 @@ export class SimulationService { undefined, [], ); - this.#status = 'paused'; + this.#status = this.#attemptAccounting.snapshot().exhausted + ? 'budget-exhausted' + : 'paused'; return record; } @@ -1431,9 +1475,11 @@ export class SimulationService { const turnNumber = pending?.turnNumber ?? this.#completedTurnCount + 1; const attemptStartedAt = this.#now(); let successfulAttemptStartedAt = attemptStartedAt; + let successfulAttemptKind: ModelAttempt['kind'] = attemptKind; const resolvedModel = this.#resolvedModel(agent.id); const selectedModel = resolvedModel.modelId!; let providerResult: ProviderDecision | undefined; + let successfulProviderMetadata: ProviderMetadata | undefined; const attemptHistory = [...(pending?.attempts ?? [])]; const deadlineAtMs = Date.now() + OPENROUTER_PROVIDER_TIMEOUT_MS; @@ -1445,8 +1491,18 @@ export class SimulationService { let validationFeedback: ProviderFailure['validationCodes'] = pending?.failure.validationCodes; for (let automaticCall = 0; automaticCall < 2; automaticCall += 1) { + const accountingPermitId = this.#attemptAccounting.startAdditional(); + if (accountingPermitId === null) { + this.#status = 'budget-exhausted'; + throw new SimulationValidationError( + 'experiment_budget_exhausted', + 'The experiment provider-attempt limit was exhausted.', + ); + } const currentAttemptStartedAt = this.#now(); successfulAttemptStartedAt = currentAttemptStartedAt; + successfulAttemptKind = nextKind; + let accountingMetadata: ProviderMetadata | undefined; try { providerResult = await this.#provider.decide( providerObservation, @@ -1458,6 +1514,12 @@ export class SimulationService { validationFeedback, }, ); + successfulProviderMetadata = safeRecoveryProviderMetadata( + providerResult.metadata, + this.#provider.mode, + selectedModel, + ); + accountingMetadata = successfulProviderMetadata; if (this.#activeRequestController.signal.aborted) throw new AgentProviderError({ code: 'cancelled', @@ -1468,6 +1530,7 @@ export class SimulationService { break; } catch (error) { const providerError = asProviderError(error); + accountingMetadata = providerError.metadata ?? accountingMetadata; if (providerError.failure.code === 'cancelled') { this.#status = 'paused'; throw new SimulationTurnCancelledError(); @@ -1562,6 +1625,11 @@ export class SimulationService { ? 'configuration-error' : 'provider-error'; return record; + } finally { + this.#attemptAccounting.finalize( + accountingPermitId, + accountingMetadata, + ); } } @@ -1697,7 +1765,9 @@ export class SimulationService { }, committedObservationEvents, ); - this.#status = 'paused'; + this.#status = this.#attemptAccounting.snapshot().exhausted + ? 'budget-exhausted' + : 'paused'; return record; } catch (error) { if ( @@ -1714,13 +1784,13 @@ export class SimulationService { }; const attempt = { attemptNumber: attemptHistory.length + 1, - kind: attemptKind, - startedAt: attemptStartedAt, + kind: successfulAttemptKind, + startedAt: successfulAttemptStartedAt, completedAt: this.#now(), modelId: selectedModel, reasoningProfile: resolvedModel.reasoningProfile, failure, - provider: { + provider: successfulProviderMetadata ?? { provider: this.#provider.mode, model: selectedModel, latencyMs: 0, @@ -1735,7 +1805,9 @@ export class SimulationService { failure, attempts, }; - this.#status = 'provider-error'; + this.#status = this.#attemptAccounting.snapshot().exhausted + ? 'budget-exhausted' + : 'provider-error'; return agentTurnRecordSchema.parse({ turnNumber, agentId: agent.id, @@ -1759,6 +1831,11 @@ export class SimulationService { ? 'paused' : 'configuration-error'; } + if ( + this.#status !== 'configuration-error' && + this.#attemptAccounting.snapshot().exhausted + ) + this.#status = 'budget-exhausted'; } } @@ -2986,3 +3063,42 @@ export function applyMemoryOperation( }, }; } + +function safeRecoveryProviderMetadata( + value: unknown, + provider: ProviderMetadata['provider'], + selectedModel: ModelId, +): ProviderMetadata { + const raw = value && typeof value === 'object' ? value : {}; + let safe = providerMetadataSchema.parse({ + provider, + model: selectedModel, + latencyMs: 0, + }); + const fields = [ + 'model', + 'selectedModel', + 'resolvedModel', + 'requestId', + 'httpStatus', + 'finishReason', + 'nativeFinishReason', + 'latencyMs', + 'promptTokens', + 'completionTokens', + 'totalTokens', + 'reasoningTokens', + 'cachedReadTokens', + 'cacheWriteTokens', + 'costCredits', + ] as const; + for (const field of fields) { + if (!(field in raw)) continue; + const parsed = providerMetadataSchema.safeParse({ + ...safe, + [field]: (raw as Record)[field], + }); + if (parsed.success) safe = parsed.data; + } + return safe; +} diff --git a/apps/world-lab/src/app/styles.css b/apps/world-lab/src/app/styles.css index 31c370b..97d9cc0 100644 --- a/apps/world-lab/src/app/styles.css +++ b/apps/world-lab/src/app/styles.css @@ -1872,7 +1872,8 @@ dd { animation: pulse 1s ease-in-out infinite alternate; } .status-dot.provider-error, -.status-dot.configuration-error { +.status-dot.configuration-error, +.status-dot.budget-exhausted { background: var(--danger); } .command-popover { diff --git a/apps/world-lab/src/components/world-lab.test.tsx b/apps/world-lab/src/components/world-lab.test.tsx index 1d87a41..96a2d32 100644 --- a/apps/world-lab/src/components/world-lab.test.tsx +++ b/apps/world-lab/src/components/world-lab.test.tsx @@ -1108,6 +1108,95 @@ describe('WorldLab', () => { expect(screen.getByText('Deterministic test model')).toBeInTheDocument(); }); + it('stops execution controls and explains exhausted provider attempts', async () => { + const exhausted = simulationSnapshotSchema.parse({ + ...initial, + status: 'budget-exhausted', + experiment: { + ...initial.experiment, + attemptAccounting: { + providerAttemptLimit: 8, + reservedPermits: 0, + attemptsStarted: 8, + attemptsFinalized: 8, + attemptsInFlight: 0, + remainingAttempts: 0, + knownCostCredits: 0.25, + attemptsWithUnknownCost: 1, + exhausted: true, + exhaustionReason: 'provider-attempt-limit', + }, + }, + }); + vi.stubGlobal( + 'fetch', + vi.fn(() => jsonResponse(exhausted)), + ); + render(); + expect(await screen.findByRole('button', { name: 'Start' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'Single tick' })).toBeDisabled(); + await userEvent + .setup() + .click( + screen.getByLabelText(/Experiment details\. Tick 0, budget exhausted/), + ); + expect(screen.getByText('Provider-attempt limit exhausted')).toBeVisible(); + expect(screen.getByText('8 / 8')).toBeVisible(); + expect(screen.getByText('1', { selector: 'dd' })).toBeVisible(); + }); + + it('reconciles the authoritative exhausted snapshot after a budget 409', async () => { + const exhausted = simulationSnapshotSchema.parse({ + ...initial, + status: 'budget-exhausted', + experiment: { + ...initial.experiment, + attemptAccounting: { + providerAttemptLimit: 1, + reservedPermits: 0, + attemptsStarted: 0, + attemptsFinalized: 0, + attemptsInFlight: 0, + remainingAttempts: 1, + knownCostCredits: 0, + attemptsWithUnknownCost: 0, + exhausted: true, + exhaustionReason: 'provider-attempt-limit', + }, + }, + }); + let snapshotReads = 0; + const fetchMock = vi.fn((input, init) => { + if (String(input).includes('/tick') && init?.method === 'POST') + return Promise.resolve( + new Response( + JSON.stringify({ + error: { + code: 'experiment_budget_exhausted', + message: 'The complete tick cannot be reserved.', + }, + }), + { status: 409, headers: { 'content-type': 'application/json' } }, + ), + ); + snapshotReads += 1; + return jsonResponse(snapshotReads === 1 ? initial : exhausted); + }); + vi.stubGlobal('fetch', fetchMock); + render(); + const singleTick = await screen.findByRole('button', { + name: 'Single tick', + }); + expect(singleTick).toBeEnabled(); + await userEvent.setup().click(singleTick); + await waitFor(() => expect(singleTick).toBeDisabled()); + expect(screen.getByRole('button', { name: 'Start' })).toBeDisabled(); + expect( + screen.getByLabelText(/Experiment details\. Tick 0, budget exhausted/), + ).toBeInTheDocument(); + expect(snapshotReads).toBe(2); + }); + it('opens World setup only from the top-right overflow menu with map semantics', async () => { const user = userEvent.setup(); render(); @@ -1137,6 +1226,10 @@ describe('WorldLab', () => { expect( screen.getByLabelText('Maximum virtual minutes per tick'), ).toHaveValue(10); + expect(screen.getByLabelText('Provider attempt limit')).toHaveValue(1000); + expect( + screen.getByLabelText('Unlimited provider attempts'), + ).not.toBeChecked(); for (const label of [ 'World simulation seed', 'Spawn assignment seed', diff --git a/apps/world-lab/src/components/world-lab.tsx b/apps/world-lab/src/components/world-lab.tsx index ad53735..819f8e1 100644 --- a/apps/world-lab/src/components/world-lab.tsx +++ b/apps/world-lab/src/components/world-lab.tsx @@ -244,6 +244,7 @@ export function WorldLab() { setSnapshot(next); if ( next.status === 'configuration-error' || + next.status === 'budget-exhausted' || next.world.hexes.every(({ state }) => state === 'infected') || (boundedRunTargetRef.current !== null && next.tickNumber >= boundedRunTargetRef.current) @@ -495,11 +496,28 @@ export function WorldLab() { { method: 'POST' }, ); if (response.status === 409) { - setUiError('Another tick is already in progress.'); + const error = (await response.json().catch(() => undefined)) as + { error?: { code?: string; message?: string } } | undefined; + const budgetExhausted = + error?.error?.code === 'experiment_budget_exhausted'; + setUiError( + budgetExhausted + ? 'The experiment provider-attempt limit is exhausted. Reset or apply a new World Setup to continue.' + : 'Another tick is already in progress.', + ); runningRef.current = false; setRunning(false); setBoundedRunTarget(null); boundedRunTargetRef.current = null; + if (budgetExhausted) { + try { + await reconcileAuthoritativeSnapshot(); + } catch { + setUiError( + 'The provider-attempt limit was reached, but the authoritative snapshot could not be refreshed. Refresh before retrying.', + ); + } + } return; } if (!response.ok) throw new Error('turn request failed'); @@ -757,7 +775,8 @@ export function WorldLab() { ? 'waiting-for-model' : snapshot.status === 'waiting-for-model' || snapshot.status === 'configuration-error' || - snapshot.status === 'provider-error' + snapshot.status === 'provider-error' || + snapshot.status === 'budget-exhausted' ? snapshot.status : running ? 'running' @@ -834,7 +853,7 @@ export function WorldLab() { {formatCost( - snapshot.experiment.metrics.aggregate.knownCostCredits, + snapshot.experiment.attemptAccounting.knownCostCredits, )} @@ -869,14 +888,44 @@ export function WorldLab() { {snapshot.experiment.metrics.aggregate.directMessagesSent} +
+
Provider attempts
+
+ {snapshot.experiment.attemptAccounting.attemptsStarted} /{' '} + {snapshot.experiment.attemptAccounting.providerAttemptLimit ?? + 'Unlimited'} +
+
+
+
In flight / reserved
+
+ {snapshot.experiment.attemptAccounting.attemptsInFlight} /{' '} + {snapshot.experiment.attemptAccounting.reservedPermits} +
+
Known credits
{formatCost( - snapshot.experiment.metrics.aggregate.knownCostCredits, + snapshot.experiment.attemptAccounting.knownCostCredits, )}
+
+
Unknown-cost attempts
+
+ { + snapshot.experiment.attemptAccounting + .attemptsWithUnknownCost + } +
+
+ {snapshot.experiment.attemptAccounting.exhaustionReason && ( +
+
Execution limit
+
Provider-attempt limit exhausted
+
+ )}
Tokens
@@ -924,7 +973,9 @@ export function WorldLab() { personalityPending || fullyInfected || !snapshot.providerConfigured || - !modelsReady + !modelsReady || + snapshot.experiment.attemptAccounting.exhausted || + snapshot.status === 'budget-exhausted' } type="button" onClick={() => { @@ -948,7 +999,9 @@ export function WorldLab() { personalityPending || !snapshot.providerConfigured || !modelsReady || - snapshot.pendingFailedTurn !== null + snapshot.pendingFailedTurn !== null || + snapshot.experiment.attemptAccounting.exhausted || + snapshot.status === 'budget-exhausted' } type="button" onClick={() => void executeTurn()} @@ -1652,7 +1705,9 @@ function RunHealthSummary({
Known cost
-
{formatCost(metrics.knownCostCredits)}
+
+ {formatCost(snapshot.experiment.attemptAccounting.knownCostCredits)} +
Successful turns
@@ -1819,6 +1874,7 @@ function WorldSetupPanel({ objectiveVersion: scenario.objectiveVersion, capabilities: scenario.capabilities, simulatedPlayer: scenario.simulatedPlayer, + executionLimits: scenario.executionLimits, }); }, [snapshot.scenario]); const [draft, setDraft] = useState(initialDraft); @@ -2238,6 +2294,41 @@ function WorldSetupPanel({ } /> + +