diff --git a/README.md b/README.md index 360ed91..39ef77e 100644 --- a/README.md +++ b/README.md @@ -64,14 +64,16 @@ repair or transient-retry charge per agent. All agents observe the same frozen pre-tick world; valid decisions resolve together while an individual provider failure is retained as that agent's final lost tick. Start is deliberately disabled when the server has no key. This development API has no -authentication or monetary cost controls and is not suitable for an unauthenticated -public deployment. +authentication or provider-account balance enforcement. Its experiment-scoped +attempt and credit-admission limits are operator safeguards, not an upstream +billing guarantee, so it is not suitable for unauthenticated public deployment. State is held only in the Game API process. The API captures one active safe experiment with bounded complete tick groups while the browser snapshot remains -bounded without splitting a tick. Schema-v10 exports add tick number, resolution -position, virtual time, interval, and lost-tick attribution; schema-v9 archive -imports remain supported. Model and reasoning-profile assignments may be changed +bounded without splitting a tick. Schema-v11 exports add an independent safe, +bounded provider-attempt ledger to schema-v10 tick attribution, including work +that did not produce a committed turn; schema-v9/v10 archive imports remain +supported. Model and reasoning-profile assignments may be changed between ticks; behavior locks after tick one. A saved slug absent from the current compatible catalog is preserved and blocks execution until explicitly replaced. Every provider decision is one plain-text response containing a @@ -86,6 +88,9 @@ World Setup also configures server-owned provider-attempt and conservative credit-admission limits. The per-attempt credit reservation bounds admission exposure using exact decimal accounting; it is not an upstream provider-account spending cap or billing guarantee. +Provider-attempt records contain only bounded sanitized attribution, usage, and +failure fields; prompts, raw responses, credentials, and private reasoning are +excluded. ## Opt-in real-provider smoke diff --git a/ROADMAP.md b/ROADMAP.md index 7ba63b0..ceedacf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -110,7 +110,8 @@ Persistent short- and long-term objectives, compact memories, plan revision, sum Scheduled turns, snapshots, replay, retries, idempotency, durable budget/attempt ledgers, failure recovery, and operation without the World Lab browser being open. The current process-local attempt and credit-admission ceilings are an -operator safety boundary, not runtime persistence or provider-account balance -enforcement. +operator safety boundary, and their schema-v11 safe ledger can be exported to +the analysis archive even when no turn committed. This is not active runtime +persistence, restart recovery, or provider-account balance enforcement. Player development begins only after these agent milestones demonstrate compelling behavior. diff --git a/apps/game-api/src/app.test.ts b/apps/game-api/src/app.test.ts index 4858ad7..47cb189 100644 --- a/apps/game-api/src/app.test.ts +++ b/apps/game-api/src/app.test.ts @@ -1150,7 +1150,7 @@ describe('game API simulation boundary', () => { const { document } = experimentExportResponseSchema.parse( await response.json(), ); - expect(document.schemaVersion).toBe(10); + expect(document.schemaVersion).toBe(11); const lostTick = document.turns.find( ({ outcome }) => outcome === 'lost-tick', ); diff --git a/apps/game-api/src/attempt-accounting.test.ts b/apps/game-api/src/attempt-accounting.test.ts index 9060153..ac64195 100644 --- a/apps/game-api/src/attempt-accounting.test.ts +++ b/apps/game-api/src/attempt-accounting.test.ts @@ -170,4 +170,60 @@ describe('AttemptAccounting', () => { expect(accounting.snapshot().attemptsStarted).toBe(0); }, ); + + it('retains independent finalized and in-flight records with honest bounds', () => { + const accounting = new AttemptAccounting(null, null, '0.0100', 2); + const details = (turn: number) => ({ + agentId: '128f3f38-6b7d-4db7-9e95-751b4ce2681e' as never, + intendedTurnNumber: turn, + intendedTickNumber: 1, + kind: 'initial' as const, + startedAt: `2026-08-13T12:00:0${turn}.000Z`, + modelId: 'deterministic-script' as never, + reasoningProfile: 'provider-default' as const, + }); + const first = accounting.startAdditional(details(1))!; + accounting.finalize(first, { + outcome: 'completed', + completedAt: '2026-08-13T12:00:02.000Z', + provider: { + provider: 'scripted-test', + model: 'deterministic-script' as never, + latencyMs: 1, + costCredits: 0.00000001, + }, + }); + accounting.startAdditional(details(2)); + accounting.startAdditional(details(3)); + expect(accounting.ledger()).toMatchObject([ + { intendedTurnNumber: 2, outcome: 'in-flight', reservedCredits: '0.01' }, + { intendedTurnNumber: 3, outcome: 'in-flight', reservedCredits: '0.01' }, + ]); + expect(accounting.retention()).toEqual({ + limit: 2, + totalStartedAttempts: 3, + retainedAttempts: 2, + droppedRecords: 1, + complete: false, + requestedRangeExtendsBeyondRetention: false, + }); + }); + + it('does not mutate accounting when safe start-record validation fails', () => { + const accounting = new AttemptAccounting(2); + expect(accounting.reserve(1)).toBe(true); + const before = accounting.snapshot(); + expect(() => + accounting.startReserved({ + agentId: 'not-a-uuid' as never, + intendedTurnNumber: 1, + kind: 'initial', + startedAt: '2026-08-13T12:00:00.000Z', + modelId: 'deterministic-script' as never, + reasoningProfile: 'provider-default', + }), + ).toThrow(); + expect(accounting.snapshot()).toEqual(before); + expect(accounting.ledger()).toEqual([]); + }); }); diff --git a/apps/game-api/src/attempt-accounting.ts b/apps/game-api/src/attempt-accounting.ts index 711b4fa..76349f3 100644 --- a/apps/game-api/src/attempt-accounting.ts +++ b/apps/game-api/src/attempt-accounting.ts @@ -1,4 +1,13 @@ -import type { ProviderMetadata } from '@hexzero/shared'; +import { + providerAttemptRecordSchema, + type AgentId, + type ModelAttempt, + type ModelId, + type ProviderAttemptRecord, + type ProviderFailure, + type ProviderMetadata, + type ReasoningProfile, +} from '@hexzero/shared'; export type AttemptExhaustionReason = | 'provider-attempt-limit' @@ -24,6 +33,23 @@ export interface AttemptAccountingSnapshot { exhaustionReason: AttemptExhaustionReason | null; } +export interface AttemptStart { + agentId: AgentId; + intendedTurnNumber: number; + intendedTickNumber?: number; + kind: ModelAttempt['kind']; + startedAt: string; + modelId: ModelId; + reasoningProfile: ReasoningProfile; +} + +export interface AttemptCompletion { + outcome: 'completed' | 'provider-error' | 'cancelled' | 'timeout'; + completedAt: string; + provider?: ProviderMetadata; + failure?: ProviderFailure; +} + interface Decimal { integer: bigint; scale: number; @@ -98,6 +124,9 @@ export class AttemptAccounting { #unknownCost = 0; #nextPermitId = 1; readonly #inFlight = new Set(); + readonly #records = new Map(); + #retainedRecords: ProviderAttemptRecord[] = []; + #droppedRecords = 0; #exhaustionReason: AttemptExhaustionReason | null = null; readonly limit: number | null; readonly creditLimit: string | null; @@ -107,6 +136,7 @@ export class AttemptAccounting { limit: number | null, creditLimit: string | null = null, reservationCreditsPerAttempt = '0.01', + readonly ledgerLimit = 10_000, ) { this.limit = limit; this.creditLimit = creditLimit === null ? null : canonical(creditLimit); @@ -142,31 +172,71 @@ export class AttemptAccounting { return true; } - startReserved(): number | null { + startReserved(details?: AttemptStart): number | null { if (this.#reserved < 1) return null; + const permitId = this.#nextPermitId; + const record = details + ? providerAttemptRecordSchema.parse({ + id: crypto.randomUUID(), + ...details, + outcome: 'in-flight', + reservedCredits: this.reservationCreditsPerAttempt, + }) + : undefined; this.#reserved -= 1; this.#started += 1; this.#committedExposure = add( this.#committedExposure, this.reservationCreditsPerAttempt, ); - const permitId = this.#nextPermitId++; + this.#nextPermitId += 1; this.#inFlight.add(permitId); + if (record) { + this.#records.set(permitId, record); + this.#retain(record); + } return permitId; } - startAdditional(): number | null { + startAdditional(details?: AttemptStart): number | null { if (!this.reserve(1)) return null; - return this.startReserved(); + return this.startReserved(details); } releaseReservations(): void { this.#reserved = 0; } - finalize(permitId: number, metadata?: ProviderMetadata): void { - if (!this.#inFlight.delete(permitId)) return; + finalize( + permitId: number, + completion?: ProviderMetadata | AttemptCompletion, + ): void { + if (!this.#inFlight.has(permitId)) return; + const detailed = completion && 'outcome' in completion ? completion : null; + const metadata = + detailed?.provider ?? + (completion && !('outcome' in completion) ? completion : undefined); + const started = this.#records.get(permitId); + let finalizedRecord: ProviderAttemptRecord | undefined; + if (started && detailed) { + finalizedRecord = providerAttemptRecordSchema.parse({ + ...started, + ...detailed, + actualCostCredits: + metadata?.costCredits === undefined + ? undefined + : canonical(String(metadata.costCredits)), + }); + } + this.#inFlight.delete(permitId); this.#finalized += 1; + if (started && finalizedRecord) { + this.#records.delete(permitId); + const index = this.#retainedRecords.findIndex( + ({ id }) => id === started.id, + ); + if (index >= 0) this.#retainedRecords[index] = finalizedRecord; + } if (metadata?.costCredits === undefined) { this.#unknownCost += 1; return; @@ -189,6 +259,31 @@ export class AttemptAccounting { } } + #retain(record: ProviderAttemptRecord): void { + this.#retainedRecords.push(record); + while (this.#retainedRecords.length > this.ledgerLimit) { + const removed = this.#retainedRecords.shift()!; + for (const [permit, candidate] of this.#records) + if (candidate.id === removed.id) this.#records.delete(permit); + this.#droppedRecords += 1; + } + } + + ledger(): readonly ProviderAttemptRecord[] { + return structuredClone(this.#retainedRecords); + } + + retention() { + return { + limit: this.ledgerLimit, + totalStartedAttempts: this.#started, + retainedAttempts: this.#retainedRecords.length, + droppedRecords: this.#droppedRecords, + complete: this.#droppedRecords === 0, + requestedRangeExtendsBeyondRetention: false, + }; + } + #reservedCredits(): string { return multiply(this.reservationCreditsPerAttempt, this.#reserved); } diff --git a/apps/game-api/src/experiment-export.ts b/apps/game-api/src/experiment-export.ts index 75e8a8d..fe1876c 100644 --- a/apps/game-api/src/experiment-export.ts +++ b/apps/game-api/src/experiment-export.ts @@ -35,10 +35,13 @@ import { type AgentGoalState, type MemoryEntry, type SimulatedPlayerEvent, + type ProviderAttemptRecord, + type ProviderAttemptRetention, + type ExperimentAttemptAccounting, } from '@hexzero/shared'; export interface ExperimentSource { - schemaVersion: 9 | 10; + schemaVersion: 9 | 10 | 11; id: ExperimentId; startedAt: string; providerMode: 'openrouter' | 'scripted-test'; @@ -56,6 +59,9 @@ export interface ExperimentSource { agentGoals: readonly { agentId: AgentId; goal: AgentGoalState | null }[]; agentMemories: readonly { agentId: AgentId; entries: MemoryEntry[] }[]; simulatedPlayerEvents: readonly SimulatedPlayerEvent[]; + providerAttempts?: readonly ProviderAttemptRecord[]; + attemptRetention?: ProviderAttemptRetention; + attemptAccounting?: ExperimentAttemptAccounting; } export class ExperimentExportValidationError extends Error { @@ -718,6 +724,7 @@ export function createExperimentExport( const communications = filterCommunications(source, request, selectedSet); const controlChanges = filterControlChanges(source, request, selectedSet); const allianceEvents = filterAllianceEvents(source, request, selectedSet); + const providerAttempts = filterProviderAttempts(source, request, selectedSet); const firstRetainedTurn = source.turns[0]?.turnNumber; const lastRetainedTurn = source.turns.at(-1)?.turnNumber; const requestedRangeExtendsBeyondRetention = rangeExtendsBeyondRetention( @@ -770,7 +777,7 @@ export function createExperimentExport( selection: { selectedAgentIds, matchingTurnCount: filtered.length, - ...(source.schemaVersion === 10 + ...(source.schemaVersion === 10 || source.schemaVersion === 11 ? { matchingTickCount: new Set( filtered.map(({ tickNumber }) => tickNumber).filter(Boolean), @@ -780,6 +787,9 @@ export function createExperimentExport( matchingCommunicationCount: communications.length, matchingControlChangeCount: controlChanges.length, matchingDiplomacyEventCount: allianceEvents.length, + ...(source.schemaVersion === 11 + ? { matchingProviderAttemptCount: providerAttempts.length } + : {}), matchingSimulatedPlayerEventCount: source.simulatedPlayerEvents.filter( (event) => new Set( @@ -863,9 +873,20 @@ export function createExperimentExport( : {}), allianceEvents: structuredClone(allianceEvents), turns: filtered.map((turn) => exportTurn(turn, request)), - ...(source.schemaVersion === 10 + ...(source.schemaVersion === 10 || source.schemaVersion === 11 ? { tickSummaries: summarizeTicks(filtered) } : {}), + ...(source.schemaVersion === 11 + ? { + providerAttempts, + attemptRetention: { + ...source.attemptRetention!, + requestedRangeExtendsBeyondRetention: + source.attemptRetention!.droppedRecords > 0, + }, + attemptAccounting: source.attemptAccounting!, + } + : {}), }; return experimentExportDocumentSchema.parse(document); } @@ -961,6 +982,13 @@ export function createExperimentPreview( new Set(document.selection.selectedAgentIds), ), ); + const ledgerKnownCost = (document.providerAttempts ?? []).reduce( + (sum, attempt) => sum + Number(attempt.actualCostCredits ?? 0), + 0, + ); + const ledgerUnknownCost = (document.providerAttempts ?? []).filter( + ({ actualCostCredits }) => actualCostCredits === undefined, + ).length; return experimentExportPreviewSchema.parse({ experimentId: source.id, matchingTurnCount: document.selection.matchingTurnCount, @@ -970,12 +998,20 @@ export function createExperimentPreview( matchingCommunicationCount: document.selection.matchingCommunicationCount, matchingControlChangeCount: document.selection.matchingControlChangeCount, matchingDiplomacyEventCount: document.selection.matchingDiplomacyEventCount, + matchingProviderAttemptCount: + document.selection.matchingProviderAttemptCount ?? 0, selectedAgentCount: document.selection.selectedAgentIds.length, firstMatchingTurn: document.selection.firstMatchingTurn, lastMatchingTurn: document.selection.lastMatchingTurn, retention: document.retention, - knownCostCredits: metrics.aggregate.knownCostCredits, - attemptsWithUnknownCost: metrics.aggregate.attemptsWithUnknownCost, + knownCostCredits: + document.schemaVersion === 11 + ? ledgerKnownCost + : metrics.aggregate.knownCostCredits, + attemptsWithUnknownCost: + document.schemaVersion === 11 + ? ledgerUnknownCost + : metrics.aggregate.attemptsWithUnknownCost, turnsWithUnknownCost: metrics.aggregate.turnsWithUnknownCost, serializedUtf8Bytes, approximateAiInputTokens: Math.ceil(serializedUtf8Bytes / 4), @@ -1027,6 +1063,36 @@ function filterTurns( return turns.map((turn) => structuredClone(turn)); } +function filterProviderAttempts( + source: ExperimentSource, + request: ExperimentExportRequest, + selected: Set, +): ProviderAttemptRecord[] { + let attempts = (source.providerAttempts ?? []).filter(({ agentId }) => + selected.has(agentId), + ); + if (request.turns.mode === 'range') { + const { fromTurn, toTurn } = request.turns; + attempts = attempts.filter( + ({ intendedTurnNumber }) => + intendedTurnNumber >= fromTurn && intendedTurnNumber <= toTurn, + ); + } else if (request.turns.mode === 'latest') { + const fromTurn = Math.max( + 1, + source.totalCompletedTurns - request.turns.count + 1, + ); + attempts = attempts.filter( + ({ intendedTurnNumber }) => intendedTurnNumber >= fromTurn, + ); + } + return structuredClone(attempts).sort( + (left, right) => + left.startedAt.localeCompare(right.startedAt) || + left.id.localeCompare(right.id), + ); +} + function filterCommunications( source: ExperimentSource, request: ExperimentExportRequest, diff --git a/apps/game-api/src/simulation-service.test.ts b/apps/game-api/src/simulation-service.test.ts index aaa77d4..b23a6c1 100644 --- a/apps/game-api/src/simulation-service.test.ts +++ b/apps/game-api/src/simulation-service.test.ts @@ -1278,6 +1278,17 @@ describe('SimulationService', () => { expect(accounting.attemptsFinalized).toBe(accounting.attemptsStarted); expect(accounting.attemptsWithUnknownCost).toBe(accounting.attemptsStarted); expect(accounting.attemptsStarted).toBe(8); + const cancelledExport = simulation.generateExperimentExport( + exportRequest('minimal'), + ); + expect(cancelledExport.turns).toEqual([]); + expect(cancelledExport.providerAttempts).toHaveLength(8); + expect( + cancelledExport.providerAttempts?.every( + ({ outcome, failure }) => + outcome === 'cancelled' && failure?.code === 'cancelled', + ), + ).toBe(true); }); it('retains safe provider cost when legacy cancellation is observed after the response', async () => { @@ -1308,6 +1319,7 @@ describe('SimulationService', () => { ); expect(simulation.getSnapshot()).toMatchObject({ turnNumber: 0, + turns: [], experiment: { attemptAccounting: { attemptsStarted: 1, @@ -1318,6 +1330,81 @@ describe('SimulationService', () => { }, }, }); + expect( + simulation.generateExperimentExport(exportRequest('minimal')), + ).toMatchObject({ + turns: [], + providerAttempts: [ + { + outcome: 'completed', + actualCostCredits: '0.125', + }, + ], + }); + expect( + simulation.generateExperimentExport(exportRequest('minimal')) + .providerAttempts?.[0], + ).not.toHaveProperty('failure'); + }); + + it('keeps completed and truly cancelled simultaneous calls distinct without committing', async () => { + const firstAgentId = defaultWorldSetupRequest().roster[0]!.id; + let completed!: () => void; + const firstCompleted = new Promise((resolve) => { + completed = resolve; + }); + const simulation = service({ + mode: 'scripted-test', + model: 'deterministic-script', + configured: true, + async decide(observation, _model, options): Promise { + if (observation.agentId === firstAgentId) { + setTimeout(completed, 0); + return { + decision: { + worldAction: { type: 'wait' }, + summary: 'Provider work completed before cancellation.', + }, + metadata: { + provider: 'scripted-test', + model: 'deterministic-script', + latencyMs: 1, + costCredits: 0.01, + }, + }; + } + await new Promise((resolve) => + options?.signal?.addEventListener('abort', () => resolve(), { + once: true, + }), + ); + throw new AgentProviderError({ + code: 'cancelled', + message: 'Provider work was actually aborted.', + retryable: false, + }); + }, + }); + const pending = simulation.executeNextTick(); + await firstCompleted; + simulation.cancelCurrentRequest(); + await expect(pending).rejects.toBeInstanceOf(SimulationTurnCancelledError); + const snapshot = simulation.getSnapshot(); + expect(snapshot).toMatchObject({ tickNumber: 0, turnNumber: 0, turns: [] }); + const exported = simulation.generateExperimentExport( + exportRequest('minimal'), + ); + expect(exported.turns).toEqual([]); + expect( + exported.providerAttempts?.filter( + ({ outcome }) => outcome === 'completed', + ), + ).toHaveLength(1); + expect( + exported.providerAttempts?.filter( + ({ outcome }) => outcome === 'cancelled', + ), + ).toHaveLength(7); }); it('dispatches per-agent model and reasoning overrides for a tick', async () => { @@ -1388,7 +1475,7 @@ describe('SimulationService', () => { ); expect( legacy.generateExperimentExport(exportRequest('minimal')).schemaVersion, - ).toBe(9); + ).toBe(11); const tick = service( new ScriptedAgentProvider( @@ -1408,7 +1495,7 @@ describe('SimulationService', () => { expect(() => tick.skipFailedTurn()).toThrow(SimulationConflictError); expect( tick.generateExperimentExport(exportRequest('minimal')).schemaVersion, - ).toBe(10); + ).toBe(11); expect( tick.generateExperimentExport(exportRequest('minimal')).experiment .decisionContractVersion, @@ -1452,14 +1539,25 @@ describe('SimulationService', () => { ], }); expect(retained.turns).toHaveLength(8); - expect(retained.schemaVersion).toBe(10); + expect(retained.schemaVersion).toBe(11); expect(new Set(retained.turns.map(({ tickNumber }) => tickNumber))).toEqual( new Set([2]), ); expect(experimentExportDocumentSchema.safeParse(retained).success).toBe( true, ); + expect( + experimentExportDocumentSchema.safeParse({ + ...retained, + simulatedPlayerMetrics: undefined, + }).success, + ).toBe(false); const legacyV10 = structuredClone(retained); + legacyV10.schemaVersion = 10; + delete legacyV10.providerAttempts; + delete legacyV10.attemptRetention; + delete legacyV10.attemptAccounting; + delete legacyV10.selection.matchingProviderAttemptCount; delete legacyV10.simulatedPlayerMetrics; expect( experimentExportDocumentSchema.parse(legacyV10).simulatedPlayerMetrics, @@ -2334,7 +2432,7 @@ describe('SimulationService', () => { actions: ['capture'], level: 'minimal', }); - expect(victimExport.schemaVersion).toBe(9); + expect(victimExport.schemaVersion).toBe(11); expect(victimExport.turns).toHaveLength(0); expect(victimExport.selection).toMatchObject({ matchingTurnCount: 0, @@ -2444,7 +2542,7 @@ describe('SimulationService', () => { outcomes: ['rejected'], actions: ['capture'], }); - expect(exported.schemaVersion).toBe(9); + expect(exported.schemaVersion).toBe(11); const behavior = exported.turns[0]!.behavior!; expect( exported.metrics!.byPersonality.find( @@ -3041,7 +3139,40 @@ describe('SimulationService', () => { controlChanges: false, }, }); - expect(minimal.schemaVersion).toBe(9); + expect(minimal.schemaVersion).toBe(11); + expect(minimal.providerAttempts).toHaveLength(1); + expect(minimal.selection.matchingProviderAttemptCount).toBe(1); + expect( + experimentExportDocumentSchema.safeParse({ + ...minimal, + providerAttempts: [ + ...minimal.providerAttempts!, + minimal.providerAttempts![0], + ], + selection: { + ...minimal.selection, + matchingProviderAttemptCount: 2, + }, + }).success, + ).toBe(false); + expect( + experimentExportDocumentSchema.safeParse({ + ...minimal, + selection: { + ...minimal.selection, + matchingProviderAttemptCount: 0, + }, + }).success, + ).toBe(false); + expect( + experimentExportDocumentSchema.safeParse({ + ...minimal, + attemptRetention: { + ...minimal.attemptRetention!, + totalStartedAttempts: 2, + }, + }).success, + ).toBe(false); expect( experimentExportDocumentSchema.safeParse({ ...minimal, @@ -4377,6 +4508,17 @@ describe('SimulationService', () => { }, }, }); + expect( + simulation.generateExperimentExport(exportRequest('minimal')) + .providerAttempts, + ).toMatchObject([ + { + intendedTurnNumber: 1, + outcome: 'provider-error', + failure: { code: 'simulation-validation' }, + actualCostCredits: '0.125', + }, + ]); const recovered = await simulation.retryFailedTurn(); expect(recovered).toMatchObject({ @@ -4712,6 +4854,9 @@ describe('SimulationService', () => { available: false, issue: 'unavailable', }); + expect(() => + simulation.importModelConfiguration({ schemaVersion: 12 }), + ).toThrow('Only schema-version 5 through 11'); const legacy = simulation.importModelConfiguration({ schemaVersion: 5 }); expect(legacy.legacy).toBe(true); expect(legacy.snapshot.modelConfiguration.globalModelId).toBeNull(); diff --git a/apps/game-api/src/simulation-service.ts b/apps/game-api/src/simulation-service.ts index 0dbba83..75237d3 100644 --- a/apps/game-api/src/simulation-service.ts +++ b/apps/game-api/src/simulation-service.ts @@ -105,11 +105,13 @@ import { AttemptAccounting } from './attempt-accounting'; function attemptAccountingForScenario( executionLimits: AppliedScenario['executionLimits'], + retentionLimit = DEFAULT_EXPERIMENT_RETENTION, ): AttemptAccounting { return new AttemptAccounting( executionLimits.providerAttemptLimit, executionLimits.creditLimit, executionLimits.reservationCreditsPerAttempt, + retentionLimit * 2, ); } @@ -360,6 +362,7 @@ export class SimulationService { }; this.#attemptAccounting = attemptAccountingForScenario( this.#scenario.executionLimits, + this.#experimentRetentionLimit, ); } @@ -463,6 +466,7 @@ export class SimulationService { ]); this.#attemptAccounting = attemptAccountingForScenario( this.#scenario.executionLimits, + this.#experimentRetentionLimit, ); this.#modelConfiguration = { ...structuredClone(this.#scenario.modelConfiguration), @@ -607,6 +611,7 @@ export class SimulationService { ]); this.#attemptAccounting = attemptAccountingForScenario( this.#scenario.executionLimits, + this.#experimentRetentionLimit, ); this.#status = this.#provider.configured ? 'paused' : 'configuration-error'; return this.getSnapshot(); @@ -749,11 +754,12 @@ export class SimulationService { version !== 7 && version !== 8 && version !== 9 && - version !== 10 + version !== 10 && + version !== 11 ) throw new SimulationValidationError( 'invalid_model_configuration', - 'Only schema-version 5 through 10 experiment exports can be imported.', + 'Only schema-version 5 through 11 experiment exports can be imported.', ); if (version === 5) { const legacyConfiguration: ExperimentModelConfiguration = { @@ -1115,8 +1121,9 @@ export class SimulationService { this.#cancellationRequested = false; this.#status = 'waiting-for-model'; const deadlineAtMs = Date.now() + OPENROUTER_PROVIDER_TIMEOUT_MS; + let dispatched: Awaited> = []; try { - const dispatched = await dispatchTickDecisions( + dispatched = await dispatchTickDecisions( this.#provider, agents.map(({ id }) => { const resolved = this.#resolvedModel(id); @@ -1132,15 +1139,28 @@ export class SimulationService { deadlineAtMs, signal: controller.signal, now: this.#now, - beginAttempt: (_job, kind) => { + deferSuccessfulFinalization: true, + beginAttempt: (job, kind, attemptStartedAt) => { + const intendedTurnNumber = + this.#completedTurnCount + + Math.max(1, order.indexOf(job.agentId) + 1); + const details = { + agentId: job.agentId, + intendedTurnNumber, + intendedTickNumber: tickNumber, + kind, + startedAt: attemptStartedAt, + modelId: job.modelId, + reasoningProfile: job.reasoningProfile, + }; const permitId = kind === 'initial' - ? this.#attemptAccounting.startReserved() - : this.#attemptAccounting.startAdditional(); + ? this.#attemptAccounting.startReserved(details) + : this.#attemptAccounting.startAdditional(details); return permitId === null ? null - : (metadata) => - this.#attemptAccounting.finalize(permitId, metadata); + : (completion) => + this.#attemptAccounting.finalize(permitId, completion); }, }, ); @@ -1321,11 +1341,32 @@ export class SimulationService { playerAdvance.events, committedObservationEvents, ); + for (const result of dispatched) + if (result.outcome === 'completed') result.finalizeAttempt?.(); this.#status = this.#attemptAccounting.snapshot().exhausted ? 'budget-exhausted' : 'paused'; return records; } catch (error) { + const cancelledAttempt = + controller.signal.aborted || + error instanceof SimulationTurnCancelledError; + const failure: ProviderFailure = cancelledAttempt + ? { + code: 'cancelled', + message: 'The model request was cancelled by the operator.', + retryable: false, + } + : { + code: 'simulation-validation', + message: 'The simultaneous tick could not be committed safely.', + retryable: true, + }; + for (const result of dispatched) + if (result.outcome === 'completed') { + if (cancelledAttempt) result.finalizeAttempt?.('completed'); + else result.finalizeAttempt?.('provider-error', failure); + } if ( controller.signal.aborted || (error && @@ -1490,6 +1531,7 @@ export class SimulationService { const selectedModel = resolvedModel.modelId!; let providerResult: ProviderDecision | undefined; let successfulProviderMetadata: ProviderMetadata | undefined; + let successfulAccountingPermitId: number | null = null; const attemptHistory = [...(pending?.attempts ?? [])]; const deadlineAtMs = Date.now() + OPENROUTER_PROVIDER_TIMEOUT_MS; @@ -1501,7 +1543,15 @@ export class SimulationService { let validationFeedback: ProviderFailure['validationCodes'] = pending?.failure.validationCodes; for (let automaticCall = 0; automaticCall < 2; automaticCall += 1) { - const accountingPermitId = this.#attemptAccounting.startAdditional(); + const currentAttemptStartedAt = this.#now(); + const accountingPermitId = this.#attemptAccounting.startAdditional({ + agentId: agent.id, + intendedTurnNumber: turnNumber, + kind: nextKind, + startedAt: currentAttemptStartedAt, + modelId: selectedModel, + reasoningProfile: resolvedModel.reasoningProfile, + }); if (accountingPermitId === null) { this.#status = 'budget-exhausted'; throw new SimulationValidationError( @@ -1509,10 +1559,10 @@ export class SimulationService { 'The experiment does not have enough provider-attempt or credit-admission capacity.', ); } - const currentAttemptStartedAt = this.#now(); successfulAttemptStartedAt = currentAttemptStartedAt; successfulAttemptKind = nextKind; let accountingMetadata: ProviderMetadata | undefined; + let accountingFailure: ProviderFailure | undefined; try { providerResult = await this.#provider.decide( providerObservation, @@ -1542,9 +1592,15 @@ export class SimulationService { const providerError = asProviderError(error); accountingMetadata = providerError.metadata ?? accountingMetadata; if (providerError.failure.code === 'cancelled') { + // A response that already returned is completed provider work even + // when cancellation prevents the later world commit. Leave its + // finalization to the outer rollback boundary. + if (!successfulProviderMetadata) + accountingFailure = providerError.failure; this.#status = 'paused'; throw new SimulationTurnCancelledError(); } + accountingFailure = providerError.failure; const attemptProvider = providerError.metadata ?? { provider: this.#provider.mode, model: providerError.failure.model ?? selectedModel, @@ -1636,10 +1692,19 @@ export class SimulationService { : 'provider-error'; return record; } finally { - this.#attemptAccounting.finalize( - accountingPermitId, - accountingMetadata, - ); + if (accountingFailure) + this.#attemptAccounting.finalize(accountingPermitId, { + outcome: + accountingFailure.code === 'cancelled' + ? 'cancelled' + : accountingFailure.code === 'timeout' + ? 'timeout' + : 'provider-error', + completedAt: this.#now(), + provider: accountingMetadata, + failure: accountingFailure, + }); + else successfulAccountingPermitId = accountingPermitId; } } @@ -1775,6 +1840,14 @@ export class SimulationService { }, committedObservationEvents, ); + if (successfulAccountingPermitId !== null) { + this.#attemptAccounting.finalize(successfulAccountingPermitId, { + outcome: 'completed', + completedAt: this.#now(), + provider: successfulProviderMetadata, + }); + successfulAccountingPermitId = null; + } this.#status = this.#attemptAccounting.snapshot().exhausted ? 'budget-exhausted' : 'paused'; @@ -1807,6 +1880,15 @@ export class SimulationService { }, } satisfies ModelAttempt; const attempts = [...attemptHistory, attempt]; + if (successfulAccountingPermitId !== null) { + this.#attemptAccounting.finalize(successfulAccountingPermitId, { + outcome: 'provider-error', + completedAt: this.#now(), + provider: attempt.provider, + failure, + }); + successfulAccountingPermitId = null; + } this.#pendingFailedTurn = { turnNumber, agentId: agent.id, @@ -1832,6 +1914,43 @@ export class SimulationService { allianceEvents: [], }); } finally { + if (successfulAccountingPermitId !== null) { + const cancelled = Boolean( + this.#activeRequestController?.signal.aborted, + ); + if (cancelled && successfulProviderMetadata) { + this.#attemptAccounting.finalize(successfulAccountingPermitId, { + outcome: 'completed', + completedAt: this.#now(), + provider: successfulProviderMetadata, + }); + successfulAccountingPermitId = null; + } + } + if (successfulAccountingPermitId !== null) { + const cancelled = Boolean( + this.#activeRequestController?.signal.aborted, + ); + const failure: ProviderFailure = cancelled + ? { + code: 'cancelled', + message: 'The model request was cancelled by the operator.', + retryable: false, + model: selectedModel, + } + : { + code: 'simulation-validation', + message: 'The provider result could not be committed safely.', + retryable: true, + model: selectedModel, + }; + this.#attemptAccounting.finalize(successfulAccountingPermitId, { + outcome: cancelled ? 'cancelled' : 'provider-error', + completedAt: this.#now(), + provider: successfulProviderMetadata, + failure, + }); + } this.#busy = false; this.#activeAgentId = null; this.#activeRequestController = null; @@ -1970,8 +2089,10 @@ export class SimulationService { modelConfiguration: this.#modelConfiguration, behaviorConfiguration: this.#behaviorConfiguration, scenario: this.#scenario, - schemaVersion: - this.#completedTickCount > 0 || this.#completedTurnCount === 0 ? 10 : 9, + schemaVersion: 11, + providerAttempts: this.#attemptAccounting.ledger(), + attemptRetention: this.#attemptAccounting.retention(), + attemptAccounting: this.#attemptAccounting.snapshot(), agentGoals: [...this.#state.agents.keys()].map((agentId) => ({ agentId, goal: structuredClone(this.#agentGoals.get(agentId) ?? null), diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9c85e49..389840b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -302,3 +302,9 @@ Versioned behavior profiles, seeded assignment, and authoritative diplomacy affo Behavior configuration is experiment-owned and includes registry version 1, assignment mode, seed, and one allowlisted personality/strategy pair per agent. Balanced random is the safe default. Reset creates a new experiment and deterministic assignment from its seed; the first completed turn locks behavior. Every retained turn copies its effective assignment, while model and reasoning changes retain their existing between-request semantics. Schema-v8 computed metrics include complete personality, strategy, observed personality/strategy-combination, and agent breakdowns derived from the same filtered retained turns and attempt records as the aggregate. Logical-turn, provider-call, failure, recovery, token, and cost counters therefore remain attributable without storing reasoning. Structural provider failures retain the broad compatibility code plus bounded details such as missing proposal IDs and contradictory diplomacy recipient fields in attempt telemetry and safe exports. Well-formed unavailable IDs remain engine-authoritative rejections and are not retried. + +# Provider-attempt accounting + +Provider work has an independent bounded lifecycle ledger. Schema-v11 exports +and archive-v4 preserve safe attempt records even when no world turn commits; +turn-owned model-attempt arrays remain compatibility detail only. diff --git a/docs/EXPERIMENT_ARCHIVE.md b/docs/EXPERIMENT_ARCHIVE.md index 2133da3..95ac3f3 100644 --- a/docs/EXPERIMENT_ARCHIVE.md +++ b/docs/EXPERIMENT_ARCHIVE.md @@ -1,6 +1,6 @@ # Local experiment archive -The archive accepts current schema-v10 exports and legacy schema-v9 exports. +The archive accepts current schema-v11 exports and legacy schema-v9/v10 exports. Schema-v10 may add safe goal revision/result and current-goal fields. Current archive imports validate and preserve compatibility while normalized goal analytics remain deferred; the archive never restores active goal state. Schema-v10 may also include compact memory requests, results, and current ledgers. The archive accepts these additive fields observationally but does not normalize, rank, retrieve, or restore memory. Migration 2 adds nullable tick number, deterministic tick position, virtual @@ -105,3 +105,11 @@ pnpm experiment:db compare --format markdown The archive stores only schema-validated safe export fields and curated notes. It rejects recognizable credential material and prohibited raw/private reasoning keys. It does not store fixed prompts, raw provider bodies, authorization headers, private chain-of-thought, or provider credentials. Agent messages and notes remain untrusted research data. MCP and embeddings are deferred because bounded local retrieval solves the immediate need without a network/tool authorization surface or derived semantic store. `ExperimentQueryService` and `ResearchNoteService` are the future extension point for a read-only MCP adapter; write/import authority remains outside that adapter. + +# Schema-v11 provider attempts + +Archive schema v4 stores schema-v11 `providerAttempts` independently of turns. +Use `pnpm experiment:db provider-attempts ` to inspect committed +and uncommitted provider work. Monetary values round-trip as canonical TEXT. +For v11 summaries this ledger is canonical; legacy model attempts are not added +again. The SQLite archive is for analysis and is not active runtime recovery. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index a8d3dbe..0d80927 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -101,9 +101,9 @@ credentials, and authorization headers are not retained. The Game API captures only schema-validated safe observations, requested world actions, optional communication and diplomacy intents, separate result records, visible concise summaries, bounded message text, typed alliance events, sanitized rejected attempts, bounded provider failures, and normalized usage metadata. Malformed identifiers use nullable or absent sanitized representations; raw provider output is never retained. It never records or exports API keys, authorization data, fixed or hidden prompts, raw provider request/response bodies, private chain-of-thought, hidden analysis, secrets, or unbounded diagnostics. Historical records are cloned and immutable. -Export requests, agent IDs, levels, ranges, outcome/world-action filters, communication channel/status filters, and Custom dependencies are runtime-validated. Filtering and metrics remain server-owned. Schema v10 requires complete tick attribution and canonical safe per-tick summaries derived from all model attempts; schema-v9 remains the legacy sequential export format and documented older safe imports remain supported by the Game API. Selected-agent exports use sender/recipient-aware communication filtering and direct multi-agent relevance for proposals and membership changes; unrelated direct messages and rejected diplomacy are excluded. Reset clears communications, alliances, proposals, alliance events, and their metrics while preserving active personality values and unlocking preserved assignments for the new experiment. +Export requests, agent IDs, levels, ranges, outcome/world-action filters, communication channel/status filters, and Custom dependencies are runtime-validated. Filtering and metrics remain server-owned. Schema v11 preserves schema-v10 tick attribution plus the independent safe provider-attempt ledger; schema-v9/v10 imports remain supported by the Game API. Selected-agent exports use sender/recipient-aware communication filtering and direct multi-agent relevance for proposals and membership changes; unrelated direct messages and rejected diplomacy are excluded. Reset clears communications, alliances, proposals, alliance events, and their metrics while preserving active personality values and unlocking preserved assignments for the new experiment. -Actual cost is accepted only from OpenRouter's safe `usage.cost`. Missing cost is unknown, never zero; scripted-test providers explicitly report zero. The active Game API enforces an attempt ceiling but still has no authentication, credit-denominated budget, restartable persistence, provider-management endpoint, upload, or sharing link. The loopback-only boundary remains mandatory. +Actual cost is accepted only from OpenRouter's safe `usage.cost`. Missing cost is unknown, never zero; scripted-test providers explicitly report zero. The active Game API enforces experiment-scoped attempt and conservative credit-admission ceilings, but has no authentication, provider-account balance enforcement, restartable persistence, provider-management endpoint, upload, or sharing link. Credit admission is not an upstream billing guarantee. The loopback-only boundary remains mandatory. The offline experiment archive adds local persistence only for complete schema-validated safe exports and explicitly curated Markdown notes. Imports scan for prohibited credential/private-reasoning fields and recognizable credential values before a transaction begins; failures roll back. Both the canonical `.hexzero/` and compatible legacy `.agentborne/` database locations are ignored. The CLI exposes bounded typed queries, not arbitrary SQL, and adds no MCP, embedding, vector-store, or network-listener surface. @@ -124,3 +124,9 @@ Strategic goals and revision reasons are bounded, agent-authored, untrusted data ## Compact memory text Compact memories are bounded self-authored recollections, not authoritative facts. They remain subordinate observation data and are never interpolated into system instructions. Memory may not retain raw prompts, provider payloads, credentials, or private chain-of-thought. Server-issued IDs and tick attribution are authoritative; memory prose is not. + +# Safe provider-attempt records + +Attempt records may contain sanitized provider metadata and bounded failures, +but never prompts, raw requests/responses, headers, credentials, or private +reasoning. They do not grant authority over world state. diff --git a/docs/TESTING.md b/docs/TESTING.md index 49e88ce..ecce9ae 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -197,3 +197,9 @@ The separately opted-in `pnpm smoke:openrouter -- [initi World Lab also offers an explicit “Test selected model” probe. It uses the production text/flat-JSON contract and selected reasoning profile, does not advance or mutate the world, may incur a small charge, and is cached by model plus profile plus contract version. It is never invoked by deterministic validation or CI. Experiment-archive tests generate current schema-v9 exports through the deterministic scripted provider and cover migration/reopen, canonical and legacy path resolution, legacy export filenames, transactional rollback, stable idempotent identifiers, normalized round trips, bounded filters, summary/comparison calculations, canonical metric agreement with preserved source inconsistencies, Patient Zero classifications, retention completeness, FTS notes/supersession, explicit persistence failures, and credential/private-reasoning rejection. They use only in-memory or temporary SQLite databases and make no live provider or geocoding requests. + +# Provider-attempt ledger + +Offline tests cover attempt success, failure, cancellation, timeout, retry, +retention, export filtering, and archive idempotency. Real-provider calls remain +explicit opt-in smoke tests and are excluded from default CI. diff --git a/docs/adr/0027-independent-provider-attempt-ledger.md b/docs/adr/0027-independent-provider-attempt-ledger.md new file mode 100644 index 0000000..39bd8b4 --- /dev/null +++ b/docs/adr/0027-independent-provider-attempt-ledger.md @@ -0,0 +1,31 @@ +# ADR 0027: Independent safe provider-attempt ledger + +## Status + +Accepted + +## Decision + +Schema-v11 experiment exports include a bounded, independent record for every +started provider call. A record is created before dispatch and finalized once +as completed, provider-error, cancelled, or timeout. It survives tick rollback +and is not derived from committed turns. The safe record contains attribution, +timing, sanitized provider usage/failure data, and canonical decimal-string +reservation and actual-cost values. It never contains prompts, raw responses, +headers, credentials, or private reasoning. + +The in-memory ledger retains twice the configured experiment turn-retention +limit, accounting for the initial call plus one bounded automatic recovery per +turn. Its retention summary discloses total, retained, and dropped records. +Applying or resetting a scenario starts a new experiment and clears the ledger. + +Archive schema v4 stores these records in `provider_attempts`, independently of +turn foreign keys. For v11, usage queries use this table as canonical and do not +add legacy `turn.modelAttempts`. Schema-v9 and v10 imports retain their existing +legacy behavior. + +## Consequences + +Cancelled and otherwise uncommitted paid work remains auditable, including a +zero-turn experiment. Repeated exports add attempt UUIDs idempotently. The +archive remains an analysis archive, not runtime persistence or crash recovery. diff --git a/packages/agent-runtime/src/tick-dispatcher.test.ts b/packages/agent-runtime/src/tick-dispatcher.test.ts index 1a39e57..9713872 100644 --- a/packages/agent-runtime/src/tick-dispatcher.test.ts +++ b/packages/agent-runtime/src/tick-dispatcher.test.ts @@ -298,7 +298,8 @@ describe('dispatchTickDecisions', () => { nowMs: () => 0, beginAttempt: () => { if (permits-- < 1) return null; - return (metadata) => finalized.push(metadata?.costCredits); + return (completion) => + finalized.push(completion.provider?.costCredits); }, }, ); @@ -328,7 +329,7 @@ describe('dispatchTickDecisions', () => { { deadlineAtMs: Date.now() + 1_000, signal: controller.signal, - beginAttempt: () => (metadata) => finalized.push(metadata), + beginAttempt: () => (completion) => finalized.push(completion), }, ); await Promise.resolve(); @@ -336,6 +337,74 @@ describe('dispatchTickDecisions', () => { await expect(pending).rejects.toMatchObject({ failure: { code: 'cancelled' }, }); - expect(finalized).toEqual([undefined]); + expect(finalized).toMatchObject([ + { outcome: 'cancelled', failure: { code: 'cancelled' } }, + ]); + }); + + it('distinguishes a completed call from a concurrently aborted call', async () => { + const controller = new AbortController(); + const finalized = new Map(); + let callOrdinal = 0; + let completedFinalization!: () => void; + const firstFinalized = new Promise((resolve) => { + completedFinalization = resolve; + }); + const pending = dispatchTickDecisions( + { + mode: 'scripted-test', + configured: true, + async decide(_observation, _model, options) { + const ordinal = callOrdinal++; + if (ordinal === 0) { + return { + decision: { + worldAction: { type: 'wait' }, + summary: 'Completed before sibling cancellation.', + }, + metadata: { + provider: 'scripted-test', + model: jobs[0]!.modelId, + latencyMs: 1, + costCredits: 0.01, + }, + }; + } + await new Promise((resolve) => + options?.signal?.addEventListener('abort', () => resolve(), { + once: true, + }), + ); + throw new Error('unsafe error after abort'); + }, + }, + jobs.slice(0, 2), + { + concurrency: 2, + deadlineAtMs: Date.now() + 1_000, + signal: controller.signal, + beginAttempt: (job) => (completion) => { + finalized.set(job.agentId, completion); + if ( + job.agentId === jobs[0]!.agentId && + completion.outcome === 'completed' + ) + completedFinalization(); + }, + }, + ); + await firstFinalized; + controller.abort(); + await expect(pending).rejects.toMatchObject({ + failure: { code: 'cancelled' }, + }); + expect(finalized.get(jobs[0]!.agentId)).toMatchObject({ + outcome: 'completed', + provider: { costCredits: 0.01 }, + }); + expect(finalized.get(jobs[1]!.agentId)).toMatchObject({ + outcome: 'cancelled', + failure: { code: 'cancelled' }, + }); }); }); diff --git a/packages/agent-runtime/src/tick-dispatcher.ts b/packages/agent-runtime/src/tick-dispatcher.ts index 0398e3b..c625995 100644 --- a/packages/agent-runtime/src/tick-dispatcher.ts +++ b/packages/agent-runtime/src/tick-dispatcher.ts @@ -24,6 +24,10 @@ export interface TickDecisionSuccess extends TickDecisionJob { outcome: 'completed'; decision: ProviderDecision; attempts: ModelAttempt[]; + finalizeAttempt?: ( + outcome?: 'completed' | 'provider-error' | 'cancelled', + failure?: ProviderFailure, + ) => void; } export interface TickDecisionFailure extends TickDecisionJob { @@ -41,10 +45,19 @@ export interface TickDispatcherOptions { now?: () => string; nowMs?: () => number; waitForMs?: (delayMs: number, signal?: AbortSignal) => Promise; + deferSuccessfulFinalization?: boolean; beginAttempt?: ( job: TickDecisionJob, kind: ModelAttempt['kind'], - ) => ((metadata?: ProviderDecision['metadata']) => void) | null; + startedAt: string, + ) => + | ((completion: { + outcome: 'completed' | 'provider-error' | 'cancelled' | 'timeout'; + completedAt: string; + provider?: ProviderDecision['metadata']; + failure?: ProviderFailure; + }) => void) + | null; } /** @@ -80,7 +93,8 @@ export async function dispatchTickDecisions( results[index] = deadlineFailure(job, attempts); break; } - const finalizeAccounting = options.beginAttempt?.(job, kind); + const startedAt = now(); + const finalizeAccounting = options.beginAttempt?.(job, kind, startedAt); if (options.beginAttempt && !finalizeAccounting) { const failure: ProviderFailure = { code: 'budget-exhausted', @@ -92,8 +106,11 @@ export async function dispatchTickDecisions( results[index] = { ...job, outcome: 'lost-tick', failure, attempts }; break; } - const startedAt = now(); let accountingMetadata: ProviderDecision['metadata'] | undefined; + let accountingFailure: ProviderFailure | undefined; + let accountingOutcome: + 'completed' | 'provider-error' | 'cancelled' | 'timeout' = + 'provider-error'; try { const decision = await decideBeforeDeadline(provider, job, { reasoningProfile: job.reasoningProfile, @@ -106,6 +123,8 @@ export async function dispatchTickDecisions( if (nowMs() >= options.deadlineAtMs) { const completedAt = now(); const failure = timeoutFailure(job); + accountingFailure = failure; + accountingOutcome = 'timeout'; results[index] = deadlineFailure(job, [ ...attempts, { @@ -130,7 +149,27 @@ export async function dispatchTickDecisions( reasoningProfile: job.reasoningProfile, provider: decision.metadata, }); - results[index] = { ...job, outcome: 'completed', decision, attempts }; + let finalized = false; + const completedResult: TickDecisionSuccess = { + ...job, + outcome: 'completed', + decision, + attempts, + finalizeAttempt: (outcome = 'completed', failure) => { + if (finalized) return; + finalized = true; + finalizeAccounting?.({ + outcome, + completedAt: now(), + provider: accountingMetadata, + failure, + }); + }, + }; + results[index] = completedResult; + accountingOutcome = 'completed'; + if (!options.deferSuccessfulFinalization) + completedResult.finalizeAttempt?.(); break; } catch (error) { const providerError = sanitizedProviderError(error, job.modelId) ?? { @@ -143,11 +182,20 @@ export async function dispatchTickDecisions( metadata: undefined, }; accountingMetadata = providerError.metadata ?? accountingMetadata; - if ( - providerError.failure.code === 'cancelled' || - options.signal?.aborted - ) + accountingFailure = providerError.failure; + if (options.signal?.aborted) { + accountingFailure = cancelled().failure; + accountingOutcome = 'cancelled'; + throw cancelled(); + } + if (providerError.failure.code === 'cancelled') { + accountingOutcome = 'cancelled'; throw cancelled(); + } + accountingOutcome = + providerError.failure.code === 'timeout' + ? 'timeout' + : 'provider-error'; attempts.push({ attemptNumber: attempts.length + 1, kind, @@ -183,7 +231,13 @@ export async function dispatchTickDecisions( }; break; } finally { - finalizeAccounting?.(accountingMetadata); + if (accountingOutcome !== 'completed') + finalizeAccounting?.({ + outcome: accountingOutcome, + completedAt: now(), + provider: accountingMetadata, + failure: accountingFailure, + }); } } } @@ -196,11 +250,21 @@ export async function dispatchTickDecisions( (entry): entry is PromiseRejectedResult => entry.status === 'rejected' && isCancelled(entry.reason), ); - if (cancellation) throw cancelled(); + if (cancellation) { + for (const result of results) + if (result?.outcome === 'completed') + result.finalizeAttempt?.('completed'); + throw cancelled(); + } const unexpected = settled.find( (entry): entry is PromiseRejectedResult => entry.status === 'rejected', ); - if (unexpected) throw unexpected.reason; + if (unexpected) { + for (const result of results) + if (result?.outcome === 'completed') + result.finalizeAttempt?.('completed'); + throw unexpected.reason; + } return results; } diff --git a/packages/experiment-archive/src/archive.test.ts b/packages/experiment-archive/src/archive.test.ts index 2d3e172..5c77b83 100644 --- a/packages/experiment-archive/src/archive.test.ts +++ b/packages/experiment-archive/src/archive.test.ts @@ -728,6 +728,108 @@ describe('experiment archive', () => { ); }); + it('imports schema-v11 provider attempts independently and idempotently', async () => { + const archive = new ArchiveDatabase({ path: ':memory:' }); + const document = await currentExport(); + expect(document.schemaVersion).toBe(11); + const first = importExperimentExport(archive, document); + const second = importExperimentExport(archive, document); + expect(first.rejected).toBe(0); + expect(second.existing).toBeGreaterThan(0); + const attempts = new ExperimentQueryService(archive).providerAttempts( + document.experiment.id, + ); + expect(attempts.rows).toHaveLength(document.providerAttempts?.length ?? 0); + expect(attempts.rows[0]).toMatchObject({ + outcome: 'completed', + reservedCredits: '0.01', + }); + expect( + archive.database + .prepare( + 'SELECT typeof(reserved_credits) AS type FROM provider_attempts LIMIT 1', + ) + .get(), + ).toEqual({ type: 'text' }); + expect( + new ExperimentQueryService(archive).summary(document.experiment.id), + ).toMatchObject({ + providerAttempts: { + total: document.providerAttempts?.length, + outcomes: { completed: document.providerAttempts?.length }, + }, + }); + archive.close(); + }); + + it('advances an archived in-flight attempt once and rejects stale regression', async () => { + const archive = new ArchiveDatabase({ path: ':memory:' }); + const finalized = await currentExport(); + const sourceAttempt = finalized.providerAttempts![0]!; + const inFlight = experimentExportDocumentSchema.parse({ + ...finalized, + generatedAt: '2026-08-13T12:00:03.000Z', + providerAttempts: [ + { + id: sourceAttempt.id, + agentId: sourceAttempt.agentId, + intendedTurnNumber: sourceAttempt.intendedTurnNumber, + ...(sourceAttempt.intendedTickNumber + ? { intendedTickNumber: sourceAttempt.intendedTickNumber } + : {}), + kind: sourceAttempt.kind, + startedAt: sourceAttempt.startedAt, + outcome: 'in-flight', + modelId: sourceAttempt.modelId, + reasoningProfile: sourceAttempt.reasoningProfile, + reservedCredits: sourceAttempt.reservedCredits, + }, + ...finalized.providerAttempts!.slice(1), + ], + attemptAccounting: { + ...finalized.attemptAccounting!, + attemptsFinalized: finalized.attemptAccounting!.attemptsStarted - 1, + attemptsInFlight: 1, + knownFinalizedCostCredits: '0', + attemptsWithUnknownCost: 0, + }, + }); + importExperimentExport(archive, inFlight); + + const finalizedLater = experimentExportDocumentSchema.parse({ + ...finalized, + generatedAt: '2026-08-13T12:00:04.000Z', + }); + importExperimentExport(archive, finalizedLater); + importExperimentExport( + archive, + experimentExportDocumentSchema.parse({ + ...inFlight, + generatedAt: '2026-08-13T12:00:05.000Z', + }), + ); + expect( + archive.database + .prepare( + 'SELECT COUNT(*) AS count, outcome, actual_cost_credits AS cost FROM provider_attempts WHERE id = ?', + ) + .get(sourceAttempt.id), + ).toEqual({ + count: 1, + outcome: sourceAttempt.outcome, + cost: sourceAttempt.actualCostCredits ?? null, + }); + expect( + new ExperimentQueryService(archive).summary(finalized.experiment.id), + ).toMatchObject({ + providerAttempts: { + retention: finalized.attemptRetention, + accounting: finalized.attemptAccounting, + }, + }); + archive.close(); + }); + it('rejects credentials and private-reasoning fields from exports and notes', async () => { const archive = new ArchiveDatabase({ path: ':memory:' }); const unsafe = { diff --git a/packages/experiment-archive/src/cli.ts b/packages/experiment-archive/src/cli.ts index 6912fb8..9151295 100644 --- a/packages/experiment-archive/src/cli.ts +++ b/packages/experiment-archive/src/cli.ts @@ -191,6 +191,7 @@ const HELP = `Usage: pnpm experiment:db alliance-events [--agent id] [--from-turn n] [--to-turn n] [--reason reason] [--limit n] pnpm experiment:db patient-zero [--agent id] [--from-turn n] [--to-turn n] [--limit n] pnpm experiment:db failures [--agent id] [--reason code] [--limit n] + pnpm experiment:db provider-attempts [--agent id] [--from-turn n] [--to-turn n] [--outcome value] [--limit n] pnpm experiment:db notes import --type --status [--tag tag] [--experiment id] [--provenance text] [--supersedes note-id] pnpm experiment:db notes search [--type type] [--status status] [--tag tag] [--experiment id] [--limit n] pnpm experiment:db notes list [filters] @@ -250,6 +251,11 @@ async function main(): Promise { requirePositional(args, 1, 'experiment ID'), detailFilters(args), ); + } else if (command === 'provider-attempts') { + result = queries.providerAttempts( + requirePositional(args, 1, 'experiment ID'), + detailFilters(args), + ); } else if (command === 'notes') { const operation = requirePositional(args, 1, 'notes operation'); if (operation === 'import') { diff --git a/packages/experiment-archive/src/importer.ts b/packages/experiment-archive/src/importer.ts index 8d9b748..f56882e 100644 --- a/packages/experiment-archive/src/importer.ts +++ b/packages/experiment-archive/src/importer.ts @@ -169,8 +169,8 @@ export function importExperimentExport( first_retained_turn, last_retained_turn, dropped_records, retention_complete, requested_range_extends_beyond_retention, source_metrics_json, source_territory_json, source_alliances_json, - metric_inconsistencies_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + metric_inconsistencies_json, attempt_retention_json, attempt_accounting_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); runInsert( experimentInsert, @@ -198,6 +198,8 @@ export function importExperimentExport( json(document.currentTerritory), json(document.currentAlliances), json(sourceMetricInconsistencies(document))!, + json(document.attemptRetention), + json(document.attemptAccounting), ], report, ); @@ -219,6 +221,20 @@ export function importExperimentExport( source_territory_json = COALESCE(?, source_territory_json), source_alliances_json = COALESCE(?, source_alliances_json), simulated_player_metrics_json = COALESCE(?, simulated_player_metrics_json), + attempt_retention_json = CASE + WHEN ? IS NULL THEN attempt_retention_json + WHEN attempt_retention_json IS NULL + OR json_extract(?, '$.totalStartedAttempts') > json_extract(attempt_retention_json, '$.totalStartedAttempts') + OR (json_extract(?, '$.totalStartedAttempts') = json_extract(attempt_retention_json, '$.totalStartedAttempts') + AND json_extract(?, '$.retainedAttempts') >= json_extract(attempt_retention_json, '$.retainedAttempts')) + THEN ? ELSE attempt_retention_json END, + attempt_accounting_json = CASE + WHEN ? IS NULL THEN attempt_accounting_json + WHEN attempt_accounting_json IS NULL + OR json_extract(?, '$.attemptsStarted') > json_extract(attempt_accounting_json, '$.attemptsStarted') + OR (json_extract(?, '$.attemptsStarted') = json_extract(attempt_accounting_json, '$.attemptsStarted') + AND json_extract(?, '$.attemptsFinalized') >= json_extract(attempt_accounting_json, '$.attemptsFinalized')) + THEN ? ELSE attempt_accounting_json END, retention_limit = MAX(retention_limit, ?), total_completed_turns = MAX(total_completed_turns, ?), retained_turns = MAX(retained_turns, ?), @@ -239,6 +255,16 @@ export function importExperimentExport( json(document.currentTerritory), json(document.currentAlliances), json(document.simulatedPlayerMetrics), + json(document.attemptRetention), + json(document.attemptRetention), + json(document.attemptRetention), + json(document.attemptRetention), + json(document.attemptRetention), + json(document.attemptAccounting), + json(document.attemptAccounting), + json(document.attemptAccounting), + json(document.attemptAccounting), + json(document.attemptAccounting), document.retention.limit, document.retention.totalCompletedTurns, document.retention.retainedTurns, @@ -273,6 +299,7 @@ export function importExperimentExport( importAgents(archive, document, report); importMap(archive, document, report); importTurns(archive, document, report); + importProviderAttempts(archive, document, report); importCommunications(archive, document, report); importAllianceEvents(archive, document, report); importWorldEvents(archive, document, report); @@ -561,6 +588,76 @@ function importTurns( } } +function importProviderAttempts( + archive: ArchiveDatabase, + document: ExperimentExportDocument, + report: ImportReport, +): void { + if (document.schemaVersion !== 11) return; + const statement = archive.database.prepare(` + INSERT OR IGNORE INTO provider_attempts( + id, experiment_id, agent_id, intended_turn_number, intended_tick_number, + kind, started_at, completed_at, outcome, model_id, reasoning_profile, + provider, failure_code, failure_message, validation_codes_json, + latency_ms, prompt_tokens, completion_tokens, total_tokens, + reasoning_tokens, cached_read_tokens, cache_write_tokens, + reserved_credits, actual_cost_credits, source_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + completed_at = excluded.completed_at, + outcome = excluded.outcome, + provider = excluded.provider, + failure_code = excluded.failure_code, + failure_message = excluded.failure_message, + validation_codes_json = excluded.validation_codes_json, + latency_ms = excluded.latency_ms, + prompt_tokens = excluded.prompt_tokens, + completion_tokens = excluded.completion_tokens, + total_tokens = excluded.total_tokens, + reasoning_tokens = excluded.reasoning_tokens, + cached_read_tokens = excluded.cached_read_tokens, + cache_write_tokens = excluded.cache_write_tokens, + actual_cost_credits = excluded.actual_cost_credits, + source_json = excluded.source_json + WHERE provider_attempts.outcome = 'in-flight' + AND excluded.outcome <> 'in-flight' + `); + for (const attempt of document.providerAttempts ?? []) { + const provider = attempt.provider; + runInsert( + statement, + [ + attempt.id, + document.experiment.id, + attempt.agentId, + attempt.intendedTurnNumber, + attempt.intendedTickNumber ?? null, + attempt.kind, + attempt.startedAt, + attempt.completedAt ?? null, + attempt.outcome, + attempt.modelId, + attempt.reasoningProfile, + provider?.provider ?? null, + attempt.failure?.code ?? null, + attempt.failure?.message ?? null, + json(attempt.failure?.validationCodes), + provider?.latencyMs ?? attempt.failure?.latencyMs ?? null, + provider?.promptTokens ?? null, + provider?.completionTokens ?? null, + provider?.totalTokens ?? null, + provider?.reasoningTokens ?? null, + provider?.cachedReadTokens ?? null, + provider?.cacheWriteTokens ?? null, + attempt.reservedCredits, + attempt.actualCostCredits ?? null, + json(attempt)!, + ], + report, + ); + } +} + function movementAttribution(document: ExperimentExportDocument) { const result = new Map< number, diff --git a/packages/experiment-archive/src/migrations.ts b/packages/experiment-archive/src/migrations.ts index 9639fcb..d7b6dd0 100644 --- a/packages/experiment-archive/src/migrations.ts +++ b/packages/experiment-archive/src/migrations.ts @@ -299,4 +299,43 @@ export const migrations: readonly Migration[] = [ ON simulated_player_activity(experiment_id, tick_number, type, id); `, }, + { + version: 4, + description: 'independent safe provider-attempt ledger', + sql: ` + ALTER TABLE experiments ADD COLUMN attempt_retention_json TEXT; + ALTER TABLE experiments ADD COLUMN attempt_accounting_json TEXT; + CREATE TABLE provider_attempts ( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + agent_id TEXT NOT NULL, + intended_turn_number INTEGER NOT NULL, + intended_tick_number INTEGER, + kind TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + outcome TEXT NOT NULL, + model_id TEXT NOT NULL, + reasoning_profile TEXT NOT NULL, + provider TEXT, + failure_code TEXT, + failure_message TEXT, + validation_codes_json TEXT, + latency_ms INTEGER, + prompt_tokens INTEGER, + completion_tokens INTEGER, + total_tokens INTEGER, + reasoning_tokens INTEGER, + cached_read_tokens INTEGER, + cache_write_tokens INTEGER, + reserved_credits TEXT NOT NULL, + actual_cost_credits TEXT, + source_json TEXT NOT NULL + ) STRICT; + CREATE INDEX provider_attempts_experiment_tick_agent_idx + ON provider_attempts(experiment_id, intended_tick_number, agent_id); + CREATE INDEX provider_attempts_experiment_outcome_start_idx + ON provider_attempts(experiment_id, outcome, started_at, id); + `, + }, ] as const; diff --git a/packages/experiment-archive/src/query-service.ts b/packages/experiment-archive/src/query-service.ts index 332a1d9..508540f 100644 --- a/packages/experiment-archive/src/query-service.ts +++ b/packages/experiment-archive/src/query-service.ts @@ -76,6 +76,25 @@ function parseJson(value: unknown, fallback: T): T { } } +function addDecimalStrings(left: string, right: string): string { + const split = (value: string) => { + const [whole, fraction = ''] = value.split('.'); + return { digits: BigInt(`${whole}${fraction}`), scale: fraction.length }; + }; + const a = split(left); + const b = split(right); + const scale = Math.max(a.scale, b.scale); + const sum = + a.digits * 10n ** BigInt(scale - a.scale) + + b.digits * 10n ** BigInt(scale - b.scale); + if (sum === 0n) return '0'; + if (scale === 0) return String(sum); + const padded = String(sum).padStart(scale + 1, '0'); + return `${padded.slice(0, -scale)}.${padded.slice(-scale)}` + .replace(/0+$/u, '') + .replace(/\.$/u, ''); +} + export class ExperimentQueryService { readonly #db: DatabaseSync; @@ -276,6 +295,49 @@ export class ExperimentQueryService { return page(rows, limit); } + providerAttempts( + experimentId: string, + filters: DetailFilters = {}, + ): QueryPage> { + const limit = boundedLimit(filters.limit); + const clauses: string[] = []; + const values: Array = []; + if (filters.agent) { + clauses.push('agent_id = ?'); + values.push(filters.agent); + } + if (filters.fromTurn !== undefined) { + clauses.push('intended_turn_number >= ?'); + values.push(filters.fromTurn); + } + if (filters.toTurn !== undefined) { + clauses.push('intended_turn_number <= ?'); + values.push(filters.toTurn); + } + if (filters.outcome) { + clauses.push('outcome = ?'); + values.push(filters.outcome); + } + const rows = this.#db + .prepare( + ` + SELECT id, agent_id AS agent, intended_turn_number AS intendedTurn, + intended_tick_number AS intendedTick, kind, started_at AS startedAt, + completed_at AS completedAt, outcome, model_id AS model, + reasoning_profile AS reasoning, provider, failure_code AS failureCode, + reserved_credits AS reservedCredits, + actual_cost_credits AS actualCostCredits + FROM provider_attempts WHERE experiment_id = ? + ${clauses.map((clause) => `AND ${clause}`).join('\n')} + ORDER BY started_at ASC, id ASC LIMIT ? + `, + ) + .all(experimentId, ...values, limit + 1) as Array< + Record + >; + return page(rows, limit); + } + patientZero( experimentId: string, filters: DetailFilters = {}, @@ -428,6 +490,7 @@ export class ExperimentQueryService { .prepare('SELECT * FROM experiments WHERE id = ?') .get(experimentId) as Record | undefined; if (!experiment) throw new Error(`Unknown experiment: ${experimentId}`); + const hasIndependentAttempts = Number(experiment.schema_version) >= 11; const roster = this.#db .prepare( ` @@ -477,7 +540,35 @@ export class ExperimentQueryService { .all(experimentId); const ticks = this.#db .prepare( - ` + hasIndependentAttempts + ? ` + WITH turn_totals AS ( + SELECT tick_number AS tick, MIN(virtual_time) AS virtualTime, + MIN(tick_interval_minutes) AS intervalMinutes, + COUNT(*) AS agentRecords, + SUM(outcome = 'lost-tick') AS lostTicks, + SUM(outcome = 'lost-tick' AND json_extract(failure_json, '$.code') = 'timeout') AS deadlineMisses + FROM turns WHERE experiment_id = ? AND tick_number IS NOT NULL + GROUP BY tick_number + ), attempt_totals AS ( + SELECT intended_tick_number AS tick, COUNT(*) AS providerCallCount, + SUM(latency_ms) AS aggregateLatencyMs, + MAX(latency_ms) AS maximumLatencyMs, + ROUND(SUM(CAST(actual_cost_credits AS REAL)), 8) AS knownCostCredits, + SUM(actual_cost_credits IS NULL) AS attemptsWithUnknownCost + FROM provider_attempts + WHERE experiment_id = ? AND intended_tick_number IS NOT NULL + GROUP BY intended_tick_number + ) + SELECT t.*, COALESCE(a.providerCallCount, 0) AS providerCallCount, + COALESCE(a.knownCostCredits, 0) AS knownCostCredits, + COALESCE(a.attemptsWithUnknownCost, 0) AS attemptsWithUnknownCost, + COALESCE(a.aggregateLatencyMs, 0) AS aggregateLatencyMs, + COALESCE(a.maximumLatencyMs, 0) AS maximumLatencyMs + FROM turn_totals t LEFT JOIN attempt_totals a ON a.tick = t.tick + ORDER BY t.tick ASC LIMIT ? + ` + : ` WITH attempt_totals AS ( SELECT turn_id, COUNT(*) AS providerCalls, SUM(latency_ms) AS aggregateLatencyMs, @@ -540,7 +631,23 @@ export class ExperimentQueryService { .all(experimentId); const usageByAgent = this.#db .prepare( - ` + hasIndependentAttempts + ? ` + SELECT agent_id AS agent, + COUNT(*) AS modelAttempts, + SUM(latency_ms) AS latencyTotalMs, + SUM(latency_ms IS NOT NULL) AS attemptsWithKnownLatency, + ROUND(AVG(latency_ms), 2) AS averageLatencyMs, + SUM(prompt_tokens) AS promptTokens, + SUM(completion_tokens) AS completionTokens, + SUM(total_tokens) AS totalTokens, + ROUND(SUM(CAST(actual_cost_credits AS REAL)), 8) AS knownCostCredits, + SUM(actual_cost_credits IS NULL) AS attemptsWithUnknownCost, + SUM(CASE WHEN actual_cost_credits IS NULL THEN CAST(reserved_credits AS REAL) ELSE 0 END) AS reservedUnknownExposure + FROM provider_attempts WHERE experiment_id = ? + GROUP BY agent_id ORDER BY agent_id + ` + : ` SELECT agent_id AS agent, COUNT(*) AS modelAttempts, SUM(latency_ms) AS latencyTotalMs, @@ -557,6 +664,40 @@ export class ExperimentQueryService { ) .all(experimentId) as Array>; const usageAggregate = aggregateUsage(usageByAgent); + const independentAttemptRows = hasIndependentAttempts + ? (this.#db + .prepare( + `SELECT agent_id AS agent, outcome, reserved_credits AS reservedCredits, + actual_cost_credits AS actualCostCredits + FROM provider_attempts WHERE experiment_id = ? + ORDER BY started_at, id`, + ) + .all(experimentId) as Array>) + : []; + const attemptOutcomes = countBy(independentAttemptRows, 'outcome'); + const attemptOutcomesByAgent = [ + ...new Set(independentAttemptRows.map(({ agent }) => String(agent))), + ].map((agent) => ({ + agent, + outcomes: countBy( + independentAttemptRows.filter((row) => row.agent === agent), + 'outcome', + ), + })); + const exactKnownCostCredits = independentAttemptRows.reduce( + (sum, row) => + row.actualCostCredits === null + ? sum + : addDecimalStrings(sum, String(row.actualCostCredits)), + '0', + ); + const exactReservedUnknownExposure = independentAttemptRows.reduce( + (sum, row) => + row.actualCostCredits !== null + ? sum + : addDecimalStrings(sum, String(row.reservedCredits)), + '0', + ); const sizeTrends = this.#db .prepare( ` @@ -691,6 +832,19 @@ export class ExperimentQueryService { ), }, usage: { aggregate: usageAggregate, byAgent: usageByAgent }, + ...(hasIndependentAttempts + ? { + providerAttempts: { + total: independentAttemptRows.length, + outcomes: attemptOutcomes, + byAgent: attemptOutcomesByAgent, + exactKnownCostCredits, + exactReservedUnknownExposure, + retention: parseJson(experiment.attempt_retention_json, null), + accounting: parseJson(experiment.attempt_accounting_json, null), + }, + } + : {}), promptAndObservationSizeTrends: sizeTrends, directionChangesAfterCommunication: directions, retention: { diff --git a/packages/shared/src/index.test.ts b/packages/shared/src/index.test.ts index 4fa8286..8424060 100644 --- a/packages/shared/src/index.test.ts +++ b/packages/shared/src/index.test.ts @@ -58,6 +58,7 @@ import { memoryOperationResultSchema, createMemoryId, archiveExperimentExportResponseSchema, + providerAttemptRecordSchema, } from '.'; const agentId = '128f3f38-6b7d-4db7-9e95-751b4ce2681e'; @@ -1574,4 +1575,50 @@ describe('personality mutation contracts', () => { }).success, ).toBe(false); }); + + it('validates safe provider-attempt lifecycle records and canonical costs', () => { + const base = { + id: '018f3f38-6b7d-7db7-8e95-751b4ce2681e', + agentId, + intendedTurnNumber: 1, + intendedTickNumber: 1, + kind: 'initial', + startedAt: '2026-08-13T12:00:00.000Z', + modelId: 'deterministic-script', + reasoningProfile: 'provider-default', + reservedCredits: '0.01', + }; + expect( + providerAttemptRecordSchema.safeParse({ ...base, outcome: 'in-flight' }) + .success, + ).toBe(true); + expect( + providerAttemptRecordSchema.safeParse({ + ...base, + outcome: 'completed', + completedAt: '2026-08-13T12:00:01.000Z', + actualCostCredits: '0.00000001', + provider: { + provider: 'scripted-test', + model: 'deterministic-script', + latencyMs: 1, + costCredits: 1e-8, + }, + }).success, + ).toBe(true); + expect( + providerAttemptRecordSchema.safeParse({ + ...base, + outcome: 'cancelled', + completedAt: '2026-08-13T12:00:01.000Z', + }).success, + ).toBe(false); + expect( + providerAttemptRecordSchema.safeParse({ + ...base, + outcome: 'in-flight', + rawResponse: 'unsafe', + }).success, + ).toBe(false); + }); }); diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 1987f25..fb21f9e 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2336,6 +2336,133 @@ export const modelAttemptSchema = z.object({ }); export type ModelAttempt = z.infer; +export const providerAttemptIdSchema = z.uuid().brand<'ProviderAttemptId'>(); +export type ProviderAttemptId = z.infer; + +export const providerAttemptOutcomeSchema = z.enum([ + 'completed', + 'provider-error', + 'cancelled', + 'timeout', + 'in-flight', +]); + +function canonicalProviderCost(value: number): string { + const raw = String(value).toLowerCase(); + if (!raw.includes('e')) { + if (!raw.includes('.')) return raw; + return raw.replace(/0+$/u, '').replace(/\.$/u, ''); + } + const [mantissa, exponentText = '0'] = raw.split('e'); + const [whole, fraction = ''] = mantissa!.split('.'); + const exponent = Number(exponentText); + const digits = `${whole}${fraction}`; + const decimal = whole!.length + exponent; + const expanded = + decimal <= 0 + ? `0.${'0'.repeat(-decimal)}${digits}` + : decimal >= digits.length + ? `${digits}${'0'.repeat(decimal - digits.length)}` + : `${digits.slice(0, decimal)}.${digits.slice(decimal)}`; + if (!expanded.includes('.')) return expanded; + const trimmed = expanded.replace(/0+$/u, '').replace(/\.$/u, ''); + return trimmed === '' ? '0' : trimmed; +} + +export const providerAttemptRecordSchema = z + .object({ + id: providerAttemptIdSchema, + agentId: agentIdSchema, + intendedTurnNumber: z.number().int().positive(), + intendedTickNumber: z.number().int().positive().optional(), + kind: modelAttemptSchema.shape.kind, + startedAt: z.iso.datetime(), + completedAt: z.iso.datetime().optional(), + outcome: providerAttemptOutcomeSchema, + modelId: modelIdSchema, + reasoningProfile: reasoningProfileSchema.default('provider-default'), + provider: providerMetadataSchema.optional(), + failure: providerFailureSchema.optional(), + reservedCredits: z.string().regex(/^(?:0|[1-9]\d*)(?:\.\d+)?$/), + actualCostCredits: z + .string() + .regex(/^(?:0|[1-9]\d*)(?:\.\d+)?$/) + .optional(), + }) + .strict() + .superRefine((attempt, context) => { + const finalized = attempt.outcome !== 'in-flight'; + if (finalized !== Boolean(attempt.completedAt)) + context.addIssue({ + code: 'custom', + path: ['completedAt'], + message: 'Only finalized provider attempts require a completion time.', + }); + if ( + attempt.outcome === 'in-flight' && + (attempt.failure || attempt.actualCostCredits) + ) + context.addIssue({ + code: 'custom', + message: 'In-flight provider attempts cannot claim final results.', + }); + if (attempt.outcome === 'completed' && attempt.failure) + context.addIssue({ + code: 'custom', + path: ['failure'], + message: 'Completed provider attempts cannot contain a failure.', + }); + if ( + ['provider-error', 'cancelled', 'timeout'].includes(attempt.outcome) && + !attempt.failure + ) + context.addIssue({ + code: 'custom', + path: ['failure'], + message: 'Unsuccessful provider attempts require a safe failure.', + }); + if ( + attempt.outcome === 'cancelled' && + attempt.failure?.code !== 'cancelled' + ) + context.addIssue({ + code: 'custom', + path: ['failure', 'code'], + message: 'Cancelled attempts require a cancelled failure.', + }); + if (attempt.outcome === 'timeout' && attempt.failure?.code !== 'timeout') + context.addIssue({ + code: 'custom', + path: ['failure', 'code'], + message: 'Timed-out attempts require a timeout failure.', + }); + if ( + attempt.provider?.costCredits !== undefined && + attempt.actualCostCredits !== + canonicalProviderCost(attempt.provider.costCredits) + ) + context.addIssue({ + code: 'custom', + path: ['actualCostCredits'], + message: 'Actual cost must agree with safe provider metadata.', + }); + }); +export type ProviderAttemptRecord = z.infer; + +export const providerAttemptRetentionSchema = z + .object({ + limit: z.number().int().positive(), + totalStartedAttempts: z.number().int().nonnegative(), + retainedAttempts: z.number().int().nonnegative(), + droppedRecords: z.number().int().nonnegative(), + complete: z.boolean(), + requestedRangeExtendsBeyondRetention: z.boolean(), + }) + .strict(); +export type ProviderAttemptRetention = z.infer< + typeof providerAttemptRetentionSchema +>; + const turnRecordBaseSchema = z.object({ turnNumber: z.number().int().positive(), tickNumber: z.number().int().positive().optional(), @@ -3816,6 +3943,7 @@ export const experimentExportPreviewSchema = z.object({ matchingCommunicationCount: z.number().int().nonnegative(), matchingControlChangeCount: z.number().int().nonnegative(), matchingDiplomacyEventCount: z.number().int().nonnegative(), + matchingProviderAttemptCount: z.number().int().nonnegative().default(0), selectedAgentCount: z .number() .int() @@ -3985,7 +4113,7 @@ export type ExperimentExportWorldState = z.infer< const experimentExportDocumentObjectSchema = z .object({ - schemaVersion: z.union([z.literal(9), z.literal(10)]), + schemaVersion: z.union([z.literal(9), z.literal(10), z.literal(11)]), generatedAt: z.iso.datetime(), experiment: experimentManifestSchema, retention: experimentRetentionSchema, @@ -4000,6 +4128,7 @@ const experimentExportDocumentObjectSchema = z matchingCommunicationCount: z.number().int().nonnegative(), matchingControlChangeCount: z.number().int().nonnegative(), matchingDiplomacyEventCount: z.number().int().nonnegative(), + matchingProviderAttemptCount: z.number().int().nonnegative().optional(), matchingSimulatedPlayerEventCount: z .number() .int() @@ -4050,6 +4179,9 @@ const experimentExportDocumentObjectSchema = z controlChanges: z.array(exportedControlChangeSchema).optional(), allianceEvents: z.array(allianceEventSchema).optional(), tickSummaries: z.array(experimentTickSummarySchema).optional(), + providerAttempts: z.array(providerAttemptRecordSchema).optional(), + attemptRetention: providerAttemptRetentionSchema.optional(), + attemptAccounting: experimentAttemptAccountingSchema.optional(), turns: z.array(experimentExportTurnSchema), }) .superRefine((document, context) => { @@ -4120,7 +4252,21 @@ const experimentExportDocumentObjectSchema = z code: 'custom', message: 'Schema-v9 turns cannot claim simultaneous tick metadata.', }); - if (document.schemaVersion === 10) { + const v11TickNative = + document.schemaVersion === 11 && document.turns.some(hasTickMetadata); + if (document.schemaVersion === 11) { + const tickMetadataCount = document.turns.filter(hasTickMetadata).length; + if ( + tickMetadataCount !== 0 && + tickMetadataCount !== document.turns.length + ) + context.addIssue({ + code: 'custom', + message: + 'Schema-v11 exports cannot mix sequential and tick-native turns.', + }); + } + if (document.schemaVersion === 10 || v11TickNative) { if (document.tickSummaries === undefined) context.addIssue({ code: 'custom', @@ -4188,6 +4334,85 @@ const experimentExportDocumentObjectSchema = z 'Schema-v10 tick counts and summaries must match exported records.', }); } + if ( + document.schemaVersion === 11 && + !v11TickNative && + document.tickSummaries?.length !== 0 + ) + context.addIssue({ + code: 'custom', + message: 'Sequential schema-v11 exports require empty tick summaries.', + }); + if (document.schemaVersion === 11) { + if ( + document.providerAttempts === undefined || + document.attemptRetention === undefined || + document.attemptAccounting === undefined || + document.selection.matchingProviderAttemptCount === undefined + ) + context.addIssue({ + code: 'custom', + message: 'Schema-v11 exports require independent attempt accounting.', + }); + const attempts = document.providerAttempts ?? []; + const selectedIds = new Set(document.selection.selectedAgentIds); + const exportedIds = new Set(document.agents.map(({ id }) => id)); + if (new Set(attempts.map(({ id }) => id)).size !== attempts.length) + context.addIssue({ + code: 'custom', + path: ['providerAttempts'], + message: 'Provider-attempt IDs must be unique.', + }); + if ( + attempts.some( + ({ agentId }) => + !selectedIds.has(agentId) || !exportedIds.has(agentId), + ) + ) + context.addIssue({ + code: 'custom', + path: ['providerAttempts'], + message: 'Provider attempts must belong to selected exported agents.', + }); + if (document.selection.matchingProviderAttemptCount !== attempts.length) + context.addIssue({ + code: 'custom', + path: ['selection', 'matchingProviderAttemptCount'], + message: 'Provider-attempt selection count must match the export.', + }); + const retention = document.attemptRetention; + const accounting = document.attemptAccounting; + if ( + retention && + (retention.totalStartedAttempts !== + retention.retainedAttempts + retention.droppedRecords || + retention.retainedAttempts > retention.limit || + retention.complete !== (retention.droppedRecords === 0)) + ) + context.addIssue({ + code: 'custom', + path: ['attemptRetention'], + message: 'Provider-attempt retention totals must be consistent.', + }); + if ( + retention && + accounting && + retention.totalStartedAttempts !== accounting.attemptsStarted + ) + context.addIssue({ + code: 'custom', + path: ['attemptRetention', 'totalStartedAttempts'], + message: 'Attempt retention and accounting totals must agree.', + }); + } else if ( + document.providerAttempts !== undefined || + document.attemptRetention !== undefined || + document.attemptAccounting !== undefined + ) + context.addIssue({ + code: 'custom', + message: 'Legacy exports cannot claim schema-v11 attempt accounting.', + }); if (document.schemaVersion === 9 && document.tickSummaries !== undefined) context.addIssue({ code: 'custom', diff --git a/tests/e2e/world-lab.spec.ts b/tests/e2e/world-lab.spec.ts index 0f77939..a12235f 100644 --- a/tests/e2e/world-lab.spec.ts +++ b/tests/e2e/world-lab.spec.ts @@ -327,10 +327,16 @@ test('runs the complete deterministic World Lab browser flow', async ({ const exported = experimentExportDocumentSchema.parse( JSON.parse(await readFile(downloadedPath!, 'utf8')), ); - expect(exported.schemaVersion).toBe(10); + expect(exported.schemaVersion).toBe(11); expect(exported.filters.level).toBe('minimal'); expect(exported.selection.selectedAgentIds).toEqual([CIPHER_ID]); expect(exported.turns).toEqual([]); + expect(exported.providerAttempts).toEqual(expect.any(Array)); + expect(exported.attemptRetention).toBeDefined(); + expect(exported.attemptAccounting).toBeDefined(); + expect(exported.selection.matchingProviderAttemptCount).toBe( + exported.providerAttempts?.length, + ); expect(exported.controlChanges).toMatchObject([ { controllerAgentId: EMBER_ID,