diff --git a/README.md b/README.md index a417976..9ece867 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,9 @@ Open the World Lab at . The Game API binds to { await expect(restoreConflict.json()).resolves.toMatchObject({ error: { code: 'personality_conflict' }, }); + const exportConflict = await app.request( + '/api/simulation/experiment/export/preview', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + agents: { mode: 'all' }, + turns: { mode: 'entire-retained' }, + outcomes: ['accepted'], + actions: ['wait'], + level: 'minimal', + }), + }, + ); + expect(exportConflict.status).toBe(409); + await expect(exportConflict.json()).resolves.toMatchObject({ + error: { code: 'export_conflict' }, + }); release({ decision: { requestedAction: { type: 'wait' }, summary: 'Done.' }, metadata: { @@ -374,4 +394,99 @@ describe('game API simulation boundary', () => { }, }); }); + + it('previews and generates schema-valid retained exports through narrow endpoints', async () => { + const app = createApp({ + provider: new ScriptedAgentProvider([ + { requestedAction: { type: 'wait' }, summary: 'Wait.' }, + ]), + }); + await app.request('/api/simulation/turn', { method: 'POST' }); + const request = { + agents: { mode: 'all' }, + turns: { mode: 'entire-retained' }, + outcomes: ['accepted', 'rejected', 'provider-error'], + actions: ['move', 'infect', 'wait'], + level: 'minimal', + }; + const previewResponse = await app.request( + '/api/simulation/experiment/export/preview', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + }, + ); + expect(previewResponse.status).toBe(200); + const preview = experimentExportPreviewSchema.parse( + await previewResponse.json(), + ); + expect(preview).toMatchObject({ + matchingRecordCount: 1, + knownCostCredits: 0, + turnsWithUnknownCost: 0, + }); + const generatedResponse = await app.request( + '/api/simulation/experiment/export', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + }, + ); + const generated = experimentExportResponseSchema.parse( + await generatedResponse.json(), + ); + expect( + generated.document.turns.map(({ turnNumber }) => turnNumber), + ).toEqual([1]); + expect(JSON.stringify(generated)).not.toMatch( + /authorization|api[_-]?key|rawPrompt|rawBody|chainOfThought|privateReasoning/i, + ); + }); + + it.each([ + [{}, 400, 'invalid_export'], + [ + { + agents: { mode: 'selected', agentIds: [] }, + turns: { mode: 'entire-retained' }, + outcomes: ['accepted'], + actions: ['wait'], + level: 'minimal', + }, + 400, + 'invalid_export', + ], + [ + { + agents: { + mode: 'selected', + agentIds: ['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'], + }, + turns: { mode: 'entire-retained' }, + outcomes: ['accepted'], + actions: ['wait'], + level: 'minimal', + }, + 404, + 'unknown_agent', + ], + ])( + 'returns typed safe export validation failures', + async (body, status, code) => { + const app = createApp({ + provider: new ScriptedAgentProvider([ + { requestedAction: { type: 'wait' }, summary: 'Wait.' }, + ]), + }); + const response = await app.request('/api/simulation/experiment/export', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + expect(response.status).toBe(status); + await expect(response.json()).resolves.toMatchObject({ error: { code } }); + }, + ); }); diff --git a/apps/game-api/src/app.ts b/apps/game-api/src/app.ts index 28b3902..aa76291 100644 --- a/apps/game-api/src/app.ts +++ b/apps/game-api/src/app.ts @@ -1,4 +1,4 @@ -import { Hono } from 'hono'; +import { Hono, type Context } from 'hono'; import { cors } from 'hono/cors'; import { BrowserTestAgentProvider, @@ -8,6 +8,9 @@ import { } from '@agentborne/agent-runtime'; import { apiErrorSchema, + experimentExportRequestSchema, + experimentExportPreviewSchema, + experimentExportResponseSchema, PERSONALITY_MAX_LENGTH, resetSimulationResponseSchema, restoreDefaultPersonalitiesResponseSchema, @@ -23,6 +26,7 @@ import { SimulationService, SimulationValidationError, } from './simulation-service'; +import { ExperimentExportValidationError } from './experiment-export'; export const healthResponseSchema = worldSnapshotSchema .pick({ generatedAt: true }) @@ -187,6 +191,56 @@ export function createApp(options: AppOptions = {}) { } }); + app.post('/api/simulation/experiment/export/preview', async (context) => { + const request = experimentExportRequestSchema.safeParse( + await context.req.json().catch(() => undefined), + ); + if (!request.success) + return context.json( + apiErrorSchema.parse({ + error: { + code: 'invalid_export', + message: 'The export filters are invalid.', + }, + }), + 400, + ); + try { + return context.json( + experimentExportPreviewSchema.parse( + service.previewExperimentExport(request.data), + ), + ); + } catch (error) { + return exportErrorResponse(context, error); + } + }); + + app.post('/api/simulation/experiment/export', async (context) => { + const request = experimentExportRequestSchema.safeParse( + await context.req.json().catch(() => undefined), + ); + if (!request.success) + return context.json( + apiErrorSchema.parse({ + error: { + code: 'invalid_export', + message: 'The export filters are invalid.', + }, + }), + 400, + ); + try { + return context.json( + experimentExportResponseSchema.parse({ + document: service.generateExperimentExport(request.data), + }), + ); + } catch (error) { + return exportErrorResponse(context, error); + } + }); + app.notFound((context) => context.json( apiErrorSchema.parse({ @@ -219,3 +273,21 @@ export function createApp(options: AppOptions = {}) { } export type GameApi = ReturnType; + +function exportErrorResponse(context: Context, error: unknown) { + if (error instanceof SimulationConflictError) + return context.json( + apiErrorSchema.parse({ + error: { code: 'export_conflict', message: error.message }, + }), + 409, + ); + if (error instanceof ExperimentExportValidationError) + return context.json( + apiErrorSchema.parse({ + error: { code: error.code, message: error.message }, + }), + error.code === 'unknown_agent' ? 404 : 400, + ); + throw error; +} diff --git a/apps/game-api/src/experiment-export.ts b/apps/game-api/src/experiment-export.ts new file mode 100644 index 0000000..f4f362f --- /dev/null +++ b/apps/game-api/src/experiment-export.ts @@ -0,0 +1,624 @@ +import { + experimentExportDocumentSchema, + experimentExportPreviewSchema, + experimentExportRequestSchema, + experimentMetricsSchema, + type Agent, + type AgentId, + type AgentObservation, + type AgentTurnRecord, + type ExperimentExportDocument, + type ExperimentExportPreview, + type ExperimentExportRequest, + type ExperimentExportWorldState, + type ExperimentId, + type ExperimentMetrics, + type PersonalityConfigurationEvent, + type ProviderMetadata, + type WorldSnapshot, +} from '@agentborne/shared'; + +export interface ExperimentSource { + id: ExperimentId; + startedAt: string; + providerMode: 'openrouter' | 'scripted-test'; + retentionLimit: number; + totalCompletedTurns: number; + turns: readonly AgentTurnRecord[]; + initialAgents: readonly Agent[]; + currentAgents: readonly Agent[]; + configurationEvents: readonly PersonalityConfigurationEvent[]; + initialWorld: WorldSnapshot; + currentWorld: WorldSnapshot; +} + +export class ExperimentExportValidationError extends Error { + constructor( + readonly code: 'invalid_export' | 'unknown_agent' | 'records_unavailable', + message: string, + ) { + super(message); + this.name = 'ExperimentExportValidationError'; + } +} + +export class ExperimentMetricAccumulator { + readonly #records = new Map(); + + constructor(agentIds: readonly AgentId[]) { + this.#records.set('aggregate', mutableMetrics()); + for (const agentId of agentIds) + this.#records.set(agentId, mutableMetrics()); + } + + add(turn: AgentTurnRecord): void { + addToMutable(this.#records.get('aggregate')!, turn); + const agent = this.#records.get(turn.agentId); + if (agent) addToMutable(agent, turn); + } + + snapshot(agentIds: readonly AgentId[]): ExperimentMetrics { + return experimentMetricsSchema.parse({ + aggregate: finalizeMutable(this.#records.get('aggregate')!), + byAgent: agentIds.map((agentId) => ({ + agentId, + metrics: finalizeMutable( + this.#records.get(agentId) ?? mutableMetrics(), + ), + })), + }); + } +} + +const metricTokenFields = [ + 'promptTokens', + 'completionTokens', + 'totalTokens', + 'reasoningTokens', + 'cachedReadTokens', + 'cacheWriteTokens', +] as const; + +interface MutableMetrics { + turns: number; + accepted: number; + rejected: number; + providerErrors: number; + requestedMoves: number; + requestedInfections: number; + requestedWaits: number; + acceptedMovements: number; + infections: number; + latencyTotal: number; + latencyCount: number; + tokens: Record<(typeof metricTokenFields)[number], number>; + tokenFieldsComplete: Record<(typeof metricTokenFields)[number], boolean>; + knownCostCredits: string; + unknownCost: number; + visited: Set; +} + +function mutableMetrics(): MutableMetrics { + return { + turns: 0, + accepted: 0, + rejected: 0, + providerErrors: 0, + requestedMoves: 0, + requestedInfections: 0, + requestedWaits: 0, + acceptedMovements: 0, + infections: 0, + latencyTotal: 0, + latencyCount: 0, + tokens: Object.fromEntries( + metricTokenFields.map((field) => [field, 0]), + ) as MutableMetrics['tokens'], + tokenFieldsComplete: Object.fromEntries( + metricTokenFields.map((field) => [field, true]), + ) as MutableMetrics['tokenFieldsComplete'], + knownCostCredits: '0', + unknownCost: 0, + visited: new Set(), + }; +} + +function addToMutable(metrics: MutableMetrics, turn: AgentTurnRecord): void { + metrics.turns += 1; + metrics[ + turn.outcome === 'provider-error' ? 'providerErrors' : turn.outcome + ] += 1; + metrics.visited.add(turn.observation.currentCell.cell); + if (turn.outcome !== 'provider-error') { + if (turn.requestedAction.type === 'move') metrics.requestedMoves += 1; + if (turn.requestedAction.type === 'infect') + metrics.requestedInfections += 1; + if (turn.requestedAction.type === 'wait') metrics.requestedWaits += 1; + } + if (turn.outcome === 'accepted' && turn.event.type === 'agent-moved') { + metrics.acceptedMovements += 1; + metrics.visited.add(turn.event.toCell); + } + if (turn.outcome === 'accepted' && turn.event.type === 'hex-infected') + metrics.infections += 1; + if (turn.provider) { + metrics.latencyTotal += turn.provider.latencyMs; + metrics.latencyCount += 1; + } + for (const field of metricTokenFields) { + const value = turn.provider?.[field]; + if (value === undefined) metrics.tokenFieldsComplete[field] = false; + else metrics.tokens[field] += value; + } + if (turn.provider?.costCredits === undefined) metrics.unknownCost += 1; + else + metrics.knownCostCredits = addDecimalValue( + metrics.knownCostCredits, + turn.provider.costCredits, + ); +} + +function finalizeMutable(metrics: MutableMetrics) { + const tokens: Record = {}; + if (metrics.turns > 0) { + for (const field of metricTokenFields) + if (metrics.tokenFieldsComplete[field]) + tokens[field] = metrics.tokens[field]; + } + return { + totalTurns: metrics.turns, + accepted: metrics.accepted, + rejected: metrics.rejected, + providerErrors: metrics.providerErrors, + requestedMoves: metrics.requestedMoves, + requestedInfections: metrics.requestedInfections, + requestedWaits: metrics.requestedWaits, + acceptedMovements: metrics.acceptedMovements, + successfullyInfectedCells: metrics.infections, + uniqueVisitedCells: metrics.visited.size, + ...(metrics.latencyCount > 0 + ? { averageLatencyMs: metrics.latencyTotal / metrics.latencyCount } + : {}), + tokens, + knownCostCredits: Number(metrics.knownCostCredits), + turnsWithUnknownCost: metrics.unknownCost, + }; +} + +export function createExperimentExport( + source: ExperimentSource, + requestInput: unknown, + generatedAt: string, +): ExperimentExportDocument { + const parsed = experimentExportRequestSchema.safeParse(requestInput); + if (!parsed.success) { + throw new ExperimentExportValidationError( + 'invalid_export', + 'The export filters are invalid.', + ); + } + const request = parsed.data; + const selectedAgentIds = resolveAgentIds(source, request); + const selectedSet = new Set(selectedAgentIds); + const filtered = filterTurns(source, request, selectedSet); + const firstRetainedTurn = source.turns[0]?.turnNumber; + const lastRetainedTurn = source.turns.at(-1)?.turnNumber; + const requestedRangeExtendsBeyondRetention = rangeExtendsBeyondRetention( + request, + source, + firstRetainedTurn, + lastRetainedTurn, + ); + const droppedRecords = source.totalCompletedTurns - source.turns.length; + const retention = { + limit: source.retentionLimit, + totalCompletedTurns: source.totalCompletedTurns, + retainedTurns: source.turns.length, + firstRetainedTurn, + lastRetainedTurn, + droppedRecords, + complete: droppedRecords === 0, + requestedRangeExtendsBeyondRetention, + }; + const include = inclusionsFor(request); + const selectedAgents = source.currentAgents + .filter(({ id }) => selectedSet.has(id)) + .map((agent) => + include.personality + ? structuredClone(agent) + : { + id: agent.id, + name: agent.name, + color: agent.color, + currentCell: agent.currentCell, + }, + ); + const document: ExperimentExportDocument = { + schemaVersion: 1, + generatedAt, + experiment: { + id: source.id, + startedAt: source.startedAt, + providerMode: source.providerMode, + ...(request.level === 'full-safe' + ? { initialAgents: structuredClone([...source.initialAgents]) } + : {}), + }, + retention, + filters: structuredClone(request), + selection: { + selectedAgentIds, + matchingRecordCount: filtered.length, + firstMatchingTurn: filtered[0]?.turnNumber, + lastMatchingTurn: filtered.at(-1)?.turnNumber, + }, + agents: selectedAgents, + ...(include.personalityHistory + ? { + configurationEvents: source.configurationEvents + .filter(({ agentId }) => selectedSet.has(agentId)) + .map((event) => structuredClone(event)), + } + : {}), + ...(include.metrics + ? { metrics: calculateExperimentMetrics(filtered, selectedAgentIds) } + : {}), + ...(include.initialWorld + ? { initialWorld: exportWorldState(source.initialWorld) } + : {}), + ...(include.currentWorld + ? { currentWorld: exportWorldState(source.currentWorld) } + : {}), + ...(request.level === 'full-safe' + ? { + worldEvents: filtered + .filter( + ( + turn, + ): turn is Extract => + turn.outcome === 'accepted', + ) + .map(({ event }) => structuredClone(event)), + } + : {}), + turns: filtered.map((turn) => exportTurn(turn, request)), + }; + return experimentExportDocumentSchema.parse(document); +} + +function exportWorldState(world: WorldSnapshot): ExperimentExportWorldState { + return { + generatedAt: world.generatedAt, + hexes: structuredClone(world.hexes), + agents: structuredClone(world.agents), + }; +} + +export function createExperimentPreview( + source: ExperimentSource, + request: unknown, + generatedAt: string, +): ExperimentExportPreview { + const document = createExperimentExport(source, request, generatedAt); + const serialized = serializeExperimentExport(document); + const serializedUtf8Bytes = new TextEncoder().encode(serialized).byteLength; + const metrics = calculateExperimentMetrics( + filterTurns( + source, + document.filters, + new Set(document.selection.selectedAgentIds), + ), + document.selection.selectedAgentIds, + ); + return experimentExportPreviewSchema.parse({ + experimentId: source.id, + matchingRecordCount: document.selection.matchingRecordCount, + selectedAgentCount: document.selection.selectedAgentIds.length, + firstMatchingTurn: document.selection.firstMatchingTurn, + lastMatchingTurn: document.selection.lastMatchingTurn, + retention: document.retention, + knownCostCredits: metrics.aggregate.knownCostCredits, + turnsWithUnknownCost: metrics.aggregate.turnsWithUnknownCost, + serializedUtf8Bytes, + approximateAiInputTokens: Math.ceil(serializedUtf8Bytes / 4), + tokenEstimateMethod: 'ceil(UTF-8 bytes / 4)', + }); +} + +function resolveAgentIds( + source: ExperimentSource, + request: ExperimentExportRequest, +): AgentId[] { + const known = new Set(source.currentAgents.map(({ id }) => id)); + const selected = + request.agents.mode === 'all' + ? source.currentAgents.map(({ id }) => id) + : request.agents.agentIds; + if (selected.some((id) => !known.has(id))) { + throw new ExperimentExportValidationError( + 'unknown_agent', + 'One or more selected agents do not exist.', + ); + } + return [...selected]; +} + +function filterTurns( + source: ExperimentSource, + request: ExperimentExportRequest, + selected: Set, +): AgentTurnRecord[] { + let turns = source.turns.filter( + (turn) => + selected.has(turn.agentId) && + request.outcomes.includes(turn.outcome) && + (turn.outcome === 'provider-error' || + request.actions.includes(turn.requestedAction.type)), + ); + if (request.turns.mode === 'range') { + const range = request.turns; + turns = turns.filter( + ({ turnNumber }) => + turnNumber >= range.fromTurn && turnNumber <= range.toTurn, + ); + } else if (request.turns.mode === 'latest') { + turns = turns.slice(-request.turns.count); + } + return turns.map((turn) => structuredClone(turn)); +} + +function rangeExtendsBeyondRetention( + request: ExperimentExportRequest, + source: ExperimentSource, + first?: number, + last?: number, +): boolean { + if (request.turns.mode === 'entire-retained') + return source.totalCompletedTurns > source.turns.length; + if (request.turns.mode !== 'range') return false; + if (!first || !last) return true; + return request.turns.fromTurn < first || request.turns.toTurn > last; +} + +function inclusionsFor(request: ExperimentExportRequest) { + if (request.level === 'full-safe') + return { + personality: true, + personalityHistory: true, + metrics: true, + initialWorld: true, + currentWorld: true, + }; + if (request.level === 'custom') { + const custom = request.custom!; + return { + personality: custom.personalityTextHistory, + personalityHistory: custom.personalityTextHistory, + metrics: custom.computedMetrics, + initialWorld: custom.initialWorldState, + currentWorld: custom.currentWorldState, + }; + } + return { + personality: true, + personalityHistory: false, + metrics: true, + initialWorld: false, + currentWorld: false, + }; +} + +function compactProvider(provider: ProviderMetadata): ProviderMetadata { + return { + provider: provider.provider, + model: provider.model, + latencyMs: provider.latencyMs, + ...(provider.promptTokens === undefined + ? {} + : { promptTokens: provider.promptTokens }), + ...(provider.completionTokens === undefined + ? {} + : { completionTokens: provider.completionTokens }), + ...(provider.totalTokens === undefined + ? {} + : { totalTokens: provider.totalTokens }), + ...(provider.reasoningTokens === undefined + ? {} + : { reasoningTokens: provider.reasoningTokens }), + ...(provider.cachedReadTokens === undefined + ? {} + : { cachedReadTokens: provider.cachedReadTokens }), + ...(provider.cacheWriteTokens === undefined + ? {} + : { cacheWriteTokens: provider.cacheWriteTokens }), + ...(provider.costCredits === undefined + ? {} + : { costCredits: provider.costCredits }), + }; +} + +function exportTurn( + turn: AgentTurnRecord, + request: ExperimentExportRequest, +): ExperimentExportDocument['turns'][number] { + const standard = request.level === 'standard'; + const full = request.level === 'full-safe'; + const custom = request.level === 'custom' ? request.custom! : undefined; + const includeObservation = full || standard || custom?.turnObservations; + const includePersonality = full || standard || custom?.personalityTextHistory; + const includeValidation = full || standard || custom?.validationDetails; + const includeEvent = full || standard || custom?.resultingEvents; + const includeProvider = + request.level !== 'custom' || Boolean(custom?.providerUsageMetadata); + const base: ExperimentExportDocument['turns'][number] = { + turnNumber: turn.turnNumber, + startedAt: turn.startedAt, + completedAt: turn.completedAt, + agentId: turn.agentId, + outcome: turn.outcome, + ...(turn.outcome === 'provider-error' + ? { failure: structuredClone(turn.failure) } + : { + requestedAction: structuredClone(turn.requestedAction), + summary: turn.summary, + ...(turn.outcome === 'rejected' + ? { validationReason: turn.validation.reason } + : { eventSummary: summarizeEvent(turn.event) }), + }), + }; + if (includePersonality) base.personality = turn.observation.personality; + if (includeObservation) { + const observation: Partial = structuredClone( + turn.observation, + ); + if (!includePersonality) delete observation.personality; + if (custom && !custom.nearbyAgents) delete observation.nearbyAgents; + if (custom && !custom.recentEvents) delete observation.recentEvents; + base.observation = observation; + } + if (turn.outcome !== 'provider-error' && includeValidation) + base.validation = structuredClone(turn.validation); + if (turn.outcome === 'accepted' && includeEvent) + base.event = structuredClone(turn.event); + if (includeProvider && turn.provider) + base.provider = full + ? structuredClone(turn.provider) + : compactProvider(turn.provider); + return base; +} + +function summarizeEvent( + event: Extract['event'], +): string { + if (event.type === 'agent-moved') + return `Moved from ${event.fromCell} to ${event.toCell}.`; + if (event.type === 'hex-infected') return `Infected ${event.cell}.`; + return 'Waited.'; +} + +export function calculateExperimentMetrics( + turns: readonly AgentTurnRecord[], + agentIds: readonly AgentId[], +): ExperimentMetrics { + const metricFor = (records: readonly AgentTurnRecord[]) => { + const latencies = records.flatMap(({ provider }) => + provider ? [provider.latencyMs] : [], + ); + const tokens: Record = {}; + for (const field of metricTokenFields) { + const known = records + .map(({ provider }) => provider?.[field]) + .filter((value): value is number => value !== undefined); + if (known.length === records.length && known.length > 0) + tokens[field] = known.reduce((sum, value) => sum + value, 0); + } + const visited = new Set(); + for (const turn of records) { + visited.add(turn.observation.currentCell.cell); + if (turn.outcome === 'accepted' && turn.event.type === 'agent-moved') + visited.add(turn.event.toCell); + } + const costs = records.flatMap(({ provider }) => + provider?.costCredits === undefined ? [] : [provider.costCredits], + ); + return { + totalTurns: records.length, + accepted: records.filter(({ outcome }) => outcome === 'accepted').length, + rejected: records.filter(({ outcome }) => outcome === 'rejected').length, + providerErrors: records.filter( + ({ outcome }) => outcome === 'provider-error', + ).length, + requestedMoves: records.filter( + (turn) => + turn.outcome !== 'provider-error' && + turn.requestedAction.type === 'move', + ).length, + requestedInfections: records.filter( + (turn) => + turn.outcome !== 'provider-error' && + turn.requestedAction.type === 'infect', + ).length, + requestedWaits: records.filter( + (turn) => + turn.outcome !== 'provider-error' && + turn.requestedAction.type === 'wait', + ).length, + acceptedMovements: records.filter( + (turn) => + turn.outcome === 'accepted' && turn.event.type === 'agent-moved', + ).length, + successfullyInfectedCells: records.filter( + (turn) => + turn.outcome === 'accepted' && turn.event.type === 'hex-infected', + ).length, + uniqueVisitedCells: visited.size, + ...(latencies.length > 0 + ? { + averageLatencyMs: + latencies.reduce((sum, value) => sum + value, 0) / + latencies.length, + } + : {}), + tokens, + knownCostCredits: sumDecimalNumbers(costs), + turnsWithUnknownCost: records.filter( + ({ provider }) => provider?.costCredits === undefined, + ).length, + }; + }; + return experimentMetricsSchema.parse({ + aggregate: metricFor(turns), + byAgent: agentIds.map((agentId) => ({ + agentId, + metrics: metricFor(turns.filter((turn) => turn.agentId === agentId)), + })), + }); +} + +export function serializeExperimentExport( + document: ExperimentExportDocument, +): string { + return document.filters.serialization === 'pretty' + ? JSON.stringify(document, null, 2) + : JSON.stringify(document); +} + +function addDecimalValue(left: string, right: number): string { + const leftParts = decimalParts(left); + const rightParts = decimalParts(right); + const scale = Math.max(leftParts.scale, rightParts.scale); + const leftInteger = + leftParts.integer * 10n ** BigInt(scale - leftParts.scale); + const rightInteger = + rightParts.integer * 10n ** BigInt(scale - rightParts.scale); + return decimalString(leftInteger + rightInteger, scale); +} + +function sumDecimalNumbers(values: readonly number[]): number { + return Number(values.reduce(addDecimalValue, '0')); +} + +function decimalParts(value: number | string): { + integer: bigint; + scale: number; +} { + const [mantissa, exponentText = '0'] = value + .toString() + .toLowerCase() + .split('e'); + const exponent = Number(exponentText); + const [whole, fraction = ''] = mantissa!.split('.'); + let integer = BigInt(`${whole}${fraction}`); + let scale = fraction.length - exponent; + if (scale < 0) { + integer *= 10n ** BigInt(-scale); + scale = 0; + } + return { integer, scale }; +} + +function decimalString(integer: bigint, scale: number): string { + if (scale === 0) return integer.toString(); + const digits = integer.toString().padStart(scale + 1, '0'); + return `${digits.slice(0, -scale)}.${digits.slice(-scale)}`; +} diff --git a/apps/game-api/src/simulation-service.test.ts b/apps/game-api/src/simulation-service.test.ts index f136c78..bc6ad43 100644 --- a/apps/game-api/src/simulation-service.test.ts +++ b/apps/game-api/src/simulation-service.test.ts @@ -18,6 +18,7 @@ import { SimulationService, SimulationValidationError, } from './simulation-service'; +import { serializeExperimentExport } from './experiment-export'; const now = () => '2026-08-13T12:00:01.000Z'; const createEventId = () => '67aa21b9-fc78-4b04-9f92-9862bf346f96'; @@ -26,6 +27,16 @@ function service(provider: AgentProvider) { return new SimulationService({ provider, now, createEventId }); } +function exportRequest(level: 'minimal' | 'standard' | 'full-safe' | 'custom') { + return { + agents: { mode: 'all' as const }, + turns: { mode: 'entire-retained' as const }, + outcomes: ['accepted', 'rejected', 'provider-error'] as const, + actions: ['move', 'infect', 'wait'] as const, + level, + }; +} + describe('SimulationService', () => { it('resets to the exact deterministic six-agent starting world', async () => { const simulation = service( @@ -35,11 +46,380 @@ describe('SimulationService', () => { ); const initial = simulation.getSnapshot(); await simulation.executeNextTurn(); - expect(simulation.reset()).toEqual(initial); + const reset = simulation.reset(); + expect(reset.world).toEqual(initial.world); + expect(reset.experiment.id).not.toBe(initial.experiment.id); + expect(reset.experiment.totalCompletedTurns).toBe(0); expect(initial.world.agents).toHaveLength(6); expect(initial.world.hexes).toHaveLength(61); }); + it('retains complete experiment records independently of the 120-turn browser snapshot', async () => { + const provider: AgentProvider = { + mode: 'scripted-test', + model: 'retention-test', + configured: true, + async decide() { + return { + decision: { + requestedAction: { type: 'wait' as const }, + summary: 'Wait.', + }, + metadata: { + provider: 'scripted-test' as const, + model: 'retention-test', + latencyMs: 1, + costCredits: 0, + }, + }; + }, + }; + const simulation = new SimulationService({ + provider, + now, + createEventId, + experimentRetentionLimit: 125, + }); + for (let index = 0; index < 125; index += 1) + await simulation.executeNextTurn(); + const snapshot = simulation.getSnapshot(); + expect(snapshot.turns).toHaveLength(120); + expect(snapshot.experiment).toMatchObject({ + totalCompletedTurns: 125, + retainedTurns: 125, + droppedRecords: 0, + complete: true, + }); + expect( + simulation.generateExperimentExport(exportRequest('full-safe')).turns, + ).toHaveLength(125); + }); + + it('reports configurable experiment truncation and absolute retained bounds', async () => { + const simulation = new SimulationService({ + provider: new ScriptedAgentProvider([ + { requestedAction: { type: 'wait' }, summary: '1' }, + { requestedAction: { type: 'wait' }, summary: '2' }, + { requestedAction: { type: 'wait' }, summary: '3' }, + ]), + now, + createEventId, + experimentRetentionLimit: 2, + }); + await simulation.executeNextTurn(); + await simulation.executeNextTurn(); + await simulation.executeNextTurn(); + expect(simulation.getSnapshot().experiment).toMatchObject({ + retainedTurns: 2, + firstRetainedTurn: 2, + lastRetainedTurn: 3, + droppedRecords: 1, + complete: false, + }); + const preview = simulation.previewExperimentExport({ + ...exportRequest('minimal'), + turns: { mode: 'range', fromTurn: 1, toTurn: 3 }, + }); + expect(preview.retention.requestedRangeExtendsBeyondRetention).toBe(true); + }); + + it('estimates the selected Compact or Pretty serialization and defaults to Compact', async () => { + const simulation = service( + new ScriptedAgentProvider([ + { requestedAction: { type: 'wait' }, summary: 'Wait.' }, + ]), + ); + await simulation.executeNextTurn(); + const compact = simulation.previewExperimentExport( + exportRequest('minimal'), + ); + const pretty = simulation.previewExperimentExport({ + ...exportRequest('minimal'), + serialization: 'pretty', + }); + expect(compact.serializedUtf8Bytes).toBeLessThan( + pretty.serializedUtf8Bytes, + ); + expect(compact.approximateAiInputTokens).toBeLessThan( + pretty.approximateAiInputTokens, + ); + expect( + simulation.generateExperimentExport(exportRequest('minimal')).filters + .serialization, + ).toBe('compact'); + }); + + it('aggregates charged cost as exact decimal input without JSON artifacts', async () => { + const provider: AgentProvider = { + mode: 'scripted-test', + model: 'decimal-cost-test', + configured: true, + async decide() { + return { + decision: { + requestedAction: { type: 'wait' as const }, + summary: 'Wait.', + }, + metadata: { + provider: 'scripted-test' as const, + model: 'decimal-cost-test', + latencyMs: 0, + costCredits: 0.14064472125, + }, + }; + }, + }; + const simulation = service(provider); + await simulation.executeNextTurn(); + await simulation.executeNextTurn(); + await simulation.executeNextTurn(); + const document = simulation.generateExperimentExport( + exportRequest('minimal'), + ); + expect(document.metrics?.aggregate.knownCostCredits).toBe(0.42193416375); + expect(serializeExperimentExport(document)).toContain( + '"knownCostCredits":0.42193416375', + ); + expect(serializeExperimentExport(document)).not.toContain( + '0.4219341637499998', + ); + }); + + it('records immutable personality configuration history and clears it on reset', () => { + let sequence = 0; + const simulation = new SimulationService({ + provider: new ScriptedAgentProvider([ + { requestedAction: { type: 'wait' }, summary: 'Wait.' }, + ]), + now, + createEventId, + createExperimentId: () => + `aaaaaaaa-aaaa-4aaa-8aaa-${String(++sequence).padStart(12, '0')}`, + }); + const before = simulation.getSnapshot(); + const agent = before.world.agents[0]!; + simulation.updateAgentPersonality(agent.id, 'Custom immutable edit.'); + simulation.restoreDefaultPersonalities(); + const full = simulation.generateExperimentExport( + exportRequest('full-safe'), + ); + expect(full.configurationEvents).toMatchObject([ + { + operation: 'custom-edit', + previousPersonality: agent.personality, + newPersonality: 'Custom immutable edit.', + }, + { + operation: 'restore-default', + previousPersonality: 'Custom immutable edit.', + newPersonality: agent.personality, + }, + ]); + const captured = structuredClone(full.configurationEvents); + simulation.updateAgentPersonality(agent.id, 'Another edit.'); + expect(full.configurationEvents).toEqual(captured); + const reset = simulation.reset(); + expect(reset.experiment.id).not.toBe(before.experiment.id); + expect( + simulation.generateExperimentExport(exportRequest('full-safe')) + .configurationEvents, + ).toEqual([]); + }); + + it('filters agents, latest/ranges, outcomes and actions chronologically with subset metrics', async () => { + let call = 0; + const provider: AgentProvider = { + mode: 'scripted-test', + model: 'filter-test', + configured: true, + async decide(observation) { + call += 1; + const requestedAction = + call === 1 + ? { type: 'infect' as const } + : call === 2 + ? { + type: 'move' as const, + targetCell: observation.adjacentCells[0]!.cell, + } + : { type: 'wait' as const }; + return { + decision: { requestedAction, summary: 'Safe summary.' }, + metadata: { + provider: 'scripted-test', + model: 'filter-test', + latencyMs: call, + promptTokens: call, + completionTokens: call, + totalTokens: call * 2, + costCredits: call === 3 ? undefined : 0.00000001, + }, + }; + }, + }; + const simulation = service(provider); + await simulation.executeNextTurn(); + await simulation.executeNextTurn(); + await simulation.executeNextTurn(); + const agents = simulation.getSnapshot().world.agents; + const selected = [agents[0]!.id, agents[1]!.id]; + const document = simulation.generateExperimentExport({ + ...exportRequest('standard'), + agents: { mode: 'selected', agentIds: selected }, + turns: { mode: 'latest', count: 10 }, + outcomes: ['accepted'], + actions: ['move', 'infect'], + }); + expect(document.selection.selectedAgentIds).toEqual(selected); + expect(document.turns.map(({ turnNumber }) => turnNumber)).toEqual([1, 2]); + expect(document.metrics?.aggregate).toMatchObject({ + totalTurns: 2, + requestedMoves: 1, + requestedInfections: 1, + acceptedMovements: 1, + successfullyInfectedCells: 1, + knownCostCredits: 0.00000002, + turnsWithUnknownCost: 0, + }); + expect(document.metrics?.aggregate.uniqueVisitedCells).toBeGreaterThan(1); + const oneAgent = simulation.generateExperimentExport({ + ...exportRequest('minimal'), + agents: { mode: 'selected', agentIds: [agents[2]!.id] }, + turns: { mode: 'range', fromTurn: 3, toTurn: 3 }, + }); + expect(oneAgent.selection).toMatchObject({ + selectedAgentIds: [agents[2]!.id], + matchingRecordCount: 1, + firstMatchingTurn: 3, + lastMatchingTurn: 3, + }); + expect( + simulation.generateExperimentExport(exportRequest('minimal')).metrics + ?.aggregate, + ).toMatchObject({ + knownCostCredits: 0.00000002, + turnsWithUnknownCost: 1, + }); + }); + + it('keeps Full safe world snapshots state-only and scopes canonical events to the export selection', async () => { + const provider: AgentProvider = { + mode: 'scripted-test', + model: 'full-safe-event-scope-test', + configured: true, + async decide() { + return { + decision: { + requestedAction: { type: 'wait' as const }, + summary: 'Wait.', + }, + metadata: { + provider: 'scripted-test' as const, + model: 'full-safe-event-scope-test', + latencyMs: 0, + costCredits: 0, + }, + }; + }, + }; + const simulation = service(provider); + for (let index = 0; index < 7; index += 1) + await simulation.executeNextTurn(); + + const agents = simulation.getSnapshot().world.agents; + const selectedAgent = agents[0]!; + const oneAgent = simulation.generateExperimentExport({ + ...exportRequest('full-safe'), + agents: { mode: 'selected', agentIds: [selectedAgent.id] }, + turns: { mode: 'range', fromTurn: 1, toTurn: 6 }, + outcomes: ['accepted'], + actions: ['wait'], + }); + + expect(oneAgent.initialWorld?.agents).toHaveLength(6); + expect(oneAgent.initialWorld?.hexes).toHaveLength(61); + expect(oneAgent.currentWorld?.agents).toHaveLength(6); + expect(oneAgent.currentWorld?.hexes).toHaveLength(61); + expect(oneAgent.initialWorld).not.toHaveProperty('events'); + expect(oneAgent.currentWorld).not.toHaveProperty('events'); + expect(oneAgent.worldEvents).toHaveLength(1); + expect( + oneAgent.worldEvents?.every( + ({ agentId }) => agentId === selectedAgent.id, + ), + ).toBe(true); + expect(oneAgent.turns.map(({ turnNumber }) => turnNumber)).toEqual([1]); + + const allAgents = simulation.generateExperimentExport( + exportRequest('full-safe'), + ); + expect(allAgents.selection.selectedAgentIds).toEqual( + agents.map(({ id }) => id), + ); + expect(allAgents.turns).toHaveLength(7); + expect(allAgents.worldEvents).toHaveLength(7); + expect(allAgents.initialWorld).not.toHaveProperty('events'); + expect(allAgents.currentWorld).not.toHaveProperty('events'); + }); + + it('produces predictable Minimal, Standard, Full safe and Custom omissions without mutation', async () => { + const simulation = service( + new ScriptedAgentProvider([ + { requestedAction: { type: 'wait' }, summary: 'Wait.' }, + ]), + ); + await simulation.executeNextTurn(); + const before = simulation.getSnapshot(); + expect(() => + simulation.generateExperimentExport({ + ...exportRequest('minimal'), + outcomes: [], + }), + ).toThrow(/invalid/i); + expect(simulation.getSnapshot()).toEqual(before); + const minimal = simulation.generateExperimentExport( + exportRequest('minimal'), + ); + const standard = simulation.generateExperimentExport( + exportRequest('standard'), + ); + const full = simulation.generateExperimentExport( + exportRequest('full-safe'), + ); + const custom = simulation.generateExperimentExport({ + ...exportRequest('custom'), + custom: { + turnObservations: false, + personalityTextHistory: false, + nearbyAgents: false, + recentEvents: false, + validationDetails: false, + resultingEvents: false, + providerUsageMetadata: false, + initialWorldState: false, + currentWorldState: false, + computedMetrics: false, + }, + }); + expect(minimal.turns[0]).not.toHaveProperty('observation'); + expect(standard.turns[0]).toHaveProperty('observation'); + expect(full).toHaveProperty('initialWorld'); + expect(full).toHaveProperty('configurationEvents'); + expect(full.initialWorld).not.toHaveProperty('events'); + expect(full.currentWorld).not.toHaveProperty('events'); + expect(full.worldEvents).toEqual([ + expect.objectContaining({ agentId: before.world.agents[0]!.id }), + ]); + expect(custom).not.toHaveProperty('metrics'); + expect(custom.turns[0]).not.toHaveProperty('provider'); + minimal.turns[0]!.outcome = 'rejected'; + expect( + simulation.generateExperimentExport(exportRequest('minimal')).turns[0] + ?.outcome, + ).toBe('accepted'); + expect(simulation.getSnapshot()).toEqual(before); + }); + it('calls the provider exactly once per turn in round-robin order', async () => { const seen: AgentObservation[] = []; const provider: AgentProvider = { @@ -478,6 +858,12 @@ describe('SimulationService', () => { expect(() => simulation.restoreDefaultPersonalities()).toThrow( SimulationConflictError, ); + expect(() => + simulation.previewExperimentExport(exportRequest('minimal')), + ).toThrow(SimulationConflictError); + expect(() => + simulation.generateExperimentExport(exportRequest('minimal')), + ).toThrow(SimulationConflictError); release({ decision: { requestedAction: { type: 'wait' }, summary: 'Done.' }, metadata: { diff --git a/apps/game-api/src/simulation-service.ts b/apps/game-api/src/simulation-service.ts index 0f55ce1..d0fe3dd 100644 --- a/apps/game-api/src/simulation-service.ts +++ b/apps/game-api/src/simulation-service.ts @@ -8,6 +8,9 @@ import { agentIdSchema, agentObservationSchema, agentTurnRecordSchema, + experimentIdSchema, + experimentExportDocumentSchema, + experimentExportPreviewSchema, h3CellSchema, PERSONALITY_MAX_LENGTH, personalitySchema, @@ -16,6 +19,10 @@ import { type AgentId, type AgentObservation, type AgentTurnRecord, + type ExperimentExportDocument, + type ExperimentExportPreview, + type ExperimentId, + type PersonalityConfigurationEvent, type H3Cell, type ProviderFailure, type SimulationSnapshot, @@ -29,10 +36,17 @@ import { toWorldState, type WorldState, } from '@agentborne/world-engine'; +import { + createExperimentExport, + createExperimentPreview, + type ExperimentSource, + ExperimentMetricAccumulator, +} from './experiment-export'; const RESET_GENERATED_AT = '2026-08-13T12:00:00.000Z'; const MAX_TURN_HISTORY = 120; const MAX_WORLD_EVENT_HISTORY = 120; +const DEFAULT_EXPERIMENT_RETENTION = 5_000; export class SimulationConflictError extends Error { constructor(message: string) { @@ -58,12 +72,16 @@ export interface SimulationServiceOptions { provider: AgentProvider; now?: () => string; createEventId?: () => string; + createExperimentId?: () => string; + experimentRetentionLimit?: number; } export class SimulationService { readonly #provider: AgentProvider; readonly #now: () => string; readonly #createEventId: () => string; + readonly #createExperimentId: () => string; + readonly #experimentRetentionLimit: number; #state: WorldState; #turns: AgentTurnRecord[] = []; #completedTurnCount = 0; @@ -71,32 +89,54 @@ export class SimulationService { #busy = false; #status: SimulationStatus; #activeAgentId: AgentId | null = null; + #experimentId: ExperimentId; + #experimentStartedAt: string; + #experimentTurns: AgentTurnRecord[] = []; + #initialExperimentAgents: Agent[]; + #initialExperimentWorld: SimulationSnapshot['world']; + #configurationEvents: PersonalityConfigurationEvent[] = []; + #experimentMetrics: ExperimentMetricAccumulator; constructor({ provider, now = () => new Date().toISOString(), createEventId = () => crypto.randomUUID(), + createExperimentId = () => crypto.randomUUID(), + experimentRetentionLimit = DEFAULT_EXPERIMENT_RETENTION, }: SimulationServiceOptions) { + if ( + !Number.isInteger(experimentRetentionLimit) || + experimentRetentionLimit < 1 + ) + throw new Error('Experiment retention limit must be a positive integer.'); this.#provider = provider; this.#now = now; this.#createEventId = createEventId; + this.#createExperimentId = createExperimentId; + this.#experimentRetentionLimit = experimentRetentionLimit; this.#state = toWorldState( createDevelopmentWorld({ generatedAt: RESET_GENERATED_AT }), ); this.#status = provider.configured ? 'paused' : 'configuration-error'; + this.#experimentId = experimentIdSchema.parse(this.#createExperimentId()); + this.#experimentStartedAt = this.#now(); + this.#initialExperimentAgents = structuredClone([ + ...this.#state.agents.values(), + ]); + this.#initialExperimentWorld = this.#worldSnapshot(); + this.#experimentMetrics = new ExperimentMetricAccumulator([ + ...this.#state.agents.keys(), + ]); } getSnapshot(): SimulationSnapshot { const agents = [...this.#state.agents.values()]; const next = agents[this.#cursor % agents.length]; if (!next) throw new Error('The development world has no agents.'); + const droppedRecords = + this.#completedTurnCount - this.#experimentTurns.length; return simulationSnapshotSchema.parse({ - world: { - generatedAt: RESET_GENERATED_AT, - hexes: [...this.#state.hexes].map(([cell, state]) => ({ cell, state })), - agents, - events: this.#state.events, - }, + world: this.#worldSnapshot(), turnNumber: this.#completedTurnCount, nextAgentId: next.id, activeAgentId: this.#activeAgentId, @@ -104,6 +144,17 @@ export class SimulationService { providerMode: this.#provider.mode, providerConfigured: this.#provider.configured, turns: this.#turns, + experiment: { + id: this.#experimentId, + startedAt: this.#experimentStartedAt, + totalCompletedTurns: this.#completedTurnCount, + retainedTurns: this.#experimentTurns.length, + firstRetainedTurn: this.#experimentTurns[0]?.turnNumber, + lastRetainedTurn: this.#experimentTurns.at(-1)?.turnNumber, + droppedRecords, + complete: droppedRecords === 0, + metrics: this.#experimentMetrics.snapshot(agents.map(({ id }) => id)), + }, }); } @@ -136,6 +187,17 @@ export class SimulationService { this.#completedTurnCount = 0; this.#cursor = 0; this.#activeAgentId = null; + this.#experimentId = experimentIdSchema.parse(this.#createExperimentId()); + this.#experimentStartedAt = this.#now(); + this.#experimentTurns = []; + this.#configurationEvents = []; + this.#initialExperimentAgents = structuredClone([ + ...this.#state.agents.values(), + ]); + this.#initialExperimentWorld = this.#worldSnapshot(); + this.#experimentMetrics = new ExperimentMetricAccumulator([ + ...this.#state.agents.keys(), + ]); this.#status = this.#provider.configured ? 'paused' : 'configuration-error'; return this.getSnapshot(); } @@ -174,6 +236,18 @@ export class SimulationService { const agents = new Map(this.#state.agents); agents.set(agent.id, updated); this.#state = { ...this.#state, agents }; + if (agent.personality !== updated.personality) { + this.#configurationEvents = [ + ...this.#configurationEvents, + { + timestamp: this.#now(), + agentId: agent.id, + previousPersonality: agent.personality, + newPersonality: updated.personality, + operation: 'custom-edit', + }, + ]; + } return updated; } @@ -189,18 +263,51 @@ export class SimulationService { personality, ]), ); + const configurationEvents: PersonalityConfigurationEvent[] = []; this.#state = { ...this.#state, agents: new Map( - [...this.#state.agents].map(([id, agent]) => [ - id, - { ...agent, personality: defaults.get(id) ?? agent.personality }, - ]), + [...this.#state.agents].map(([id, agent]) => { + const personality = defaults.get(id) ?? agent.personality; + if (personality !== agent.personality) + configurationEvents.push({ + timestamp: this.#now(), + agentId: id, + previousPersonality: agent.personality, + newPersonality: personality, + operation: 'restore-default', + }); + return [id, { ...agent, personality }]; + }), ), }; + this.#configurationEvents = [ + ...this.#configurationEvents, + ...configurationEvents, + ]; return this.getSnapshot(); } + previewExperimentExport(request: unknown): ExperimentExportPreview { + if (this.#busy) + throw new SimulationConflictError( + 'Export is unavailable while a model turn is in progress.', + ); + return experimentExportPreviewSchema.parse( + createExperimentPreview(this.#experimentSource(), request, this.#now()), + ); + } + + generateExperimentExport(request: unknown): ExperimentExportDocument { + if (this.#busy) + throw new SimulationConflictError( + 'Export is unavailable while a model turn is in progress.', + ); + return experimentExportDocumentSchema.parse( + createExperimentExport(this.#experimentSource(), request, this.#now()), + ); + } + async executeNextTurn(): Promise { if (this.#busy) { throw new SimulationConflictError('A model turn is already in progress.'); @@ -308,10 +415,40 @@ export class SimulationService { this.#state = state; this.#turns = turns; + this.#experimentTurns = [ + ...this.#experimentTurns, + structuredClone(record), + ].slice(-this.#experimentRetentionLimit); + this.#experimentMetrics.add(record); this.#completedTurnCount = record.turnNumber; this.#cursor = cursor; } + #worldSnapshot(): SimulationSnapshot['world'] { + return { + generatedAt: RESET_GENERATED_AT, + hexes: [...this.#state.hexes].map(([cell, state]) => ({ cell, state })), + agents: structuredClone([...this.#state.agents.values()]), + events: structuredClone([...this.#state.events]), + }; + } + + #experimentSource(): ExperimentSource { + return { + id: this.#experimentId, + startedAt: this.#experimentStartedAt, + providerMode: this.#provider.mode, + retentionLimit: this.#experimentRetentionLimit, + totalCompletedTurns: this.#completedTurnCount, + turns: this.#experimentTurns, + initialAgents: this.#initialExperimentAgents, + currentAgents: [...this.#state.agents.values()], + configurationEvents: this.#configurationEvents, + initialWorld: this.#initialExperimentWorld, + currentWorld: this.#worldSnapshot(), + }; + } + #buildObservation(agentId: AgentId): AgentObservation { const agent = this.#state.agents.get(agentId); if (!agent) throw new Error('The active agent does not exist.'); diff --git a/apps/world-lab/src/app/styles.css b/apps/world-lab/src/app/styles.css index bb311af..26fc17e 100644 --- a/apps/world-lab/src/app/styles.css +++ b/apps/world-lab/src/app/styles.css @@ -27,6 +27,17 @@ textarea { font: inherit; } +fieldset { + margin: 14px 0; + padding: 12px; + border: 1px solid #34433f; + border-radius: 6px; +} +legend { + color: #aebdb8; + font-size: 0.75rem; +} + .topbar { min-height: 88px; padding: 18px clamp(18px, 3vw, 42px); @@ -179,6 +190,65 @@ h2 { padding: 22px; border-bottom: 1px solid #2d3835; } +.usage-meter, +.agent-usage { + display: flex; + flex-wrap: wrap; + gap: 6px 12px; + margin-top: 14px; + padding: 10px; + border: 1px solid #34433f; + border-radius: 6px; + color: #aebdb8; + font-size: 0.72rem; +} +.usage-meter strong, +.agent-usage strong { + flex-basis: 100%; + color: #e6ece8; +} +.export-panel > summary { + cursor: pointer; + color: #e6ece8; + font-weight: 650; +} +.export-panel > .muted { + margin-top: 10px; +} +.export-panel > label, +.range-row label { + display: grid; + gap: 6px; + margin-top: 12px; + color: #9fb1ab; + font-size: 0.76rem; +} +.checkbox-row { + display: flex; + align-items: center; + gap: 8px; + margin: 7px 0; + color: #c1cec9; + font-size: 0.76rem; +} +.selection-actions, +.export-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin: 8px 0; +} +.range-row { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 10px; +} +.preview-grid { + margin-top: 14px; + padding: 10px; + border: 1px solid #34433f; + border-radius: 6px; +} .selected-panel h2 { font-size: 1.35rem; margin-bottom: 16px; diff --git a/apps/world-lab/src/components/world-lab.test.tsx b/apps/world-lab/src/components/world-lab.test.tsx index e7d137d..6fd3592 100644 --- a/apps/world-lab/src/components/world-lab.test.tsx +++ b/apps/world-lab/src/components/world-lab.test.tsx @@ -3,6 +3,7 @@ import userEvent from '@testing-library/user-event'; import { gridDisk } from 'h3-js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { + experimentExportDocumentSchema, simulationSnapshotSchema, type SimulationSnapshot, } from '@agentborne/shared'; @@ -174,8 +175,41 @@ const initial = simulationSnapshotSchema.parse({ providerMode: 'scripted-test', providerConfigured: true, turns: [], + experiment: { + id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + startedAt: '2026-08-13T12:00:00.000Z', + totalCompletedTurns: 0, + retainedTurns: 0, + droppedRecords: 0, + complete: true, + metrics: { + aggregate: emptyMetrics(), + byAgent: world.agents.map(({ id }) => ({ + agentId: id, + metrics: emptyMetrics(), + })), + }, + }, }); +function emptyMetrics() { + return { + totalTurns: 0, + accepted: 0, + rejected: 0, + providerErrors: 0, + requestedMoves: 0, + requestedInfections: 0, + requestedWaits: 0, + acceptedMovements: 0, + successfullyInfectedCells: 0, + uniqueVisitedCells: 0, + tokens: {}, + knownCostCredits: 0, + turnsWithUnknownCost: 0, + }; +} + function afterInfection(): SimulationSnapshot { const agent = world.agents[0]!; const event = { @@ -213,6 +247,10 @@ function afterInfection(): SimulationSnapshot { provider: 'scripted-test' as const, model: 'test', latencyMs: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + costCredits: 0, }, }; return simulationSnapshotSchema.parse({ @@ -229,6 +267,40 @@ function afterInfection(): SimulationSnapshot { turnNumber: 1, nextAgentId: world.agents[1]!.id, turns: [turn], + experiment: { + ...initial.experiment, + totalCompletedTurns: 1, + retainedTurns: 1, + firstRetainedTurn: 1, + lastRetainedTurn: 1, + metrics: { + aggregate: { + ...emptyMetrics(), + totalTurns: 1, + accepted: 1, + requestedInfections: 1, + successfullyInfectedCells: 1, + uniqueVisitedCells: 1, + averageLatencyMs: 0, + }, + byAgent: initial.experiment.metrics.byAgent.map((entry, index) => + index === 0 + ? { + ...entry, + metrics: { + ...emptyMetrics(), + totalTurns: 1, + accepted: 1, + requestedInfections: 1, + successfullyInfectedCells: 1, + uniqueVisitedCells: 1, + averageLatencyMs: 0, + }, + } + : entry, + ), + }, + }, }); } @@ -267,6 +339,10 @@ beforeEach(() => { 'fetch', vi.fn(() => jsonResponse(initial)), ); + vi.stubGlobal( + 'confirm', + vi.fn(() => true), + ); }); afterEach(() => { @@ -472,6 +548,9 @@ describe('WorldLab', () => { await screen.findByText('Infection · ' + world.agents[0]!.currentCell), ).toBeInTheDocument(); await user.click(screen.getByRole('button', { name: 'Reset world' })); + expect(confirm).toHaveBeenCalledWith( + expect.stringContaining('unexported telemetry'), + ); await waitFor(() => expect( screen.getByText('Development world loaded with six agents.'), @@ -757,8 +836,234 @@ describe('WorldLab', () => { ), ).toBeInTheDocument(); }); + + it('shows current and selected-agent experiment usage', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(() => jsonResponse(afterInfection())), + ); + render(); + expect( + await screen.findByLabelText('Current experiment usage'), + ).toHaveTextContent('1 turns'); + expect(screen.getByLabelText('Current experiment usage')).toHaveTextContent( + '0.0 credits known cost', + ); + expect(screen.getByLabelText('Selected agent usage')).toHaveTextContent( + '1 turns', + ); + }); + + it('preselects one agent, supports multi-select and previews server-owned export', async () => { + const user = userEvent.setup(); + render(); + await user.click( + await screen.findByRole('button', { name: 'Export this agent' }), + ); + expect(screen.getByRole('checkbox', { name: /Ember/ })).toBeChecked(); + expect(screen.getByRole('checkbox', { name: /Rook/ })).not.toBeChecked(); + await user.click(screen.getByRole('checkbox', { name: /Rook/ })); + vi.mocked(fetch).mockImplementationOnce(() => + jsonResponse({ + experimentId: initial.experiment.id, + matchingRecordCount: 0, + selectedAgentCount: 2, + retention: { + limit: 5000, + totalCompletedTurns: 0, + retainedTurns: 0, + droppedRecords: 0, + complete: true, + requestedRangeExtendsBeyondRetention: false, + }, + knownCostCredits: 0, + turnsWithUnknownCost: 0, + serializedUtf8Bytes: 900, + approximateAiInputTokens: 225, + tokenEstimateMethod: 'ceil(UTF-8 bytes / 4)', + }), + ); + await user.click(screen.getByRole('button', { name: 'Preview export' })); + expect(await screen.findByLabelText('Export preview')).toHaveTextContent( + '900 bytes', + ); + const request = JSON.parse( + String(vi.mocked(fetch).mock.calls.at(-1)?.[1]?.body), + ); + expect(request.agents).toEqual({ + mode: 'selected', + agentIds: [world.agents[0]!.id, world.agents[1]!.id], + }); + await user.click(screen.getByRole('button', { name: 'Clear' })); + expect( + screen.getByRole('button', { name: 'Preview export' }), + ).toBeDisabled(); + await user.click(screen.getByRole('button', { name: 'Select all' })); + for (const agent of world.agents) + expect( + screen.getByRole('checkbox', { name: new RegExp(agent.name) }), + ).toBeChecked(); + }); + + it('offers every tier, turn selector, outcome/action filters, and dependent Custom switches', async () => { + const user = userEvent.setup(); + render(); + await user.click(await screen.findByText('Experiment Export')); + const level = screen.getByLabelText('Export level'); + expect(level).toHaveTextContent('Minimal'); + expect(level).toHaveTextContent('Standard'); + expect(level).toHaveTextContent('Full safe'); + expect(level).toHaveTextContent('Custom'); + expect(screen.getByLabelText('JSON serialization')).toHaveValue('compact'); + await user.selectOptions( + screen.getByLabelText('JSON serialization'), + 'pretty', + ); + await user.selectOptions(level, 'custom'); + expect(screen.getByText('Advanced Custom switches')).toBeInTheDocument(); + const observations = screen.getByRole('checkbox', { + name: 'Turn observations', + }); + await user.click(observations); + expect( + screen.getByRole('checkbox', { name: 'Nearby agents' }), + ).toBeDisabled(); + expect( + screen.getByRole('checkbox', { name: 'Recent events' }), + ).toBeDisabled(); + await user.selectOptions(screen.getByLabelText('Turn range'), 'range'); + expect(screen.getByLabelText('From turn')).toBeInTheDocument(); + await user.click(screen.getByRole('checkbox', { name: 'accepted' })); + await user.click(screen.getByRole('checkbox', { name: 'rejected' })); + await user.click(screen.getByRole('checkbox', { name: 'provider error' })); + expect( + screen.getByRole('button', { name: 'Preview export' }), + ).toBeDisabled(); + }); + + it('copies and downloads the exact same validated generated JSON and revokes its URL', async () => { + const user = userEvent.setup(); + const progressed = afterInfection(); + vi.stubGlobal( + 'fetch', + vi.fn(() => jsonResponse(progressed)), + ); + const clipboardWrite = vi.fn(async () => undefined); + Object.defineProperty(navigator, 'clipboard', { + configurable: true, + value: { writeText: clipboardWrite }, + }); + const createObjectURL = vi.fn(() => 'blob:experiment'); + const revokeObjectURL = vi.fn(); + vi.stubGlobal('URL', { ...URL, createObjectURL, revokeObjectURL }); + const click = vi + .spyOn(HTMLAnchorElement.prototype, 'click') + .mockImplementation(() => undefined); + render(); + await user.click( + await screen.findByRole('button', { name: 'Export this agent' }), + ); + const document = minimalExportDocument(progressed); + vi.mocked(fetch).mockImplementationOnce(() => jsonResponse({ document })); + await user.click(screen.getByRole('button', { name: 'Generate JSON' })); + await user.click(await screen.findByRole('button', { name: 'Copy JSON' })); + expect(clipboardWrite).toHaveBeenCalledWith( + JSON.stringify(experimentExportDocumentSchema.parse(document)), + ); + clipboardWrite.mockRejectedValueOnce(new Error('denied')); + await user.click(screen.getByRole('button', { name: 'Copy JSON' })); + expect(await screen.findByText(/Copy failed/)).toBeInTheDocument(); + await user.click(screen.getByRole('button', { name: 'Download JSON' })); + expect(createObjectURL).toHaveBeenCalledTimes(1); + expect(revokeObjectURL).toHaveBeenCalledWith('blob:experiment'); + expect(click).toHaveBeenCalledTimes(1); + click.mockRestore(); + }); + + it('auto-pauses a fully infected world and disables automatic Start only', async () => { + const infected = simulationSnapshotSchema.parse({ + ...initial, + world: { + ...initial.world, + hexes: initial.world.hexes.map((hex) => ({ + ...hex, + state: 'infected' as const, + })), + }, + }); + vi.stubGlobal( + 'fetch', + vi.fn(() => jsonResponse(infected)), + ); + render(); + expect( + await screen.findByText(/Development world fully infected/), + ).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Start' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'Single turn' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'Reset world' })).toBeEnabled(); + }); }); +function minimalExportDocument(snapshot: SimulationSnapshot) { + const turn = snapshot.turns[0]!; + const agent = snapshot.world.agents[0]!; + return { + schemaVersion: 1 as const, + generatedAt: '2026-08-13T12:00:02.000Z', + experiment: { + id: snapshot.experiment.id, + startedAt: snapshot.experiment.startedAt, + providerMode: snapshot.providerMode, + }, + retention: { + limit: 5000, + totalCompletedTurns: 1, + retainedTurns: 1, + firstRetainedTurn: 1, + lastRetainedTurn: 1, + droppedRecords: 0, + complete: true, + requestedRangeExtendsBeyondRetention: false, + }, + filters: { + agents: { mode: 'selected' as const, agentIds: [agent.id] }, + turns: { mode: 'entire-retained' as const }, + outcomes: ['accepted', 'rejected', 'provider-error'] as const, + actions: ['move', 'infect', 'wait'] as const, + level: 'minimal' as const, + }, + selection: { + selectedAgentIds: [agent.id], + matchingRecordCount: 1, + firstMatchingTurn: 1, + lastMatchingTurn: 1, + }, + agents: [agent], + metrics: { + aggregate: snapshot.experiment.metrics.aggregate, + byAgent: [snapshot.experiment.metrics.byAgent[0]!], + }, + turns: [ + { + turnNumber: turn.turnNumber, + startedAt: turn.startedAt, + completedAt: turn.completedAt, + agentId: turn.agentId, + outcome: turn.outcome, + ...(turn.outcome === 'accepted' + ? { + requestedAction: turn.requestedAction, + summary: turn.summary, + eventSummary: `Infected ${agent.currentCell}.`, + provider: turn.provider, + } + : {}), + }, + ], + }; +} + function apiBaseForTest() { return process.env.NEXT_PUBLIC_GAME_API_BASE_URL ?? '/api/game/simulation'; } diff --git a/apps/world-lab/src/components/world-lab.tsx b/apps/world-lab/src/components/world-lab.tsx index 2b4e9a2..107b4d3 100644 --- a/apps/world-lab/src/components/world-lab.tsx +++ b/apps/world-lab/src/components/world-lab.tsx @@ -3,6 +3,9 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { PERSONALITY_MAX_LENGTH, + experimentExportPreviewSchema, + experimentExportRequestSchema, + experimentExportResponseSchema, personalitySchema, resetSimulationResponseSchema, restoreDefaultPersonalitiesResponseSchema, @@ -12,6 +15,10 @@ import { updateAgentPersonalityResponseSchema, type AgentId, type AgentTurnRecord, + type CustomExportOptions, + type ExperimentExportDocument, + type ExperimentExportPreview, + type ExperimentExportRequest, type H3Cell, type SimulationSnapshot, } from '@agentborne/shared'; @@ -40,10 +47,14 @@ export function WorldLab() { ); const [speed, setSpeed] = useState(1_000); const [uiError, setUiError] = useState(null); + const [exportOpen, setExportOpen] = useState(false); + const [exportAgentIds, setExportAgentIds] = useState([]); const inFlightRef = useRef(false); const applySnapshot = useCallback((next: SimulationSnapshot) => { setSnapshot(next); + if (next.world.hexes.every(({ state }) => state === 'infected')) + setRunning(false); setSelectedCell((current) => current ?? next.world.hexes[0]!.cell); setSelectedAgentId((current) => current ?? next.world.agents[0]!.id); }, []); @@ -98,6 +109,14 @@ export function WorldLab() { const reset = async () => { if (inFlightRef.current) return; + if ( + snapshot && + snapshot.experiment.totalCompletedTurns > 0 && + !window.confirm( + `Reset World will discard ${snapshot.experiment.totalCompletedTurns} completed experiment turn records and all unexported telemetry. Continue?`, + ) + ) + return; setRunning(false); setResetting(true); setUiError(null); @@ -204,6 +223,8 @@ export function WorldLab() { } }; + const fullyInfected = + snapshot?.world.hexes.every(({ state }) => state === 'infected') ?? false; if (!snapshot) { return (
@@ -241,6 +262,12 @@ export function WorldLab() { resetting || personalityPending || snapshot.activeAgentId !== null; + const exportMutationPending = + running || + inFlight || + resetting || + personalityPending || + snapshot.activeAgentId !== null; return (
@@ -298,6 +325,7 @@ export function WorldLab() { disabled={ inFlight || personalityPending || + fullyInfected || !snapshot.providerConfigured } type="button" @@ -346,6 +374,13 @@ export function WorldLab() { + + {fullyInfected && ( +

+ Development world fully infected. Automatic playback is paused; + Single turn remains a manual cost-incurring diagnostic action. +

+ )} {!snapshot.providerConfigured && (

Model calls unavailable. Set OPENROUTER_API_KEY on the Game API @@ -380,9 +415,28 @@ export function WorldLab() { mutationDisabled={personalityControlsDisabled} mutationPending={personalityPending} onApplyPersonality={updatePersonality} + metrics={ + snapshot.experiment.metrics.byAgent.find( + ({ agentId }) => agentId === selectedAgent.id, + )?.metrics + } + onExportAgent={(agentId) => { + setRunning(false); + setExportAgentIds([agentId]); + setExportOpen(true); + }} /> )} + + {selectedHex && (

Selected hex

@@ -418,6 +472,8 @@ function AgentInspector({ mutationDisabled, mutationPending, onApplyPersonality, + metrics, + onExportAgent, }: { agent: SimulationSnapshot['world']['agents'][number]; cellState: 'open' | 'infected'; @@ -429,6 +485,8 @@ function AgentInspector({ agentId: AgentId, personality: string, ) => Promise; + metrics?: SimulationSnapshot['experiment']['metrics']['aggregate']; + onExportAgent: (agentId: AgentId) => void; }) { const [editing, setEditing] = useState(false); const [draft, setDraft] = useState(agent.personality); @@ -474,6 +532,17 @@ function AgentInspector({
{cellState}
+
+ Experiment usage + {metrics?.totalTurns ?? 0} turns + {formatCost(metrics?.knownCostCredits ?? 0)} known cost + {(metrics?.turnsWithUnknownCost ?? 0) > 0 && ( + {metrics?.turnsWithUnknownCost} unknown-cost turns + )} +
+

Active personality

{!editing && ( @@ -666,6 +735,496 @@ function AgentInspector({ ); } +function ExperimentUsageMeter({ snapshot }: { snapshot: SimulationSnapshot }) { + const metrics = snapshot.experiment.metrics.aggregate; + return ( +
+ Current experiment + {snapshot.experiment.totalCompletedTurns} turns + {formatCost(metrics.knownCostCredits)} known cost + + {metrics.tokens.totalTokens ?? + (metrics.tokens.promptTokens ?? 0) + + (metrics.tokens.completionTokens ?? 0)}{' '} + tokens + + {metrics.turnsWithUnknownCost > 0 && ( + {metrics.turnsWithUnknownCost} unknown-cost turns + )} +
+ ); +} + +const defaultCustomOptions: CustomExportOptions = { + turnObservations: true, + personalityTextHistory: true, + nearbyAgents: true, + recentEvents: true, + validationDetails: true, + resultingEvents: true, + providerUsageMetadata: true, + initialWorldState: false, + currentWorldState: true, + computedMetrics: true, +}; + +function ExperimentExportPanel({ + agents, + disabled, + open, + selectedAgentIds, + onOpenChange, + onSelectionChange, +}: { + agents: SimulationSnapshot['world']['agents']; + disabled: boolean; + open: boolean; + selectedAgentIds: AgentId[]; + onOpenChange: (open: boolean) => void; + onSelectionChange: (ids: AgentId[]) => void; +}) { + const [level, setLevel] = + useState('minimal'); + const [serialization, setSerialization] = + useState('compact'); + const [turnMode, setTurnMode] = useState< + 'entire-retained' | 'latest' | 'range' + >('entire-retained'); + const [latestCount, setLatestCount] = useState<10 | 25 | 50 | 120>(120); + const [fromTurn, setFromTurn] = useState(1); + const [toTurn, setToTurn] = useState(120); + const [outcomes, setOutcomes] = useState< + Array<'accepted' | 'rejected' | 'provider-error'> + >(['accepted', 'rejected', 'provider-error']); + const [actions, setActions] = useState>([ + 'move', + 'infect', + 'wait', + ]); + const [custom, setCustom] = useState(defaultCustomOptions); + const [preview, setPreview] = useState(null); + const [document, setDocument] = useState( + null, + ); + const [generatedRequestJson, setGeneratedRequestJson] = useState< + string | null + >(null); + const [pending, setPending] = useState(false); + const [notice, setNotice] = useState(null); + const detailsRef = useRef(null); + + useEffect(() => { + if (open && detailsRef.current) detailsRef.current.open = true; + }, [open]); + + const requestInput = { + agents: + selectedAgentIds.length === agents.length + ? { mode: 'all' as const } + : { mode: 'selected' as const, agentIds: selectedAgentIds }, + turns: + turnMode === 'entire-retained' + ? { mode: 'entire-retained' as const } + : turnMode === 'latest' + ? { mode: 'latest' as const, count: latestCount } + : { mode: 'range' as const, fromTurn, toTurn }, + outcomes, + actions, + level, + serialization, + ...(level === 'custom' ? { custom } : {}), + }; + const parsedRequest = experimentExportRequestSchema.safeParse(requestInput); + const generationDisabled = disabled || pending || !parsedRequest.success; + const currentRequestJson = parsedRequest.success + ? JSON.stringify(parsedRequest.data) + : null; + const documentIsCurrent = + document !== null && generatedRequestJson === currentRequestJson; + + const requestExport = async (previewOnly: boolean) => { + if (!parsedRequest.success) return; + setPending(true); + setNotice(null); + try { + const response = await fetch( + `${apiBase}/experiment/export${previewOnly ? '/preview' : ''}`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(parsedRequest.data), + }, + ); + if (response.status === 409) { + setNotice( + 'Pause playback and wait for pending mutations before exporting.', + ); + return; + } + if (!response.ok) throw new Error('export request failed'); + if (previewOnly) { + setPreview(experimentExportPreviewSchema.parse(await response.json())); + setDocument(null); + setGeneratedRequestJson(null); + setNotice('Export preview updated.'); + } else { + const payload = experimentExportResponseSchema.parse( + await response.json(), + ); + setDocument(payload.document); + setGeneratedRequestJson(JSON.stringify(parsedRequest.data)); + setNotice('Export JSON generated and schema-validated.'); + } + } catch { + setNotice('Export failed safely. Review the selection and try again.'); + } finally { + setPending(false); + } + }; + + const copyJson = async () => { + if (!document || !documentIsCurrent) return; + try { + await navigator.clipboard.writeText(serializeExportDocument(document)); + setNotice('Export JSON copied to the clipboard.'); + } catch { + setNotice('Copy failed. Clipboard permission may be unavailable.'); + } + }; + + const downloadJson = () => { + if (!document || !documentIsCurrent) return; + try { + const json = serializeExportDocument(document); + const url = URL.createObjectURL( + new Blob([json], { type: 'application/json' }), + ); + const link = window.document.createElement('a'); + const scope = + document.selection.selectedAgentIds.length === agents.length + ? 'all-agents' + : document.selection.selectedAgentIds.length === 1 + ? 'one-agent' + : `${document.selection.selectedAgentIds.length}-agents`; + const range = + document.filters.turns.mode === 'range' + ? `turns-${document.filters.turns.fromTurn}-${document.filters.turns.toTurn}` + : document.filters.turns.mode === 'latest' + ? `latest-${document.filters.turns.count}` + : 'entire-retained'; + link.href = url; + link.download = `agentborne-experiment-${document.experiment.id}-${scope}-${range}.json`; + link.click(); + URL.revokeObjectURL(url); + setNotice('Export JSON download started.'); + } catch { + setNotice('Download failed safely.'); + } + }; + + const toggle = (values: T[], value: T): T[] => + values.includes(value) + ? values.filter((candidate) => candidate !== value) + : [...values, value]; + + return ( +
{ + const nextOpen = event.currentTarget.open; + onOpenChange(nextOpen); + if (nextOpen && selectedAgentIds.length === 0) + onSelectionChange(agents.map(({ id }) => id)); + }} + > + Experiment Export +

+ Server-owned retained telemetry; exports never change prompts or model + usage. +

+
+ Agents +
+ + +
+ {agents.map((agent) => ( + + ))} +
+ +
+ Advanced JSON options + +
+ + {turnMode === 'latest' && ( + + )} + {turnMode === 'range' && ( +
+ + +
+ )} + setOutcomes(toggle(outcomes, value))} + /> + setActions(toggle(actions, value))} + /> + {level === 'custom' && ( +
+ Advanced Custom switches + {(Object.keys(custom) as Array).map( + (key) => ( + + ), + )} +
+ )} + {open && !parsedRequest.success && ( +

+ Select at least one agent, outcome, and action, and enter a valid + range. +

+ )} + {disabled && ( +

+ Pause playback and wait for all turn, reset, and personality work to + finish. +

+ )} +
+ + + + +
+ {preview && ( +
+
+
Matching
+
{preview.matchingRecordCount} records
+
+
+
Size
+
{preview.serializedUtf8Bytes} bytes
+
+
+
Approx. AI input
+
{preview.approximateAiInputTokens} tokens
+
+
+
Selected cost
+
{formatCost(preview.knownCostCredits)}
+
+
+ )} + {notice && ( +

+ {notice} +

+ )} +
+ ); +} + +function FilterChecks({ + label, + options, + selected, + onToggle, +}: { + label: string; + options: readonly T[]; + selected: T[]; + onToggle: (value: T) => void; +}) { + return ( +
+ {label} + {options.map((option) => ( + + ))} +
+ ); +} + +function customOptionLabel(key: keyof CustomExportOptions): string { + return { + turnObservations: 'Turn observations', + personalityTextHistory: 'Personality text and history', + nearbyAgents: 'Nearby agents', + recentEvents: 'Recent events', + validationDetails: 'Validation details', + resultingEvents: 'Resulting events', + providerUsageMetadata: 'Provider usage metadata', + initialWorldState: 'Initial world state', + currentWorldState: 'Current world state', + computedMetrics: 'Computed metrics', + }[key]; +} + +function formatCost(cost: number): string { + return `${cost.toFixed(8).replace(/0+$/, '').replace(/\.$/, '.0')} credits`; +} + +function serializeExportDocument(document: ExperimentExportDocument): string { + return document.filters.serialization === 'pretty' + ? JSON.stringify(document, null, 2) + : JSON.stringify(document); +} + function EventLog({ turns }: { turns: AgentTurnRecord[] }) { return (
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b0f8429..8209528 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -8,6 +8,8 @@ World Lab → Game API / simulation service → agent runtime → requested acti snapshot / turn record ← world engine validation ``` +The Game API also owns one process-local experiment record. Each completed safe turn is captured once, independently from the browser snapshot, and server-side export levels filter that record without affecting provider requests. + ## Applications `apps/world-lab` is a Next.js App Router developer/admin surface. It fetches runtime-validated simulation snapshots through a local rewrite, controls one turn at a time, and updates MapLibre's existing H3 GeoJSON source without recreating the map. Agent markers are fully visible and use deterministic offsets when sharing cells. @@ -19,6 +21,8 @@ World Lab → Game API / simulation service → agent runtime → requested acti - `POST /api/simulation/reset` — deterministic reset, rejected while a turn is active - `POST /api/simulation/agents/:agentId/personality` — trim, validate, and replace one active personality - `POST /api/simulation/personalities/restore-defaults` — restore all six original personality directives without resetting progress +- `POST /api/simulation/experiment/export/preview` — validate filters and report subset size, retention, cost, and approximate sharing tokens +- `POST /api/simulation/experiment/export` — construct one schema-versioned safe JSON document The legacy `GET /api/development-world` and `GET /health` endpoints remain for low-level diagnostics. @@ -34,6 +38,14 @@ The engine alone accepts or rejects actions and creates world events. Rejections Only failures thrown by the provider decision call become sanitized provider-failure turn records. Those failures remain non-mutating, count as completed recorded turns, advance round robin, and do not prevent later turns. The public turn number is the total completed-turn count and is independent of the retained history: snapshots keep only the newest 120 turn records and newest 120 world events. Observations select the newest eight relevant public events from that retained event history in chronological order. +## Experiment telemetry and export + +The active experiment has a runtime-validated UUID, start time, initial six-agent configuration, immutable personality-change events, initial world, and up to 5,000 complete safe turns. The existing browser snapshot and world-event list remain capped at 120. Absolute numbering, first/last retained turns, dropped count, and completeness disclose truncation. Reset creates a new experiment and clears telemetry/cost while preserving active personalities; no previous experiments survive reset or process restart. + +Metrics and filtering are deterministic Game API responsibilities. Export metrics always describe the filtered retained subset. Decimal-string accumulation prevents binary floating-point artifacts in aggregate provider cost while retaining tiny charges. Minimal and Standard provide progressively more turn context, Full safe includes the richest safe structured record and world/configuration context, and Custom applies validated inclusion dependencies. Full-safe initial/current world states retain all agents and hex states for spatial context but deliberately omit event history; the agent-filtered top-level `worldEvents` array is the canonical exported event stream. Compact JSON is the AI-sharing default and Pretty JSON is available for human review. Preview serializes the same prospective document with the selected encoding and estimates AI input as `ceil(UTF-8 bytes / 4)` without a tokenizer dependency. + +The agent runtime follows [OpenRouter's usage-accounting contract](https://openrouter.ai/docs/cookbook/administration/usage-accounting) and normalizes optional non-streaming usage fields: prompt, completion, total, reasoning, cached-read, cache-write tokens, and actual `usage.cost` as `costCredits`. It never derives price from a table. Safe usage already returned with a billable response is retained on later decision JSON/schema failure; network and HTTP failures without usage remain unknown. Scripted providers explicitly report zero tokens and zero cost. + ## Packages `packages/shared` owns all public schemas: profiles, personality mutation requests/responses, observations, PR 2 decisions, provider metadata/failures, turn outcomes, snapshots, and API responses. Types are inferred from Zod. @@ -46,3 +58,4 @@ The provider abort timeout covers the complete response lifecycle, including bod The rationale and deferrals are recorded in [ADR 0002](adr/0002-first-visible-llm-invasion.md). Personality ownership and reset semantics are recorded in [ADR 0003](adr/0003-session-personality-configuration.md). +Experiment capture and export semantics are recorded in [ADR 0004](adr/0004-server-owned-experiment-telemetry.md). diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 800b7a5..b375dc9 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -24,6 +24,14 @@ World Lab personality edits are also untrusted, bounded text. The Game API trims Personality mutation errors use typed, generic response bodies. They do not expose raw prompts, provider responses, credentials, diagnostics, stack traces, or internal service details. There is still no authentication or persistence; these endpoints remain limited to the loopback development surface. +## Experiment telemetry and exports + +The Game API captures only schema-validated safe observations, requested actions, visible concise summaries, validation outcomes, world events, bounded provider failures, and normalized usage metadata. It never records or exports API keys, authorization headers, fixed system prompts, raw provider request/response bodies, private chain-of-thought, hidden analysis, or unbounded diagnostics. Historical turns and configuration events are cloned and immutable. + +Export requests, agent IDs, levels, ranges, outcome/action filters, and Custom dependencies are runtime-validated. Filtering and metrics remain server-owned so the browser cannot promote its 120-turn snapshot into authoritative history. Preview and generation reject overlap with turns and mutations. Generated JSON is schema-versioned and omits excluded fields instead of emitting misleading nulls. Export world-state snapshots contain spatial agents/hexes but no embedded event history; filtered top-level `worldEvents` is the only canonical event stream, preventing out-of-scope agent events from leaking through current-world context. + +Actual cost is accepted only from OpenRouter's safe `usage.cost`. Missing cost is unknown, never zero; scripted-test providers explicitly report zero. There is still no authentication, budget enforcement, persistence, provider-management endpoint, upload, or sharing link. The loopback-only boundary remains mandatory. + ## Reporting This is a private repository. Report suspected vulnerabilities privately to the repository owners rather than opening a public issue. diff --git a/docs/TESTING.md b/docs/TESTING.md index 0997a80..6eeb84d 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -17,7 +17,6 @@ pnpm test pnpm build pnpm exec playwright install chromium # only if Chromium is not already installed pnpm test:e2e -pnpm smoke:openrouter ``` `pnpm validate` aggregates formatting, lint, type checking, unit/integration tests, and builds. Playwright remains separate. Its web server starts both applications with the explicit deterministic provider; `AGENTBORNE_PROVIDER=scripted` is never an implicit OpenRouter fallback. @@ -26,16 +25,22 @@ pnpm smoke:openrouter - Shared schema tests cover valid/invalid observations and decisions, all turn outcomes, response snapshots, 120-record turn and 120-event history bounds, and model-authored string limits. - Shared schema tests cover trimmed personality updates, empty/oversized/malformed values, update/restore response contracts, and typed mutation errors. +- Shared schema tests cover optional provider usage and tiny costs, experiment identities/manifests/retention/configuration events, all export levels and filters, Custom dependencies, invalid empty/range/agent selections, previews, generated documents, and level-specific omissions. - World-engine tests cover adjacent and non-adjacent movement, infection persistence, repeated-infection rejection, wait, immutability on rejection, and deterministic 61-cell/six-agent construction. - Agent-runtime tests recursively inspect the actual strict OpenRouter schema and Gemini-compatible parameters, prompt placement, explicit model overrides, parsing, runtime validation, malformed/unsupported responses, body-inclusive timeouts and timer cleanup, bounded safe HTTP diagnostics, secret non-leakage, missing configuration, and explicit scripted output. +- Agent-runtime tests normalize successful/missing reasoning/cache/token/cost usage, retain known usage through malformed or unsupported decisions, and prove scripted zero cost. - Simulation-service tests cover deterministic reset, round robin beyond retained history, monotonic total turn numbering, exactly one call per turn, latest-state observations, 120-record turn retention, 120-event retention, accepted/rejected/failure records, atomic rollback on internal validation failure, recovery, and turn/reset concurrency. +- Simulation-service tests cover independent 5,000-versus-120 retention, configurable truncation, experiment identity/reset, immutable personality history, agent/range/outcome/action filtering, every export tier, eventless spatial world snapshots with canonical filtered `worldEvents`, Compact/Pretty byte estimates, artifact-free decimal cost aggregation, subset metrics, tiny/unknown costs, visitation/infection metrics, ordering, immutability, and export concurrency/recovery. - Personality service tests cover single-agent updates, unknown/invalid rejection without mutation, next-observation use, preservation of progress and history, reset preservation, six-profile default restoration, active-turn conflicts, and recovery. - API integration tests validate snapshot, turn, reset, missing configuration, provider failure, internal-failure propagation and recovery, rejection, and typed conflict responses. - API integration tests validate personality update/default-restore success, boundary validation, unknown IDs, conflicts, and safe error/response bodies. +- API integration tests validate export preview/generation, unknown/empty/malformed selections, retention disclosure, typed conflicts, and safe-response exclusions. - React tests cover controls/statuses, playback configuration, single turn, reset, agent/hex selection, inspector output, H3 readiness, and safe configuration errors. - React tests cover explicit edit/cancel/apply, every preset, Custom matching, character feedback, playback/pending disabling, reset preservation, restore confirmation, and active-versus-historical personality clarity. +- React tests cover experiment/agent usage, export agent selection, levels and Custom switches, filters, preview, copy/download identity and URL cleanup, reset telemetry warnings, and fully infected auto-pause. - Playwright uses real MapLibre feature inspection and the explicit scripted provider to verify personality mutation through the UI/API, next-observation use, reset preservation, default restoration without progress loss, 61 unique rendered H3 cells, infection counts, and six markers. +- Playwright previews one agent, downloads and runtime-validates a multi-agent Minimal export and its metrics/order/zero cost, then confirms no OpenRouter request occurred. ## Real-provider smoke -The separately opted-in `pnpm smoke:openrouter` command makes one bounded OpenRouter call and validates the decision. Node 24's built-in environment parser reads only `OPENROUTER_API_KEY` and `AGENTBORNE_MODEL` from the repository-root `.env` for this command, and file values override stale exported values for the smoke process. A missing file still yields the sanitized missing-configuration failure. The call incurs third-party cost and is intentionally absent from `pnpm test`, `pnpm validate`, Playwright, and CI. On failure, the CLI may print only the bounded safe diagnostic fields retained by the adapter. +The separately opted-in `pnpm smoke:openrouter` command makes one bounded OpenRouter call and validates the decision. It is not part of the default owner validation sequence and must be run only when the owner deliberately opts into a charged provider smoke. Node 24's built-in environment parser reads only `OPENROUTER_API_KEY` and `AGENTBORNE_MODEL` from the repository-root `.env` for this command, and file values override stale exported values for the smoke process. A missing file still yields the sanitized missing-configuration failure. The call incurs third-party cost and is intentionally absent from `pnpm test`, `pnpm validate`, Playwright, and CI. On failure, the CLI may print only the bounded safe diagnostic fields retained by the adapter. diff --git a/docs/adr/0004-server-owned-experiment-telemetry.md b/docs/adr/0004-server-owned-experiment-telemetry.md new file mode 100644 index 0000000..06d8195 --- /dev/null +++ b/docs/adr/0004-server-owned-experiment-telemetry.md @@ -0,0 +1,24 @@ +# ADR 0004: Server-owned safe experiment telemetry and tiered export + +- Status: Accepted +- Date: 2026-08-14 + +## Context + +Personality Lab experiments need enough safe behavioral and billing context for external analysis without changing model prompts, depending on the browser's bounded snapshot, or introducing persistence. Operators also need to distinguish actual known provider cost from unavailable accounting. + +## Decision + +The Game API owns one process-local experiment. Startup and World Reset generate a runtime-validated experiment UUID and capture the start time, initial six-agent configuration, initial world, immutable personality configuration events, and completed safe turns. It retains at most 5,000 complete experiment turns independently of the existing newest-120 browser snapshot. Absolute bounds, dropped count, and completeness disclose truncation. Reset preserves active personalities but discards the prior experiment and cost totals; restart loses all state. + +The agent runtime normalizes optional OpenRouter non-streaming usage fields and actual `usage.cost` as `costCredits`. No pricing table is used. Known metadata survives decision parsing/schema failures after a billable response; unavailable usage remains unknown. Explicit scripted providers report deterministic zero usage and cost. + +The Game API validates agent, turn, outcome, action, level, serialization, and Custom inclusion filters, computes metrics over the selected retained subset, and constructs schema-versioned Minimal, Standard, Full safe, or Custom documents. Provider costs use exact decimal-string accumulation before returning the JSON number, avoiding binary summation artifacts without discarding tiny charges. Full-safe world snapshots keep all agents and hex states as spatial context but omit event history; filtered top-level `worldEvents` is canonical. Compact JSON is the AI-sharing default; Pretty JSON is optional for human review. Preview serializes the prospective document using the selected encoding for an exact UTF-8 byte count and uses `ceil(bytes / 4)` as a clearly approximate model-agnostic AI-input-token estimate. Browser copy and download consume the same validated generated document and encoding. + +Captured and exported data excludes credentials, authorization headers, fixed prompts, raw provider payloads, private reasoning, and unbounded diagnostics. Export is rejected during concurrent simulation mutation. The world engine remains unaware of telemetry and billing. + +World Lab stops automatic playback and disables Start when all 61 cells are infected. This is a browser cost safeguard, not a world-engine win condition; Reset, export, and deliberate Single turn remain available. + +## Consequences + +Operators can inspect cost and safely share compact or rich experiment records without altering inference usage. Long experiments disclose loss beyond 5,000 retained records, and there is no recovery after reset or process restart. There is no database, historical experiment list, cloud sharing, automatic AI submission, hard budget, or account-balance integration. diff --git a/packages/agent-runtime/src/index.test.ts b/packages/agent-runtime/src/index.test.ts index d132346..fc3c27a 100644 --- a/packages/agent-runtime/src/index.test.ts +++ b/packages/agent-runtime/src/index.test.ts @@ -140,6 +140,132 @@ describe('OpenRouterAgentProvider', () => { expect(fetchImplementation).toHaveBeenCalledTimes(1); }); + it('normalizes complete OpenRouter usage accounting without rounding tiny cost', async () => { + const provider = new OpenRouterAgentProvider({ + apiKey: 'secret-test-key', + fetchImplementation: vi.fn( + async () => + new Response( + JSON.stringify({ + id: 'safe-id', + model: 'safe/model', + choices: [ + { + message: { + content: JSON.stringify({ + requestedAction: { type: 'wait' }, + summary: 'Wait.', + }), + }, + }, + ], + usage: { + prompt_tokens: 101, + completion_tokens: 23, + total_tokens: 124, + cost: 0.00000017, + completion_tokens_details: { reasoning_tokens: 7 }, + prompt_tokens_details: { + cached_tokens: 80, + cache_write_tokens: 4, + }, + }, + }), + ), + ), + }); + await expect(provider.decide(observation)).resolves.toMatchObject({ + metadata: { + promptTokens: 101, + completionTokens: 23, + totalTokens: 124, + reasoningTokens: 7, + cachedReadTokens: 80, + cacheWriteTokens: 4, + costCredits: 0.00000017, + }, + }); + }); + + it('supports a successful response with usage omitted', async () => { + const provider = new OpenRouterAgentProvider({ + apiKey: 'secret-test-key', + fetchImplementation: vi.fn( + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { + content: JSON.stringify({ + requestedAction: { type: 'wait' }, + summary: 'Wait.', + }), + }, + }, + ], + }), + ), + ), + }); + const result = await provider.decide(observation); + expect(result.metadata).not.toHaveProperty('costCredits'); + expect(result.metadata).not.toHaveProperty('totalTokens'); + }); + + it('never copies secrets, observations or raw provider payloads into safe metadata', async () => { + const provider = new OpenRouterAgentProvider({ + apiKey: 'sk-or-secret-test-key', + fetchImplementation: vi.fn( + async () => + new Response( + JSON.stringify({ + id: `Bearer sk-or-secret-test-key ${observation.personality}`, + model: JSON.stringify({ + observation, + authorization: 'sk-or-secret-test-key', + }), + choices: [ + { + message: { + content: JSON.stringify({ + requestedAction: { type: 'wait' }, + summary: 'Wait.', + }), + }, + }, + ], + usage: { cost: 0.00000001 }, + }), + ), + ), + }); + const result = await provider.decide(observation); + const serialized = JSON.stringify(result.metadata); + expect(serialized).not.toContain('sk-or-secret-test-key'); + expect(serialized).not.toContain(observation.personality); + expect(serialized).not.toContain('authorization'); + }); + + it.each([ + 'not-json', + JSON.stringify({ requestedAction: { type: 'teleport' }, summary: 'No.' }), + ])( + 'retains known safe usage when decision content is malformed or unsupported', + async (content) => { + const provider = new OpenRouterAgentProvider({ + apiKey: 'secret-test-key', + fetchImplementation: vi.fn(async () => response(content)), + }); + await expect(provider.decide(observation)).rejects.toMatchObject({ + metadata: { + promptTokens: 20, + completionTokens: 12, + }, + }); + }, + ); + it.each([ ['not-json', 'malformed-response'], [ @@ -371,7 +497,13 @@ describe('ScriptedAgentProvider', () => { ]); await expect(provider.decide(observation)).resolves.toMatchObject({ decision: { requestedAction: { type: 'wait' } }, - metadata: { provider: 'scripted-test' }, + metadata: { + provider: 'scripted-test', + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + costCredits: 0, + }, }); await expect(provider.decide(observation)).resolves.toMatchObject({ decision: { requestedAction: { type: 'infect' } }, diff --git a/packages/agent-runtime/src/index.ts b/packages/agent-runtime/src/index.ts index 5987b05..ca69ba9 100644 --- a/packages/agent-runtime/src/index.ts +++ b/packages/agent-runtime/src/index.ts @@ -138,11 +138,24 @@ const openRouterResponseSchema = z.object({ message: z.object({ content: z.string().nullable() }), }), ), - usage: z + usage: z.unknown().optional(), +}); + +const openRouterUsageSchema = z.object({ + prompt_tokens: z.number().int().nonnegative().optional(), + completion_tokens: z.number().int().nonnegative().optional(), + total_tokens: z.number().int().nonnegative().optional(), + cost: z.number().nonnegative().finite().optional(), + completion_tokens_details: z + .object({ reasoning_tokens: z.number().int().nonnegative().optional() }) + .passthrough() + .optional(), + prompt_tokens_details: z .object({ - prompt_tokens: z.number().int().nonnegative().optional(), - completion_tokens: z.number().int().nonnegative().optional(), + cached_tokens: z.number().int().nonnegative().optional(), + cache_write_tokens: z.number().int().nonnegative().optional(), }) + .passthrough() .optional(), }); @@ -246,54 +259,71 @@ export class OpenRouterAgentProvider implements AgentProvider { } const parsedResponse = openRouterResponseSchema.safeParse(raw); + const safeMetadata = providerMetadataFromResponse( + raw, + response, + this.model, + Date.now() - started, + collectSensitiveValues({ + apiKey: this.#apiKey, + observation, + additionalValues: [ + requestBody, + ...providerRequest.messages.map(({ content }) => content), + ], + }), + ); if (!parsedResponse.success) { - throw new AgentProviderError({ - code: 'unsupported-response', - message: 'The model provider returned an unsupported response shape.', - retryable: true, - }); + throw new AgentProviderError( + { + code: 'unsupported-response', + message: + 'The model provider returned an unsupported response shape.', + retryable: true, + }, + safeMetadata, + ); } const content = parsedResponse.data.choices[0]?.message.content; if (!content) { - throw new AgentProviderError({ - code: 'unsupported-response', - message: 'The model provider returned no structured decision.', - retryable: true, - }); + throw new AgentProviderError( + { + code: 'unsupported-response', + message: 'The model provider returned no structured decision.', + retryable: true, + }, + safeMetadata, + ); } let decisionInput: unknown; try { decisionInput = JSON.parse(content); } catch { - throw new AgentProviderError({ - code: 'malformed-response', - message: 'The model decision was not valid JSON.', - retryable: true, - }); + throw new AgentProviderError( + { + code: 'malformed-response', + message: 'The model decision was not valid JSON.', + retryable: true, + }, + safeMetadata, + ); } const decision = agentDecisionSchema.safeParse(decisionInput); if (!decision.success) { - throw new AgentProviderError({ - code: 'unsupported-response', - message: 'The model decision failed runtime schema validation.', - retryable: true, - }); + throw new AgentProviderError( + { + code: 'unsupported-response', + message: 'The model decision failed runtime schema validation.', + retryable: true, + }, + safeMetadata, + ); } return { decision: decision.data, - metadata: { - provider: 'openrouter', - model: parsedResponse.data.model ?? this.model, - requestId: - parsedResponse.data.id ?? - response.headers.get('x-request-id') ?? - undefined, - latencyMs: Date.now() - started, - promptTokens: parsedResponse.data.usage?.prompt_tokens, - completionTokens: parsedResponse.data.usage?.completion_tokens, - }, + metadata: safeMetadata, }; } finally { clearTimeout(timeout); @@ -301,6 +331,61 @@ export class OpenRouterAgentProvider implements AgentProvider { } } +function providerMetadataFromResponse( + raw: unknown, + response: Response, + fallbackModel: string, + latencyMs: number, + sensitiveValues: string[], +): ProviderMetadata { + const root = asRecord(raw); + const usage = openRouterUsageSchema.safeParse(root?.usage); + const parsedUsage = usage.success ? usage.data : undefined; + const requestId = + sanitizeDiagnosticCode(root?.id, sensitiveValues, 160) ?? + sanitizeDiagnosticCode( + response.headers.get('x-request-id'), + sensitiveValues, + 160, + ); + return { + provider: 'openrouter', + model: + sanitizeDiagnosticMessage(root?.model, sensitiveValues, 120) ?? + sanitizeDiagnosticMessage(fallbackModel, sensitiveValues, 120) ?? + DEFAULT_OPENROUTER_MODEL, + latencyMs, + ...(requestId === undefined ? {} : { requestId }), + ...(parsedUsage?.prompt_tokens === undefined + ? {} + : { promptTokens: parsedUsage.prompt_tokens }), + ...(parsedUsage?.completion_tokens === undefined + ? {} + : { completionTokens: parsedUsage.completion_tokens }), + ...(parsedUsage?.total_tokens === undefined + ? {} + : { totalTokens: parsedUsage.total_tokens }), + ...(parsedUsage?.completion_tokens_details?.reasoning_tokens === undefined + ? {} + : { + reasoningTokens: + parsedUsage.completion_tokens_details.reasoning_tokens, + }), + ...(parsedUsage?.prompt_tokens_details?.cached_tokens === undefined + ? {} + : { cachedReadTokens: parsedUsage.prompt_tokens_details.cached_tokens }), + ...(parsedUsage?.prompt_tokens_details?.cache_write_tokens === undefined + ? {} + : { + cacheWriteTokens: + parsedUsage.prompt_tokens_details.cache_write_tokens, + }), + ...(parsedUsage?.cost === undefined + ? {} + : { costCredits: parsedUsage.cost }), + }; +} + function requestFailure(timedOut: boolean): AgentProviderError { return new AgentProviderError({ code: timedOut ? 'timeout' : 'network', @@ -526,6 +611,13 @@ export class ScriptedAgentProvider implements AgentProvider { provider: 'scripted-test', model: this.model, latencyMs: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + reasoningTokens: 0, + cachedReadTokens: 0, + cacheWriteTokens: 0, + costCredits: 0, }, }; } @@ -558,6 +650,13 @@ export class BrowserTestAgentProvider implements AgentProvider { provider: 'scripted-test', model: this.model, latencyMs: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + reasoningTokens: 0, + cachedReadTokens: 0, + cacheWriteTokens: 0, + costCredits: 0, }, }; } diff --git a/packages/shared/src/index.test.ts b/packages/shared/src/index.test.ts index db4d39f..ee8f265 100644 --- a/packages/shared/src/index.test.ts +++ b/packages/shared/src/index.test.ts @@ -6,6 +6,10 @@ import { agentDecisionSchema, agentObservationSchema, agentTurnRecordSchema, + experimentExportRequestSchema, + experimentIdSchema, + personalityConfigurationEventSchema, + providerMetadataSchema, restoreDefaultPersonalitiesResponseSchema, simulationSnapshotSchema, updateAgentPersonalityRequestSchema, @@ -64,6 +68,32 @@ const snapshot = { providerMode: 'openrouter', providerConfigured: true, turns: [], + experiment: { + id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + startedAt: '2026-08-13T12:00:00.000Z', + totalCompletedTurns: 0, + retainedTurns: 0, + droppedRecords: 0, + complete: true, + metrics: { + aggregate: { + totalTurns: 0, + accepted: 0, + rejected: 0, + providerErrors: 0, + requestedMoves: 0, + requestedInfections: 0, + requestedWaits: 0, + acceptedMovements: 0, + successfullyInfectedCells: 0, + uniqueVisitedCells: 0, + tokens: {}, + knownCostCredits: 0, + turnsWithUnknownCost: 0, + }, + byAgent: [], + }, + }, }; describe('agent observation and decision schemas', () => { @@ -179,6 +209,99 @@ describe('turn and snapshot schemas', () => { }); }); +describe('experiment telemetry and export contracts', () => { + it('accepts complete, partial and tiny-cost provider usage without fabricating unknowns', () => { + expect( + providerMetadataSchema.parse({ + ...provider, + promptTokens: 12, + completionTokens: 3, + totalTokens: 15, + reasoningTokens: 1, + cachedReadTokens: 8, + cacheWriteTokens: 2, + costCredits: 0.00000001, + }).costCredits, + ).toBe(0.00000001); + expect(providerMetadataSchema.parse(provider)).not.toHaveProperty( + 'costCredits', + ); + }); + + it('validates experiment identities and immutable configuration events', () => { + expect(experimentIdSchema.safeParse('not-an-id').success).toBe(false); + expect( + personalityConfigurationEventSchema.safeParse({ + timestamp: '2026-08-13T12:00:00.000Z', + agentId, + previousPersonality: 'Before.', + newPersonality: 'After.', + operation: 'custom-edit', + }).success, + ).toBe(true); + }); + + it('validates all levels and rejects empty, malformed, duplicate and inverted selections', () => { + const base = { + agents: { mode: 'selected', agentIds: [agentId] }, + turns: { mode: 'entire-retained' }, + outcomes: ['accepted'], + actions: ['wait'], + }; + for (const level of ['minimal', 'standard', 'full-safe']) + expect( + experimentExportRequestSchema.safeParse({ ...base, level }).success, + ).toBe(true); + expect( + experimentExportRequestSchema.parse({ ...base, level: 'minimal' }) + .serialization, + ).toBe('compact'); + expect( + experimentExportRequestSchema.safeParse({ + ...base, + level: 'minimal', + serialization: 'pretty', + }).success, + ).toBe(true); + expect( + experimentExportRequestSchema.safeParse({ + ...base, + level: 'custom', + custom: { + turnObservations: false, + personalityTextHistory: false, + nearbyAgents: false, + recentEvents: false, + validationDetails: false, + resultingEvents: false, + providerUsageMetadata: false, + initialWorldState: false, + currentWorldState: false, + computedMetrics: false, + }, + }).success, + ).toBe(true); + for (const invalid of [ + { ...base, agents: { mode: 'selected', agentIds: [] }, level: 'minimal' }, + { ...base, outcomes: [], level: 'minimal' }, + { ...base, actions: [], level: 'minimal' }, + { + ...base, + turns: { mode: 'range', fromTurn: 9, toTurn: 2 }, + level: 'minimal', + }, + { + ...base, + agents: { mode: 'selected', agentIds: ['bad-id'] }, + level: 'minimal', + }, + ]) + expect(experimentExportRequestSchema.safeParse(invalid).success).toBe( + false, + ); + }); +}); + describe('personality mutation contracts', () => { it('trims a valid update and validates its response', () => { const request = updateAgentPersonalityRequestSchema.parse({ diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index c97c320..60a5578 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -168,6 +168,11 @@ export const providerMetadataSchema = z.object({ latencyMs: z.number().int().nonnegative().max(300_000), promptTokens: z.number().int().nonnegative().optional(), completionTokens: z.number().int().nonnegative().optional(), + totalTokens: z.number().int().nonnegative().optional(), + reasoningTokens: z.number().int().nonnegative().optional(), + cachedReadTokens: z.number().int().nonnegative().optional(), + cacheWriteTokens: z.number().int().nonnegative().optional(), + costCredits: z.number().nonnegative().finite().optional(), }); export type ProviderMetadata = z.infer; @@ -242,6 +247,17 @@ export const simulationSnapshotSchema = z.object({ providerMode: providerModeSchema, providerConfigured: z.boolean(), turns: z.array(agentTurnRecordSchema).max(120), + experiment: z.object({ + id: z.uuid().brand<'ExperimentId'>(), + startedAt: z.iso.datetime(), + totalCompletedTurns: z.number().int().nonnegative(), + retainedTurns: z.number().int().nonnegative(), + firstRetainedTurn: z.number().int().positive().optional(), + lastRetainedTurn: z.number().int().positive().optional(), + droppedRecords: z.number().int().nonnegative(), + complete: z.boolean(), + metrics: z.lazy(() => experimentMetricsSchema), + }), }); export type SimulationSnapshot = z.infer; @@ -288,6 +304,9 @@ export const apiErrorCodeSchema = z.enum([ 'unknown_agent', 'invalid_personality', 'invalid_request', + 'invalid_export', + 'export_conflict', + 'records_unavailable', 'not_found', 'internal_error', ]); @@ -300,3 +319,347 @@ export const apiErrorSchema = z.object({ }), }); export type ApiError = z.infer; + +export const experimentIdSchema = z.uuid().brand<'ExperimentId'>(); +export type ExperimentId = z.infer; + +export const personalityConfigurationEventSchema = z.object({ + timestamp: z.iso.datetime(), + agentId: agentIdSchema, + previousPersonality: personalitySchema, + newPersonality: personalitySchema, + operation: z.enum(['custom-edit', 'restore-default']), +}); +export type PersonalityConfigurationEvent = z.infer< + typeof personalityConfigurationEventSchema +>; + +export const experimentManifestSchema = z.object({ + id: experimentIdSchema, + startedAt: z.iso.datetime(), + generatedAt: z.iso.datetime().optional(), + providerMode: providerModeSchema, + initialAgents: z.array(agentProfileSchema).length(6).optional(), +}); +export type ExperimentManifest = z.infer; + +export const experimentRetentionSchema = z.object({ + limit: z.number().int().positive(), + totalCompletedTurns: z.number().int().nonnegative(), + retainedTurns: z.number().int().nonnegative(), + firstRetainedTurn: z.number().int().positive().optional(), + lastRetainedTurn: z.number().int().positive().optional(), + droppedRecords: z.number().int().nonnegative(), + complete: z.boolean(), + requestedRangeExtendsBeyondRetention: z.boolean(), +}); +export type ExperimentRetention = z.infer; + +export const tokenTotalsSchema = z.object({ + promptTokens: z.number().int().nonnegative().optional(), + completionTokens: z.number().int().nonnegative().optional(), + totalTokens: z.number().int().nonnegative().optional(), + reasoningTokens: z.number().int().nonnegative().optional(), + cachedReadTokens: z.number().int().nonnegative().optional(), + cacheWriteTokens: z.number().int().nonnegative().optional(), +}); + +export const metricCountsSchema = z.object({ + totalTurns: z.number().int().nonnegative(), + accepted: z.number().int().nonnegative(), + rejected: z.number().int().nonnegative(), + providerErrors: z.number().int().nonnegative(), + requestedMoves: z.number().int().nonnegative(), + requestedInfections: z.number().int().nonnegative(), + requestedWaits: z.number().int().nonnegative(), + acceptedMovements: z.number().int().nonnegative(), + successfullyInfectedCells: z.number().int().nonnegative(), + uniqueVisitedCells: z.number().int().nonnegative(), + averageLatencyMs: z.number().nonnegative().optional(), + tokens: tokenTotalsSchema, + knownCostCredits: z.number().nonnegative().finite(), + turnsWithUnknownCost: z.number().int().nonnegative(), +}); +export type MetricCounts = z.infer; + +export const experimentMetricsSchema = z.object({ + aggregate: metricCountsSchema, + byAgent: z.array( + z.object({ agentId: agentIdSchema, metrics: metricCountsSchema }), + ), +}); +export type ExperimentMetrics = z.infer; + +export const exportOutcomeSchema = z.enum([ + 'accepted', + 'rejected', + 'provider-error', +]); +export const exportActionSchema = z.enum(['move', 'infect', 'wait']); +export const exportLevelSchema = z.enum([ + 'minimal', + 'standard', + 'full-safe', + 'custom', +]); +export const exportSerializationSchema = z.enum(['compact', 'pretty']); + +export const customExportOptionsSchema = z + .object({ + turnObservations: z.boolean(), + personalityTextHistory: z.boolean(), + nearbyAgents: z.boolean(), + recentEvents: z.boolean(), + validationDetails: z.boolean(), + resultingEvents: z.boolean(), + providerUsageMetadata: z.boolean(), + initialWorldState: z.boolean(), + currentWorldState: z.boolean(), + computedMetrics: z.boolean(), + }) + .strict() + .superRefine((value, context) => { + if (!value.turnObservations && (value.nearbyAgents || value.recentEvents)) { + context.addIssue({ + code: 'custom', + message: 'Nearby agents and recent events require turn observations.', + }); + } + }); +export type CustomExportOptions = z.infer; + +const exportSelectionSchema = z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('all') }).strict(), + z + .object({ + mode: z.literal('selected'), + agentIds: z.array(agentIdSchema).min(1).max(6), + }) + .strict() + .refine((value) => new Set(value.agentIds).size === value.agentIds.length, { + message: 'Agent IDs must be unique.', + }), +]); + +const exportTurnSelectionSchema = z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('entire-retained') }).strict(), + z + .object({ + mode: z.literal('latest'), + count: z.union([ + z.literal(10), + z.literal(25), + z.literal(50), + z.literal(120), + ]), + }) + .strict(), + z + .object({ + mode: z.literal('range'), + fromTurn: z.number().int().positive(), + toTurn: z.number().int().positive(), + }) + .strict() + .refine((value) => value.fromTurn <= value.toTurn, { + message: 'The first turn must not exceed the last turn.', + }), +]); + +export const experimentExportRequestSchema = z + .object({ + agents: exportSelectionSchema, + turns: exportTurnSelectionSchema, + outcomes: z.array(exportOutcomeSchema).min(1).max(3), + actions: z.array(exportActionSchema).min(1).max(3), + level: exportLevelSchema, + serialization: exportSerializationSchema.default('compact'), + custom: customExportOptionsSchema.optional(), + }) + .strict() + .superRefine((value, context) => { + if (new Set(value.outcomes).size !== value.outcomes.length) + context.addIssue({ code: 'custom', message: 'Outcomes must be unique.' }); + if (new Set(value.actions).size !== value.actions.length) + context.addIssue({ code: 'custom', message: 'Actions must be unique.' }); + if (value.level === 'custom' && !value.custom) + context.addIssue({ + code: 'custom', + message: 'Custom options are required.', + }); + if (value.level !== 'custom' && value.custom) + context.addIssue({ + code: 'custom', + message: 'Custom options are only valid for Custom exports.', + }); + }); +export type ExperimentExportRequest = z.infer< + typeof experimentExportRequestSchema +>; + +export const experimentExportPreviewSchema = z.object({ + experimentId: experimentIdSchema, + matchingRecordCount: z.number().int().nonnegative(), + selectedAgentCount: z.number().int().positive().max(6), + firstMatchingTurn: z.number().int().positive().optional(), + lastMatchingTurn: z.number().int().positive().optional(), + retention: experimentRetentionSchema, + knownCostCredits: z.number().nonnegative().finite(), + turnsWithUnknownCost: z.number().int().nonnegative(), + serializedUtf8Bytes: z.number().int().nonnegative(), + approximateAiInputTokens: z.number().int().nonnegative(), + tokenEstimateMethod: z.literal('ceil(UTF-8 bytes / 4)'), +}); +export type ExperimentExportPreview = z.infer< + typeof experimentExportPreviewSchema +>; + +export const experimentExportTurnSchema = z.object({ + turnNumber: z.number().int().positive(), + startedAt: z.iso.datetime(), + completedAt: z.iso.datetime(), + agentId: agentIdSchema, + outcome: exportOutcomeSchema, + requestedAction: agentMapActionSchema.optional(), + summary: z.string().trim().min(1).max(MODEL_SUMMARY_MAX_LENGTH).optional(), + validationReason: invalidActionReasonSchema.optional(), + eventSummary: z.string().trim().min(1).max(300).optional(), + personality: personalitySchema.optional(), + observation: agentObservationSchema.partial().optional(), + validation: z + .union([ + z.object({ accepted: z.literal(true) }), + z.object({ + accepted: z.literal(false), + reason: invalidActionReasonSchema, + details: z.string().min(1).max(300), + }), + ]) + .optional(), + event: worldEventSchema.optional(), + failure: providerFailureSchema.optional(), + provider: providerMetadataSchema.optional(), +}); + +export const experimentExportWorldStateSchema = worldSnapshotSchema.omit({ + events: true, +}); +export type ExperimentExportWorldState = z.infer< + typeof experimentExportWorldStateSchema +>; + +export const experimentExportDocumentSchema = z + .object({ + schemaVersion: z.literal(1), + generatedAt: z.iso.datetime(), + experiment: experimentManifestSchema, + retention: experimentRetentionSchema, + filters: experimentExportRequestSchema, + selection: z.object({ + selectedAgentIds: z.array(agentIdSchema).min(1).max(6), + matchingRecordCount: z.number().int().nonnegative(), + firstMatchingTurn: z.number().int().positive().optional(), + lastMatchingTurn: z.number().int().positive().optional(), + }), + agents: z + .array( + agentProfileSchema.omit({ personality: true }).extend({ + personality: personalitySchema.optional(), + }), + ) + .min(1) + .max(6), + configurationEvents: z + .array(personalityConfigurationEventSchema) + .optional(), + metrics: experimentMetricsSchema.optional(), + initialWorld: experimentExportWorldStateSchema.optional(), + currentWorld: experimentExportWorldStateSchema.optional(), + worldEvents: z.array(worldEventSchema).optional(), + turns: z.array(experimentExportTurnSchema), + }) + .superRefine((document, context) => { + const level = document.filters.level; + const custom = level === 'custom' ? document.filters.custom : undefined; + const requiresMetrics = level !== 'custom' || custom?.computedMetrics; + if (Boolean(document.metrics) !== Boolean(requiresMetrics)) + context.addIssue({ + code: 'custom', + message: 'Metrics inclusion does not match the export level.', + }); + const personalityHistory = + level === 'full-safe' || custom?.personalityTextHistory; + if (Boolean(document.configurationEvents) !== Boolean(personalityHistory)) + context.addIssue({ + code: 'custom', + message: + 'Personality history inclusion does not match the export level.', + }); + const initialWorld = level === 'full-safe' || custom?.initialWorldState; + const currentWorld = level === 'full-safe' || custom?.currentWorldState; + if ( + Boolean(document.initialWorld) !== Boolean(initialWorld) || + Boolean(document.currentWorld) !== Boolean(currentWorld) + ) + context.addIssue({ + code: 'custom', + message: 'World-state inclusion does not match the export level.', + }); + if (Boolean(document.worldEvents) !== (level === 'full-safe')) + context.addIssue({ + code: 'custom', + message: 'World event inclusion does not match the export level.', + }); + for (const turn of document.turns) { + const observation = + level === 'standard' || + level === 'full-safe' || + custom?.turnObservations; + const personality = + level === 'standard' || + level === 'full-safe' || + custom?.personalityTextHistory; + const validation = + level === 'standard' || + level === 'full-safe' || + custom?.validationDetails; + const event = + level === 'standard' || + level === 'full-safe' || + custom?.resultingEvents; + const provider = level !== 'custom' || custom?.providerUsageMetadata; + if ( + Boolean(turn.observation) !== Boolean(observation) || + Boolean(turn.personality) !== Boolean(personality) + ) + context.addIssue({ + code: 'custom', + message: 'Turn context inclusion does not match the export level.', + }); + if ( + turn.outcome !== 'provider-error' && + Boolean(turn.validation) !== Boolean(validation) + ) + context.addIssue({ + code: 'custom', + message: 'Validation inclusion does not match the export level.', + }); + if (turn.outcome === 'accepted' && Boolean(turn.event) !== Boolean(event)) + context.addIssue({ + code: 'custom', + message: 'Event inclusion does not match the export level.', + }); + if (!provider && turn.provider) + context.addIssue({ + code: 'custom', + message: 'Provider inclusion does not match the export level.', + }); + } + }); +export type ExperimentExportDocument = z.infer< + typeof experimentExportDocumentSchema +>; + +export const experimentExportResponseSchema = z.object({ + document: experimentExportDocumentSchema, +}); diff --git a/tests/e2e/world-lab.spec.ts b/tests/e2e/world-lab.spec.ts index 53d1b55..ecd5e78 100644 --- a/tests/e2e/world-lab.spec.ts +++ b/tests/e2e/world-lab.spec.ts @@ -1,4 +1,6 @@ import { expect, test } from '@playwright/test'; +import { readFile } from 'node:fs/promises'; +import { experimentExportDocumentSchema } from '@agentborne/shared'; test('runs the complete deterministic World Lab browser flow', async ({ page, @@ -89,6 +91,36 @@ test('runs the complete deterministic World Lab browser flow', async ({ '1 rendered infected', ); + await page.getByRole('button', { name: 'Single turn' }).click(); + await page.getByRole('button', { name: 'Single turn' }).click(); + await expect(page.getByText('Turn 3', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Select agent Ember' }).click(); + await page.getByRole('button', { name: 'Export this agent' }).click(); + await expect(page.getByRole('checkbox', { name: /Ember/ })).toBeChecked(); + await page.getByRole('button', { name: 'Preview export' }).click(); + await expect(page.getByLabel('Export preview')).toContainText('1 records'); + await page.getByRole('checkbox', { name: /Rook/ }).check(); + await page.getByRole('button', { name: 'Generate JSON' }).click(); + await expect(page.getByText(/schema-validated/)).toBeVisible(); + const downloadPromise = page.waitForEvent('download'); + await page.getByRole('button', { name: 'Download JSON' }).click(); + const download = await downloadPromise; + const downloadedPath = await download.path(); + expect(downloadedPath).not.toBeNull(); + const exported = experimentExportDocumentSchema.parse( + JSON.parse(await readFile(downloadedPath!, 'utf8')), + ); + expect(exported.filters.level).toBe('minimal'); + expect(exported.selection.selectedAgentIds).toHaveLength(2); + expect(exported.metrics?.aggregate.knownCostCredits).toBe(0); + expect(exported.metrics?.aggregate.turnsWithUnknownCost).toBe(0); + expect(exported.turns.map(({ turnNumber }) => turnNumber)).toEqual( + exported.turns + .map(({ turnNumber }) => turnNumber) + .toSorted((a, b) => a - b), + ); + + page.once('dialog', (dialog) => dialog.accept()); await page.getByRole('button', { name: 'Reset world' }).click(); await expect(page.getByText('Turn 0')).toBeVisible(); await expect(