refactor: make experiment telemetry swarm-native

apps/game-api/src/experiment-export.ts

@@ -28,7 +28,7 @@ import {
 } from '@hexzero/shared';
 
 export interface ExperimentSource {
-  schemaVersion: 9 | 10 | 11;
+  schemaVersion: 12;
   id: ExperimentId;
   startedAt: string;
   providerMode: 'openrouter' | 'scripted-test';
@@ -41,11 +41,11 @@ export interface ExperimentSource {
   currentWorld: WorldSnapshot;
   modelConfiguration: ExperimentModelConfiguration;
   scenario: AppliedScenario;
-  swarmTicks?: readonly SwarmTickRecord[];
+  swarmTicks: readonly SwarmTickRecord[];
   simulatedPlayerEvents: readonly SimulatedPlayerEvent[];
-  providerAttempts?: readonly ProviderAttemptRecord[];
-  attemptRetention?: ProviderAttemptRetention;
-  attemptAccounting?: ExperimentAttemptAccounting;
+  providerAttempts: readonly ProviderAttemptRecord[];
+  attemptRetention: ProviderAttemptRetention;
+  attemptAccounting: ExperimentAttemptAccounting;
 }
 
 export class ExperimentExportValidationError extends Error {
@@ -105,9 +105,7 @@ function selectTickNumbers(
   request: ExperimentExportRequest,
 ): Set<number> | 'all' {
   if (request.turns.mode === 'entire-retained') return 'all';
-  const allTicks = (source.swarmTicks ?? []).map(
-    ({ tickNumber }) => tickNumber,
-  );
+  const allTicks = source.swarmTicks.map(({ tickNumber }) => tickNumber);
   if (request.turns.mode === 'range') {
     const { fromTurn, toTurn } = request.turns;
     return new Set(allTicks.filter((n) => n >= fromTurn && n <= toTurn));
@@ -121,7 +119,7 @@ function requestFilteredActions(
   tickNumbers: Set<number> | 'all',
 ): ResolvedWorldAction[] {
   const zeroAgentId = source.scenario.patientZeroAgentId;
-  const all = resolvedActionsFromTicks(source.swarmTicks ?? [], zeroAgentId);
+  const all = resolvedActionsFromTicks(source.swarmTicks, zeroAgentId);
   return all.filter(
     ({ tickNumber, action, actionResult }) =>
       request.outcomes.includes(
@@ -162,9 +160,9 @@ export function createExperimentExport(
     selectedSet,
     tickNumbers,
   );
-  const retainedTicks = source.swarmTicks?.length ?? 0;
-  const firstRetainedTick = source.swarmTicks?.[0]?.tickNumber;
-  const lastRetainedTick = source.swarmTicks?.at(-1)?.tickNumber;
+  const retainedTicks = source.swarmTicks.length;
+  const firstRetainedTick = source.swarmTicks[0]?.tickNumber;
+  const lastRetainedTick = source.swarmTicks.at(-1)?.tickNumber;
   const droppedRecords = source.totalCompletedTicks - retainedTicks;
   const retention = {
     limit: source.retentionLimit,
@@ -186,7 +184,7 @@ export function createExperimentExport(
     source.scenario.swarmArchitectureVersion === 'zero-swarm-v1' &&
     request.agents.mode === 'all' &&
     request.turns.mode === 'entire-retained'
-      ? [...structuredClone(source.swarmTicks ?? [])]
+      ? [...structuredClone(source.swarmTicks)]
       : undefined;
   const simulatedPlayerEventsIncluded = source.simulatedPlayerEvents.filter(
     (event) =>
@@ -226,20 +224,14 @@ export function createExperimentExport(
     filters: structuredClone(request),
     selection: {
       selectedAgentIds,
-      ...(source.schemaVersion === 10 || source.schemaVersion === 11
-        ? {
-            matchingTickCount:
-              exportedSwarmTicks?.length ??
-              new Set(agentFiltered.map(({ tickNumber }) => tickNumber)).size,
-          }
-        : {}),
+      matchingTickCount:
+        exportedSwarmTicks?.length ??
+        new Set(agentFiltered.map(({ tickNumber }) => tickNumber)).size,
       ...(exportedSwarmTicks
         ? { matchingSwarmTickCount: exportedSwarmTicks.length }
         : {}),
       matchingControlChangeCount: controlChanges.length,
-      ...(source.schemaVersion === 11
-        ? { matchingProviderAttemptCount: providerAttempts.length }
-        : {}),
+      matchingProviderAttemptCount: providerAttempts.length,
       matchingSimulatedPlayerEventCount: simulatedPlayerEventsIncluded.length,
     },
     agents: selectedAgents,
@@ -293,28 +285,19 @@ export function createExperimentExport(
       ? { controlChanges: structuredClone(controlChanges) }
       : {}),
     ...(exportedSwarmTicks ? { swarmTicks: exportedSwarmTicks } : {}),
-    ...(source.schemaVersion === 10 || source.schemaVersion === 11
-      ? {
-          tickSummaries: summarizeTicks(
-            (source.swarmTicks ?? []).filter(
-              (tick) =>
-                tickNumbers === 'all' || tickNumbers.has(tick.tickNumber),
-            ),
-            providerAttempts,
-          ),
-        }
-      : {}),
-    ...(source.schemaVersion === 11
-      ? {
-          providerAttempts,
-          attemptRetention: {
-            ...source.attemptRetention!,
-            requestedRangeExtendsBeyondRetention:
-              source.attemptRetention!.droppedRecords > 0,
-          },
-          attemptAccounting: source.attemptAccounting!,
-        }
-      : {}),
+    tickSummaries: summarizeTicks(
+      source.swarmTicks.filter(
+        (tick) => tickNumbers === 'all' || tickNumbers.has(tick.tickNumber),
+      ),
+      providerAttempts,
+    ),
+    providerAttempts,
+    attemptRetention: {
+      ...source.attemptRetention,
+      requestedRangeExtendsBeyondRetention:
+        source.attemptRetention.droppedRecords > 0,
+    },
+    attemptAccounting: source.attemptAccounting,
   };
   return experimentExportDocumentSchema.parse(document);
 }
@@ -362,9 +345,7 @@ export function createExperimentPreview(
   ).length;
   return experimentExportPreviewSchema.parse({
     experimentId: source.id,
-    ...(document.selection.matchingTickCount === undefined
-      ? {}
-      : { matchingTickCount: document.selection.matchingTickCount }),
+    matchingTickCount: document.selection.matchingTickCount,
     ...(document.selection.matchingSwarmTickCount === undefined
       ? {}
       : { matchingSwarmTickCount: document.selection.matchingSwarmTickCount }),
@@ -373,14 +354,8 @@ export function createExperimentPreview(
       document.selection.matchingProviderAttemptCount ?? 0,
     selectedAgentCount: document.selection.selectedAgentIds.length,
     retention: document.retention,
-    knownCostCredits:
-      document.schemaVersion === 11
-        ? ledgerKnownCost
-        : (document.metrics?.aggregate.knownCostCredits ?? 0),
-    attemptsWithUnknownCost:
-      document.schemaVersion === 11
-        ? ledgerUnknownCost
-        : (document.metrics?.aggregate.attemptsWithUnknownCost ?? 0),
+    knownCostCredits: ledgerKnownCost,
+    attemptsWithUnknownCost: ledgerUnknownCost,
     serializedUtf8Bytes,
     approximateAiInputTokens: Math.ceil(serializedUtf8Bytes / 4),
     tokenEstimateMethod: 'ceil(UTF-8 bytes / 4)',
@@ -411,7 +386,7 @@ function filterProviderAttempts(
   selected: Set<AgentId>,
   tickNumbers: Set<number> | 'all',
 ): ProviderAttemptRecord[] {
-  let attempts = (source.providerAttempts ?? []).filter(({ agentId }) =>
+  let attempts = source.providerAttempts.filter(({ agentId }) =>
     selected.has(agentId),
   );
   if (tickNumbers !== 'all')
@@ -457,7 +432,7 @@ function rangeExtendsBeyondRetention(
   last?: number,
 ): boolean {
   if (request.turns.mode === 'entire-retained')
-    return source.totalCompletedTicks > (source.swarmTicks?.length ?? 0);
+    return source.totalCompletedTicks > source.swarmTicks.length;
   if (request.turns.mode !== 'range') return false;
   if (!first || !last) return true;
   return request.turns.fromTurn < first || request.turns.toTurn > last;

apps/game-api/src/simulation-service.ts

@@ -1654,7 +1654,7 @@ export class SimulationService {
       currentWorld: this.#worldSnapshot(),
       modelConfiguration: this.#modelConfiguration,
       scenario: this.#scenario,
-      schemaVersion: 11,
+      schemaVersion: 12,
       providerAttempts: this.#attemptAccounting.ledger(),
       attemptRetention: this.#attemptAccounting.retention(),
       attemptAccounting: this.#attemptAccounting.snapshot(),

docs/EXPERIMENT_ARCHIVE.md

@@ -1,23 +1,22 @@
 # Local experiment archive
 
-The archive accepts current schema-v11 exports and legacy schema-v9/v10 exports.
-Schema-v10 may add safe goal revision/result and current-goal fields. Current archive imports validate and preserve compatibility while normalized goal analytics remain deferred; the archive never restores active goal state.
-Schema-v10 may also include compact memory requests, results, and current ledgers. The archive accepts these additive fields observationally but does not normalize, rank, retrieve, or restore memory.
+> **Note:** this document predates the zero-swarm migration and still describes
+> several removed legacy features (per-agent communications, diplomacy,
+> alliances, and the schema-v9/v10/v11 compatibility described below). The
+> archive now accepts only schema-v12 exports and rejects anything else; see
+> ADR 0033. A full rewrite of this document is tracked separately.
+
+The archive accepts only schema-v12 exports and rejects any other schema version outright.
 Migration 2 adds nullable tick number, deterministic tick position, virtual
-time, and interval columns so legacy rows remain valid. Schema-v10 lost-tick
-outcomes are preserved as source outcomes; canonical summaries continue to be
-derived from normalized rows and remain distinct from source metrics. Bounded
-queries order tick-attributed records by tick and tick position where exposed;
-the CLI still provides no arbitrary SQL surface.
-Legacy scenarios with a null Patient Zero designation remain valid historical
-records. They retain null attribution and Patient Zero queries return no
-coordinator activity; current live setup requirements do not rewrite them.
+time, and interval columns. Bounded queries order tick-attributed records by
+tick and tick position where exposed; the CLI still provides no arbitrary SQL
+surface.
 Migration 3 adds aggregate simulated-player metrics to experiments and the
 `simulated_player_activity` table. Every metrics-bearing safe export preserves
 movement/clean/block totals; Full Safe additionally preserves tick-attributed
 activity without deriving player behavior from agent turns.
 
-The experiment archive is a durable, local research surface for completed or partially retained exports. It does not participate in an active simulation: the Game API's in-memory engine remains authoritative, and an archive write cannot change an accepted game outcome. It imports schema-v10 and compatible schema-v9 JSON exports; it is not crash recovery, restartable simulation state, or a scheduler.
+The experiment archive is a durable, local research surface for completed or partially retained exports. It does not participate in an active simulation: the Game API's in-memory engine remains authoritative, and an archive write cannot change an accepted game outcome. It imports schema-v12 JSON exports only; it is not crash recovery, restartable simulation state, or a scheduler.
 
 ## Storage and configuration
 
@@ -37,7 +36,7 @@ For an optional manual migration, stop all Hex Zero processes, create the
 `-shm` sidecars from `.agentborne`, verify the copied archive opens, and only
 then remove the legacy files if desired.
 
-The schema normalizes experiments, source exports, roster/assignments, topology, turns, model attempts, communications and recipients, diplomacy attempts, alliance events, world events, configuration changes, and research notes. Stable source IDs are retained; deterministic experiment/turn/attempt IDs make imports idempotent. Source aggregate metrics remain audit evidence, while summaries derive canonical metrics from stored records.
+The schema normalizes experiments, source exports, agents, map cells, swarm ticks, provider attempts, world events, configuration changes, simulated-player activity, and research notes. Stable source IDs are retained; deterministic experiment/attempt IDs make imports idempotent. Source aggregate metrics remain audit evidence, while summaries derive canonical metrics from stored records.
 
 Retention is never silently upgraded. A filtered import, missing optional observation, or truncated source remains visible as incomplete or missing data. Import validates the complete export and rejects prohibited credential/private-reasoning fields before beginning its transaction. Persistence errors roll back and surface explicitly.
 
@@ -48,24 +47,13 @@ pnpm experiment:db import ./exports/run.json
 pnpm experiment:db list
 pnpm experiment:db summary <experiment-id>
 pnpm experiment:db compare <experiment-id-a> <experiment-id-b>
-pnpm experiment:db turns <experiment-id> --agent <agent-id> --from-turn 40 --to-turn 80
-pnpm experiment:db communications <experiment-id> --channel direct --recipient <agent-id>
-pnpm experiment:db alliance-events <experiment-id> --reason expired
-pnpm experiment:db patient-zero <experiment-id> --from-turn 1 --to-turn 100
+pnpm experiment:db provider-attempts <experiment-id>
 pnpm experiment:db failures <experiment-id> --reason invalid-json
 ```
 
 Commands support `--format table|json|markdown`; table is the concise default. Detail queries default to 50 rows and clamp limits to 500. Filters are exact, ordering is deterministic, and arbitrary SQL is not exposed.
 
-Summary covers scenario/roster attribution, outcomes and retries, action distribution, territory, communication channels, alliance lifecycle/rejections, Patient Zero behavior, usage, size trends, retention, and inconsistencies. Comparison reports absolute totals plus per-turn, per-agent-turn, per-active-agent, and per-Patient-Zero-turn rates.
-
-`directionChangesAfterCommunication` has one canonical definition: for each agent independently, count an accepted move whose direction differs from that agent's previous accepted move when the retained observation contains an inbound direct or alliance message after the previous move and no later than the current turn. Aggregate is exactly the sum of per-agent counts. Original imported values remain unchanged in source-metric audit JSON, and disagreements are reported.
-
-Patient Zero output distinguishes:
-
-- `message-to-patient-zero`: an accepted direct message without a qualifying earlier directive.
-- `reply-after-directive`: an accepted direct message after a Zero directive addressed to that sender and before a later directive to that sender.
-- `observable-compliance`: the next archived action when the directive contains exactly one unambiguous action word (`move`, `infect`, `capture`, or `wait`). Other directives are `indeterminate`.
+Summary covers scenario/roster attribution, outcomes, action distribution, territory, provider attempt/attempt-accounting usage, size trends, retention, and inconsistencies. Comparison reports absolute totals plus per-tick and per-active-agent rates.
 
 ## Curated notes
 
@@ -106,24 +94,26 @@ The archive stores only schema-validated safe export fields and curated notes. I
 
 MCP and embeddings are deferred because bounded local retrieval solves the immediate need without a network/tool authorization surface or derived semantic store. `ExperimentQueryService` and `ResearchNoteService` are the future extension point for a read-only MCP adapter; write/import authority remains outside that adapter.
 
-# Schema-v11 provider attempts
+# Provider attempts
 
-Archive schema v4 stores schema-v11 `providerAttempts` independently of turns.
-Use `pnpm experiment:db provider-attempts <experiment-id>` to inspect committed
-and uncommitted provider work. Monetary values round-trip as canonical TEXT.
-For v11 summaries this ledger is canonical; legacy model attempts are not added
-again. The SQLite archive is for analysis and is not active runtime recovery.
+Archive schema v4 stores `providerAttempts` independently. Use
+`pnpm experiment:db provider-attempts <experiment-id>` to inspect committed and
+uncommitted provider work. Monetary values round-trip as canonical TEXT. This
+ledger is canonical for every current (schema-v12) export, which always
+carries independent attempt accounting. The SQLite archive is for analysis
+and is not active runtime recovery.
 
 Archive schema v5 adds `swarm_ticks` for safe committed zero-swarm plans,
 directives, physical action results, and worker choice telemetry. Full
-all-agent exports carry these records separately from legacy `turns`.
-Provider attempts remain in the independent v4 ledger, including attempts
-from cancelled or rolled-back swarm ticks.
+all-agent exports carry these records. Provider attempts remain in the
+independent v4 ledger, including attempts from cancelled or rolled-back
+swarm ticks.
 
 ## Zero-swarm comparisons
 
-The archive can preserve safe schema-v11 swarm tick records and independent
-provider attempts, but its `compare` command remains centered on normalized
-legacy turns. Use `pnpm compare:offline` for PR F's reproducible three-variant
-same-scenario fixture report. The runner does not import, write, or modify this
-archive. See [Zero-swarm offline comparison](ZERO_SWARM_COMPARISON.md).
+The archive preserves safe schema-v12 swarm tick records and independent
+provider attempts, but its `compare` command is not the same-scenario,
+per-tick swarm harness. Use `pnpm compare:offline` for the reproducible
+zero-swarm-vs-deterministic-workers fixture report. The runner does not
+import, write, or modify this archive. See
+[Zero-swarm offline comparison](ZERO_SWARM_COMPARISON.md).