Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 8 additions & 18 deletions ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,14 @@ architecture and removed all legacy infrastructure:
brought README, this roadmap, architecture, gameplay foundation, security,
testing, and the experiment-archive guide in line with the delivered
architecture. See ADR 0033.
- **Live metrics fix** (`fix(game-api): compute live swarm experiment metrics`):
live World Lab metrics had read zero because the snapshot passed no resolved
actions to the metrics calculation; they now come from the retained swarm
ticks through the same derivation as an all-agents, entire-retained export.
The movement-pattern metrics (`movementDirectionDistribution`,
`longestRepeatedDirectionStreak`, `recentCellRevisits`), which nothing had
ever assigned since the migration, are computed per agent from accepted
moves.

The retirement case is structural rather than measured: the legacy path made one
full generative provider call per active agent per tick, so provider attempts,
Expand All @@ -44,22 +52,6 @@ call) is retained as the ablation control for the swarm comparisons, not as a
second production architecture. Historical milestones below remain as
implementation history.

## Known open work

The following issues are known and owned by a follow-on pull request:

- `simulation-service.ts` passes an empty resolved-action array to
`calculateExperimentMetrics`, so all live World Lab experiment metrics read
zero even though `swarmTicks` now carries the data needed to populate them.
- `movementDirectionDistribution`, `longestRepeatedDirectionStreak`, and
`recentCellRevisits` are declared in the shared metrics schema but nothing
ever assigns them, so they always fall back to their schema defaults. The
direction helper itself already exists
(`geographicDirectionBetweenCells` in `apps/game-api/src/geographic-direction.ts`,
used by swarm pressure and reflex execution); what is missing is the
originating cell on the resolved-action record — which carries only the move
target — and the metric computation itself.

## Agent Zero planner

Agent Zero is the sole generative planner. It makes one OpenRouter call per
Expand Down Expand Up @@ -182,6 +174,4 @@ operator safety boundary, and their schema-v12 safe ledger can be exported to
the analysis archive even when no turn committed. This is not active runtime
persistence, restart recovery, or provider-account balance enforcement.

This milestone also owns the two known open metrics defects noted above.

Player development begins only after these agent milestones demonstrate compelling behavior.
125 changes: 125 additions & 0 deletions apps/game-api/src/experiment-export.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
import { gridDisk, latLngToCell } from 'h3-js';
import { describe, expect, it } from 'vitest';
import {
agentIdSchema,
eventIdSchema,
h3CellSchema,
type AgentId,
type H3Cell,
type WorldAction,
type WorldActionResult,
} from '@hexzero/shared';
import { calculateExperimentMetrics } from './experiment-export';
import { geographicDirectionBetweenCells } from './geographic-direction';

const agentX = agentIdSchema.parse('00000000-0000-4000-8000-000000000001');
const agentY = agentIdSchema.parse('00000000-0000-4000-8000-000000000002');
const origin = h3CellSchema.parse(latLngToCell(41.6528, -83.5379, 9));

function neighbors(cell: H3Cell): H3Cell[] {
return gridDisk(cell, 1)
.filter((candidate) => candidate !== cell)
.map((candidate) => h3CellSchema.parse(candidate));
}

/**
* Finds origin -> a -> b where both outbound steps share one direction and
* both return steps share another.
*/
function straightLine() {
for (const a of neighbors(origin)) {
const direction = geographicDirectionBetweenCells(origin, a);
const opposite = geographicDirectionBetweenCells(a, origin);
const b = neighbors(a).find(
(candidate) =>
candidate !== origin &&
geographicDirectionBetweenCells(a, candidate) === direction &&
geographicDirectionBetweenCells(candidate, a) === opposite,
);
if (b) return { a, b, direction, opposite };
}
throw new Error('No straight two-step H3 line from the origin.');
}

let eventCount = 0;
function moved(tickNumber: number, agentId: AgentId, from: H3Cell, to: H3Cell) {
eventCount += 1;
const action: WorldAction = { type: 'move', targetCell: to };
const actionResult: WorldActionResult = {
accepted: true,
event: {
id: eventIdSchema.parse(
`00000000-0000-4000-8000-${String(eventCount).padStart(12, '0')}`,
),
agentId,
occurredAt: '2026-08-13T12:00:00.000Z',
type: 'agent-moved',
fromCell: from,
toCell: to,
},
};
return { tickNumber, agentId, action, actionResult };
}

function rejectedMove(tickNumber: number, agentId: AgentId, to: H3Cell) {
const action: WorldAction = { type: 'move', targetCell: to };
const actionResult: WorldActionResult = {
accepted: false,
reason: 'not-adjacent',
details: 'Target is not adjacent.',
};
return { tickNumber, agentId, action, actionResult };
}

describe('movement-pattern metrics', () => {
it('walks each agent path separately for direction streaks and revisits', () => {
const { a, b, direction, opposite } = straightLine();
const yTarget = neighbors(origin).find(
(cell) => geographicDirectionBetweenCells(origin, cell) === direction,
)!;

const metrics = calculateExperimentMetrics(
[
moved(1, agentX, origin, a),
// Agent Y moves in the same direction between X's two moves. A single
// interleaved walk would report a streak of three.
moved(1, agentY, origin, yTarget),
rejectedMove(2, agentX, origin),
moved(3, agentX, a, b),
moved(4, agentX, b, a),
moved(5, agentX, a, origin),
],
[agentX, agentY],
);

const x = metrics.byAgent.find(({ agentId }) => agentId === agentX)!;
expect(x.metrics.longestRepeatedDirectionStreak).toBe(2);
expect(x.metrics.recentCellRevisits).toBe(2);
expect(x.metrics.movementDirectionDistribution).toEqual(
expect.arrayContaining([
{ direction, count: 2 },
{ direction: opposite, count: 2 },
]),
);

expect(metrics.aggregate.longestRepeatedDirectionStreak).toBe(2);
expect(metrics.aggregate.recentCellRevisits).toBe(2);
expect(metrics.aggregate.movementDirectionDistribution).toEqual(
expect.arrayContaining([
{ direction, count: 3 },
{ direction: opposite, count: 2 },
]),
);
});

it('reports no movement pattern when no move was accepted', () => {
const metrics = calculateExperimentMetrics(
[rejectedMove(1, agentX, origin)],
[agentX],
);

expect(metrics.aggregate.movementDirectionDistribution).toEqual([]);
expect(metrics.aggregate.longestRepeatedDirectionStreak).toBe(0);
expect(metrics.aggregate.recentCellRevisits).toBe(0);
});
});
105 changes: 104 additions & 1 deletion apps/game-api/src/experiment-export.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ import {
type WorldActionResult,
type WorldSnapshot,
} from '@hexzero/shared';
import {
geographicDirectionBetweenCells,
type GeographicDirection,
} from './geographic-direction';

export interface ExperimentSource {
schemaVersion: 12;
Expand Down Expand Up @@ -411,7 +415,14 @@ function filterControlChanges(
!request.actions.includes('capture')
)
return [];
return requestFiltered.flatMap(({ tickNumber, actionResult }) => {
return capturesAffecting(requestFiltered, selected);
}

function capturesAffecting(
actions: readonly ResolvedWorldAction[],
selected: Set<AgentId>,
): ExportedControlChange[] {
return actions.flatMap(({ tickNumber, actionResult }) => {
if (!actionResult.accepted || actionResult.event.type !== 'hex-captured')
return [];
const event = actionResult.event;
Expand Down Expand Up @@ -575,6 +586,64 @@ function attemptMetrics(attempts: readonly ProviderAttemptRecord[]) {
};
}

const movementDirections: readonly GeographicDirection[] = [
'N',
'NE',
'SE',
'S',
'SW',
'NW',
];

/**
* Direction streaks and revisits only mean something along one agent's own
* path, so each agent's accepted moves are walked separately. A scope spanning
* several agents sums direction counts and revisits and reports the longest
* single-agent streak. A revisit is an accepted move into any cell the agent
* has already occupied within the scope, including its first move's origin.
*/
function movementMetrics(scopeActions: readonly ResolvedWorldAction[]) {
const counts = new Map<GeographicDirection, number>();
const paths = new Map<
AgentId,
{
previous: GeographicDirection | null;
streak: number;
visited: Set<string>;
}
>();
let longestRepeatedDirectionStreak = 0;
let recentCellRevisits = 0;
for (const { agentId, actionResult } of scopeActions) {
if (!actionResult.accepted || actionResult.event.type !== 'agent-moved')
continue;
const { fromCell, toCell } = actionResult.event;
const direction = geographicDirectionBetweenCells(fromCell, toCell);
let path = paths.get(agentId);
if (!path) {
path = { previous: null, streak: 0, visited: new Set([fromCell]) };
paths.set(agentId, path);
}
counts.set(direction, (counts.get(direction) ?? 0) + 1);
path.streak = direction === path.previous ? path.streak + 1 : 1;
path.previous = direction;
longestRepeatedDirectionStreak = Math.max(
longestRepeatedDirectionStreak,
path.streak,
);
if (path.visited.has(toCell)) recentCellRevisits += 1;
path.visited.add(toCell);
}
return {
movementDirectionDistribution: movementDirections.flatMap((direction) => {
const count = counts.get(direction);
return count ? [{ direction, count }] : [];
}),
longestRepeatedDirectionStreak,
recentCellRevisits,
};
}

function metricCountsFor(
scopeActions: readonly ResolvedWorldAction[],
scopeAttempts: readonly ProviderAttemptRecord[],
Expand Down Expand Up @@ -654,6 +723,7 @@ function metricCountsFor(
territoryGainedThroughCapture,
territoryLostThroughCapture,
uniqueVisitedCells: visited.size,
...movementMetrics(scopeActions),
...attemptMetrics(scopeAttempts),
};
}
Expand Down Expand Up @@ -684,6 +754,39 @@ export function calculateExperimentMetrics(
});
}

/**
* Metrics over every known agent and the entire retained experiment: the same
* values an all-agents, entire-retained export reports, so live World Lab
* metrics cannot drift from exported ones.
*/
export function calculateRetainedExperimentMetrics(
source: Pick<
ExperimentSource,
| 'swarmTicks'
| 'scenario'
| 'initialAgents'
| 'currentAgents'
| 'providerAttempts'
>,
): ExperimentMetrics {
const agentIds = [
...new Set(
[...source.initialAgents, ...source.currentAgents].map(({ id }) => id),
),
];
const selected = new Set(agentIds);
const resolved = resolvedActionsFromTicks(
source.swarmTicks,
source.scenario.patientZeroAgentId,
);
return calculateExperimentMetrics(
resolved.filter(({ agentId }) => selected.has(agentId)),
agentIds,
source.providerAttempts.filter(({ agentId }) => selected.has(agentId)),
capturesAffecting(resolved, selected),
);
}

export function serializeExperimentExport(
document: ExperimentExportDocument,
): string {
Expand Down
44 changes: 44 additions & 0 deletions apps/game-api/src/simulation-service.swarm.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,50 @@ describe('zero-swarm SimulationService tick', () => {
expect(tick.swarmTick?.tickNumber).toBe(1);
});

it('reports live experiment metrics equal to an all-agents entire-retained export', async () => {
const simulation = setup(
new InspectingPlanner(),
new ScriptedReflexProvider(
Array.from({ length: 21 }, () => ({ chosenCandidateId: 'action_0' })),
),
);

await simulation.executeNextTick();
await simulation.executeNextTick();
await simulation.executeNextTick();

const snapshot = simulation.getSnapshot();
const resolvedActionCount = (snapshot.swarmTicks ?? []).reduce(
(count, tick) =>
count +
(tick.zeroAction && tick.zeroActionResult ? 1 : 0) +
tick.workers.filter(
({ action, actionResult }) => action && actionResult,
).length,
0,
);
expect(resolvedActionCount).toBeGreaterThan(0);
expect(snapshot.experiment.metrics.aggregate.totalTurns).toBe(
resolvedActionCount,
);

const exported = simulation.generateExperimentExport({
agents: { mode: 'all' },
turns: { mode: 'entire-retained' },
outcomes: [
'accepted',
'rejected',
'lost-tick',
'provider-error',
'operator-skipped',
],
actions: ['move', 'infect', 'capture', 'wait'],
level: 'full-safe',
serialization: 'compact',
});
expect(snapshot.experiment.metrics).toEqual(exported.metrics);
});

it('freezes player-advanced facts for Zero, uses only reflex choices, and resolves physical actions in engine order', async () => {
const planner = new InspectingPlanner();
const simulation = setup(
Expand Down
13 changes: 8 additions & 5 deletions apps/game-api/src/simulation-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ import {
type WorldState,
} from '@hexzero/world-engine';
import {
calculateExperimentMetrics,
calculateRetainedExperimentMetrics,
createExperimentExport,
createExperimentPreview,
type ExperimentSource,
Expand Down Expand Up @@ -417,10 +417,13 @@ export class SimulationService {
id: this.#experimentId,
startedAt: this.#experimentStartedAt,
attemptAccounting: this.#attemptAccounting.snapshot(),
metrics: calculateExperimentMetrics(
[],
agents.map(({ id }) => id),
),
metrics: calculateRetainedExperimentMetrics({
swarmTicks: this.#experimentSwarmTicks,
scenario: this.#scenario,
initialAgents: this.#initialExperimentAgents,
currentAgents: agents,
providerAttempts: this.#attemptAccounting.ledger(),
}),
currentTerritory: this.#territoryScoreboard(),
simulatedPlayerMetrics: this.#state.simulatedPlayer?.metrics ?? {
movements: 0,
Expand Down
Loading
Loading