diff --git a/ROADMAP.md b/ROADMAP.md index d84040b..9cb6ba9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -13,6 +13,11 @@ policies. The first comparison retains legacy mode pending real-provider trials; see `docs/ZERO_SWARM_COMPARISON.md`. See ADRs 0028-0032. Historical milestones below remain as implementation history. +PR #58 closes the pre-live Jev capture-context and Agent Zero cost-accounting +gaps. PR #59 adds an explicitly opted-in live comparison of Jev workers against +deterministic workers while holding the Zero model and scenario inputs constant. +Legacy retirement remains a later decision after measured live results. + ## Experimental Patient Zero coordinator The current focused slice requires one normal physical agent, provides a diff --git a/apps/game-api/package.json b/apps/game-api/package.json index a163349..327b7f2 100644 --- a/apps/game-api/package.json +++ b/apps/game-api/package.json @@ -6,6 +6,7 @@ "scripts": { "build": "esbuild src/server.ts --bundle --platform=node --format=esm --outfile=dist/server.js", "compare:offline": "node --import tsx src/swarm-comparison-cli.ts", + "compare:live": "node --import tsx src/live-swarm-comparison-cli.ts", "dev": "tsx watch src/server.ts", "lint": "eslint .", "start": "node dist/server.js", diff --git a/apps/game-api/src/live-swarm-comparison-cli.test.ts b/apps/game-api/src/live-swarm-comparison-cli.test.ts new file mode 100644 index 0000000..fe368c7 --- /dev/null +++ b/apps/game-api/src/live-swarm-comparison-cli.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + markdownSummary, + parseLiveComparisonArguments, + runLiveComparisonCli, +} from './live-swarm-comparison-cli'; + +describe('live swarm comparison CLI', () => { + it('refuses before constructing providers or calling the runner without cost acknowledgement', async () => { + const runComparison = vi.fn(); + const providers = vi.fn(); + await expect( + runLiveComparisonCli(['--model', 'openrouter/test'], { + runComparison, + providers, + readFile: vi.fn(), + env: { OPENROUTER_API_KEY: 'openrouter', TYPESAFE_API_KEY: 'typesafe' }, + } as never), + ).rejects.toThrow('--confirm-provider-costs'); + expect(runComparison).not.toHaveBeenCalled(); + expect(providers).not.toHaveBeenCalled(); + }); + + it('requires an explicit model after acknowledgement without invoking providers', async () => { + const runComparison = vi.fn(); + const providers = vi.fn(); + await expect( + runLiveComparisonCli(['--confirm-provider-costs'], { + runComparison, + providers, + readFile: vi.fn(), + env: { OPENROUTER_API_KEY: 'openrouter', TYPESAFE_API_KEY: 'typesafe' }, + } as never), + ).rejects.toThrow('--model is required'); + expect(runComparison).not.toHaveBeenCalled(); + expect(providers).not.toHaveBeenCalled(); + }); + + it('reads a saved report for a summary without acknowledgement, credentials, or providers', async () => { + const runComparison = vi.fn(); + const providers = vi.fn(); + const report = { + configuration: { modelId: 'openrouter/test', seeds: ['a'], tickCap: 20 }, + variants: [ + { + variant: 'live-zero-jev', + aggregate: { + totalCaptures: 1, + totalWorkerStalls: 2, + openRouterTokens: { prompt: 10, completion: 4, total: 14 }, + }, + }, + ], + }; + await expect( + runLiveComparisonCli(['--summary-from', '/tmp/report.json'], { + runComparison, + providers, + readFile: vi.fn().mockResolvedValue(JSON.stringify(report)), + env: {}, + } as never), + ).resolves.toContain('live-zero-jev'); + expect(runComparison).not.toHaveBeenCalled(); + expect(providers).not.toHaveBeenCalled(); + }); + + it('rejects unsafe aggregate text in a hand-edited saved report', async () => { + await expect( + runLiveComparisonCli(['--summary-from', '/tmp/report.json'], { + runComparison: vi.fn(), + providers: vi.fn(), + readFile: vi.fn().mockResolvedValue( + JSON.stringify({ + configuration: { + modelId: 'openrouter/test', + seeds: ['a'], + tickCap: 20, + }, + variants: [ + { + variant: 'live-zero-jev', + aggregate: { captures: '| injected Markdown |' }, + }, + ], + }), + ), + env: {}, + } as never), + ).rejects.toThrow('invalid shape'); + }); + + it('uses the bounded defaults and rejects oversized live inputs', () => { + expect( + parseLiveComparisonArguments([ + '--confirm-provider-costs', + '--model', + 'openrouter/test', + ]), + ).toMatchObject({ + seeds: ['worker-capture-spawn', 'comparison-seed-b', 'comparison-seed-c'], + ticks: 20, + agents: 8, + attempts: 200, + creditLimit: '3', + reservation: '0.02', + }); + expect(() => + parseLiveComparisonArguments([ + '--model', + 'openrouter/test', + '--ticks', + '31', + ]), + ).toThrow('between 1 and 30'); + expect(() => + parseLiveComparisonArguments([ + '--model', + 'openrouter/test', + '--agents', + '1', + ]), + ).toThrow('between 2 and 8'); + }); + + it('keeps survival, territory, stalls, replans, and cost separate in summaries', () => { + expect( + markdownSummary({ + configuration: { + modelId: 'openrouter/test', + seeds: ['a'], + tickCap: 20, + }, + variants: [ + { variant: 'live-zero-jev', aggregate: { totalCaptures: 1 } }, + ], + } as never), + ).toContain('This summary does not select a winner.'); + }); +}); diff --git a/apps/game-api/src/live-swarm-comparison-cli.ts b/apps/game-api/src/live-swarm-comparison-cli.ts new file mode 100644 index 0000000..1d1519c --- /dev/null +++ b/apps/game-api/src/live-swarm-comparison-cli.ts @@ -0,0 +1,324 @@ +#!/usr/bin/env node +import { + OpenRouterSwarmPlanner, + TypeSafeJevReflexProvider, +} from '@hexzero/agent-runtime'; +import { + runLiveComparison, + type LiveComparisonProviders, + type LiveComparisonReport, +} from './live-swarm-comparison.js'; + +const DEFAULT_SEEDS = [ + 'worker-capture-spawn', + 'comparison-seed-b', + 'comparison-seed-c', +]; +const DEFAULT_TICKS = 20; +const DEFAULT_AGENTS = 8; +const DEFAULT_ATTEMPTS = 200; +const DEFAULT_CREDIT_LIMIT = '3'; +const DEFAULT_RESERVATION = '0.02'; +const MAX_SEEDS = 5; +const MAX_TICKS = 30; +const MAX_AGENTS = 8; +const MAX_ATTEMPTS = 300; +const MAX_CREDIT_LIMIT = 10; +const MAX_RESERVATION = 0.5; +const MAX_SEED_LENGTH = 80; + +export interface LiveComparisonCliOptions { + confirmProviderCosts: boolean; + model?: string; + seeds: readonly string[]; + ticks: number; + agents: number; + attempts: number; + creditLimit: string; + reservation: string; + summaryFrom?: string; +} + +export interface LiveComparisonCliDependencies { + runComparison: typeof runLiveComparison; + readFile: (path: string) => Promise; + providers: (keys: { + openRouter: string; + typeSafe: string; + }) => LiveComparisonProviders; + env: NodeJS.ProcessEnv; +} + +export function usage(): string { + return `Usage: + pnpm compare:live --confirm-provider-costs --model [options] + pnpm compare:live --summary-from + +Live mode makes paid provider requests only after the explicit acknowledgement. +It requires OPENROUTER_API_KEY and TYPESAFE_API_KEY after acknowledgement. +Options: --seeds (max ${MAX_SEEDS}), --ticks <1-${MAX_TICKS}>, --agents <2-${MAX_AGENTS}>, +--attempts <1-${MAX_ATTEMPTS}>, --credit-limit <0-${MAX_CREDIT_LIMIT}>, +--reservation <0-${MAX_RESERVATION}>. Defaults: 3 seeds, ${DEFAULT_TICKS} ticks, ${DEFAULT_AGENTS} agents.`; +} + +function integer( + value: string, + option: string, + maximum: number, + minimum = 1, +): number { + if (!/^[1-9][0-9]*$/.test(value)) + throw new Error(`${option} must be a positive integer.`); + const number = Number(value); + if (number < minimum || number > maximum) + throw new Error(`${option} must be between ${minimum} and ${maximum}.`); + return number; +} + +function credit(value: string, option: string, maximum: number): string { + if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]+)?$/.test(value) || !/[1-9]/.test(value)) + throw new Error(`${option} must be a positive decimal.`); + if (!Number.isFinite(Number(value)) || Number(value) > maximum) + throw new Error(`${option} must be at most ${maximum}.`); + return value; +} + +function seeds(value: string): string[] { + const parsed = value.split(',').map((entry) => entry.trim()); + if ( + !parsed.length || + parsed.length > MAX_SEEDS || + parsed.some( + (entry) => + !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(entry) || + entry.length > MAX_SEED_LENGTH, + ) + ) + throw new Error( + `--seeds must contain 1 to ${MAX_SEEDS} stable identifiers.`, + ); + const unique = [...new Set(parsed)]; + if (unique.length !== parsed.length) + throw new Error('--seeds must not contain duplicates.'); + return unique; +} + +function modelId(value: string): string { + if (!/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._:-]+$/.test(value) || value.length > 200) + throw new Error('--model must be a provider/model identifier.'); + return value; +} + +export function parseLiveComparisonArguments( + input: readonly string[], +): LiveComparisonCliOptions { + const options: LiveComparisonCliOptions = { + confirmProviderCosts: false, + seeds: DEFAULT_SEEDS, + ticks: DEFAULT_TICKS, + agents: DEFAULT_AGENTS, + attempts: DEFAULT_ATTEMPTS, + creditLimit: DEFAULT_CREDIT_LIMIT, + reservation: DEFAULT_RESERVATION, + }; + const seen = new Set(); + for (let index = 0; index < input.length; index += 1) { + const option = input[index]!; + if (option === '--confirm-provider-costs') { + options.confirmProviderCosts = true; + continue; + } + if (option === '--help' || option === '-h') throw new Error('help'); + if (seen.has(option)) throw new Error(`${option} was supplied twice.`); + seen.add(option); + const value = input[index + 1]; + if (!value || value.startsWith('--')) + throw new Error(`Option ${option} requires a value.`); + if (option === '--model') options.model = modelId(value); + else if (option === '--seeds') options.seeds = seeds(value); + else if (option === '--ticks') + options.ticks = integer(value, option, MAX_TICKS); + else if (option === '--agents') + options.agents = integer(value, option, MAX_AGENTS, 2); + else if (option === '--attempts') + options.attempts = integer(value, option, MAX_ATTEMPTS); + else if (option === '--credit-limit') + options.creditLimit = credit(value, option, MAX_CREDIT_LIMIT); + else if (option === '--reservation') + options.reservation = credit(value, option, MAX_RESERVATION); + else if (option === '--summary-from') options.summaryFrom = value; + else throw new Error(`Unknown option: ${option}`); + index += 1; + } + if ( + options.summaryFrom && + (options.confirmProviderCosts || options.model || seen.size > 1) + ) + throw new Error('--summary-from cannot be combined with live-run options.'); + return options; +} + +export function markdownSummary(report: LiveComparisonReport): string { + const configuration = report.configuration; + const variants = report.variants.map((variant) => { + const aggregate = variant.aggregate as Record; + const get = (...names: string[]) => + names + .map((name) => aggregate[name]) + .find((value) => value !== undefined && value !== null) ?? 'unknown'; + return `| ${variant.variant} | ${get('patientZeroCapturedRuns')} | ${get('finalActiveWorkers')} | ${get('totalCaptures', 'captures')} | ${get('totalControlledCells', 'controlledCells', 'finalControlledCells')} | ${get('totalWorkerStalls', 'workerStalls')} | ${get('totalReplanSignals', 'workerReplanSignals')} | ${get('totalGenerativeAttempts', 'zeroPlanningAttempts')} | ${get('totalReflexAttempts', 'jevAttempts')} | ${get('jevFallbacks')} | ${get('knownOpenRouterCostCredits', 'actualOpenRouterCostCredits')} | ${get('attemptsWithUnknownCost', 'unknownProviderCostAttempts')} |`; + }); + return `# Live swarm comparison\n\nInputs are reproducible; provider outputs are nondeterministic. Model: \`${configuration.modelId}\`. Seeds: ${configuration.seeds.join(', ')}. Tick cap: ${configuration.tickCap}.\n\n| Variant | Patient Zero captures | Final active workers | Captures | Territory | Worker stalls | Replan signals | Zero planning attempts | Jev attempts | Jev fallbacks | Reported OpenRouter cost (credits) | Unknown cost fields |\n| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n${variants.join('\n')}\n\nReview survival and captures, territory, stalls, replans, and provider effort/cost separately. This summary does not select a winner.`; +} + +function summaryReport(value: unknown): LiveComparisonReport { + if (!value || typeof value !== 'object') + throw new Error('Saved report has an invalid shape.'); + const report = value as { configuration?: unknown; variants?: unknown }; + if ( + !report.configuration || + typeof report.configuration !== 'object' || + !Array.isArray(report.variants) + ) + throw new Error('Saved report has an invalid shape.'); + const configuration = report.configuration as { + modelId?: unknown; + seeds?: unknown; + tickCap?: unknown; + }; + if ( + typeof configuration.modelId !== 'string' || + configuration.modelId.length > 200 || + !/^[a-zA-Z0-9._/:@-]+$/.test(configuration.modelId) || + !Array.isArray(configuration.seeds) || + !configuration.seeds.every( + (seed) => + typeof seed === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(seed), + ) || + !Number.isInteger(configuration.tickCap) + ) + throw new Error('Saved report has an invalid shape.'); + if ( + !report.variants.every( + (variant) => + variant && + typeof variant === 'object' && + ['live-zero-jev', 'live-zero-deterministic-workers'].includes( + (variant as { variant?: unknown }).variant as string, + ) && + typeof (variant as { aggregate?: unknown }).aggregate === 'object' && + (variant as { aggregate?: unknown }).aggregate !== null && + Object.values( + (variant as { aggregate: Record }).aggregate, + ).every((entry) => safeSummaryValue(entry)), + ) + ) + throw new Error('Saved report has an invalid shape.'); + return value as LiveComparisonReport; +} + +function safeSummaryValue(value: unknown, depth = 0): boolean { + if (value === null) return true; + if (typeof value === 'number') return Number.isFinite(value); + if (typeof value === 'string') return /^\d+(?:\.\d+)?$/.test(value); + return ( + depth === 0 && + !!value && + typeof value === 'object' && + !Array.isArray(value) && + Object.values(value).every((entry) => safeSummaryValue(entry, 1)) + ); +} + +export function defaultProviders(keys: { + openRouter: string; + typeSafe: string; +}): LiveComparisonProviders { + return { + createPlanner: () => + new OpenRouterSwarmPlanner({ apiKey: keys.openRouter }), + createReflex: () => + new TypeSafeJevReflexProvider({ apiKey: keys.typeSafe }), + }; +} + +const defaults: LiveComparisonCliDependencies = { + runComparison: runLiveComparison, + readFile: (path) => + import('node:fs/promises').then(({ readFile }) => readFile(path, 'utf8')), + providers: defaultProviders, + env: process.env, +}; + +/** Runs only after parsing and acknowledgement checks; no implicit dotenv loading occurs. */ +export async function runLiveComparisonCli( + input: readonly string[], + dependencies = defaults, +): Promise { + const options = parseLiveComparisonArguments(input); + if (options.summaryFrom) + return markdownSummary( + summaryReport( + JSON.parse(await dependencies.readFile(options.summaryFrom)), + ), + ); + if (!options.confirmProviderCosts) + throw new Error( + 'Refusing provider calls: pass --confirm-provider-costs to acknowledge provider costs.', + ); + if (!options.model?.trim()) + throw new Error( + '--model is required; select the OpenRouter model explicitly.', + ); + const openRouter = dependencies.env.OPENROUTER_API_KEY?.trim(); + const typeSafe = dependencies.env.TYPESAFE_API_KEY?.trim(); + if (!openRouter || !typeSafe) + throw new Error( + 'OPENROUTER_API_KEY and TYPESAFE_API_KEY are required after acknowledgement.', + ); + const report = await dependencies.runComparison( + { + confirmed: true, + modelId: options.model, + seeds: options.seeds, + tickCap: options.ticks, + agentCount: options.agents, + providerAttemptLimit: options.attempts, + creditLimit: options.creditLimit, + reservationCreditsPerAttempt: options.reservation, + }, + dependencies.providers({ openRouter, typeSafe }), + ); + return JSON.stringify(report, null, 2); +} + +async function main(): Promise { + try { + process.stdout.write( + `${await runLiveComparisonCli(process.argv.slice(2))}\n`, + ); + } catch (error) { + if (error instanceof Error && error.message === 'help') { + process.stdout.write(`${usage()}\n`); + return; + } + const message = safeCliError(error); + process.stderr.write(`${message}\n${usage()}\n`); + process.exitCode = 1; + } +} + +/** Provider errors are deliberately not echoed: a gateway may include a raw body. */ +function safeCliError(error: unknown): string { + const message = error instanceof Error ? error.message : ''; + if (message.startsWith('Unknown option:')) return 'Unknown option.'; + if ( + /^(?:Option |--(?:model|seeds|ticks|agents|attempts|credit-limit|reservation|summary-from)|Refusing provider calls:|OPENROUTER_API_KEY and TYPESAFE_API_KEY)/.test( + message, + ) + ) + return message; + return 'Live comparison failed without emitting provider response details.'; +} + +if (import.meta.url === `file://${process.argv[1]}`) void main(); diff --git a/apps/game-api/src/live-swarm-comparison.test.ts b/apps/game-api/src/live-swarm-comparison.test.ts new file mode 100644 index 0000000..b70487a --- /dev/null +++ b/apps/game-api/src/live-swarm-comparison.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + ScriptedReflexProvider, + type SwarmPlanner, +} from '@hexzero/agent-runtime'; +import { + runLiveComparison, + type LiveComparisonConfig, + type LiveComparisonProviders, +} from './live-swarm-comparison'; + +const config = (): LiveComparisonConfig => ({ + confirmed: true, + modelId: 'operator/model', + seeds: ['worker-capture-spawn'], + tickCap: 2, + agentCount: 8, + providerAttemptLimit: 12, + creditLimit: '1', + reservationCreditsPerAttempt: '0.01', +}); + +describe('runLiveComparison admission guard', () => { + it('rejects missing explicit confirmation before constructing a provider', async () => { + const createPlanner = vi.fn(); + const providers = { + createPlanner, + createReflex: vi.fn(), + createLegacy: vi.fn(), + } as unknown as LiveComparisonProviders; + + await expect( + runLiveComparison({ ...config(), confirmed: false } as never, providers), + ).rejects.toThrow('explicit provider-cost confirmation'); + expect(createPlanner).not.toHaveBeenCalled(); + }); + + it('enforces the fixed live experiment caps before constructing a provider', async () => { + const createPlanner = vi.fn(); + const providers = { + createPlanner, + createReflex: vi.fn(), + createLegacy: vi.fn(), + } as unknown as LiveComparisonProviders; + + await expect( + runLiveComparison({ ...config(), tickCap: 31 }, providers), + ).rejects.toThrow('tickCap must be an integer between 1 and 30'); + expect(createPlanner).not.toHaveBeenCalled(); + }); + + it('runs matched seeded inputs while omitting Jev calls in the deterministic control', async () => { + const observations: string[] = []; + const decide = vi.fn(); + const providers: LiveComparisonProviders = { + createPlanner: () => + ({ + mode: 'scripted-swarm-test', + configured: true, + async plan(observation, model, options) { + observations.push(JSON.stringify(observation)); + const plan = { + strategySummary: 'Hold the seeded perimeter.', + zeroActionCandidateId: observation.legalZeroActions.find( + ({ action }) => action.type === 'wait', + )!.id, + directives: observation.agents + .filter(({ agentId }) => agentId !== observation.zeroAgentId) + .map((agent, index) => ({ + id: `hold-${observation.tickNumber}-${index}`, + agentId: agent.agentId, + mission: 'hold' as const, + targetCell: agent.position, + priority: 'normal' as const, + riskTolerance: 'low' as const, + issuedAtTick: observation.tickNumber, + expiresAtTick: observation.tickNumber + 4, + })), + }; + const metadata = { + provider: 'openrouter' as const, + model, + latencyMs: 2, + promptTokens: 10, + completionTokens: 4, + totalTokens: 14, + costCredits: 0.001, + }; + options?.beginAttempt?.('initial')?.({ + outcome: 'completed', + provider: metadata, + swarmPlan: plan, + }); + return { plan, metadata }; + }, + }) satisfies SwarmPlanner, + createReflex: () => { + const scripted = new ScriptedReflexProvider( + Array.from({ length: 8 }, () => ({ chosenCandidateId: 'action_0' })), + ); + return { + ...scripted, + decide: decide.mockImplementation(scripted.decide.bind(scripted)), + }; + }, + }; + const report = await runLiveComparison( + { ...config(), tickCap: 1 }, + providers, + ); + expect(report.variants.map(({ variant }) => variant)).toEqual([ + 'live-zero-jev', + 'live-zero-deterministic-workers', + ]); + expect(observations).toHaveLength(2); + expect(observations[0]).toBe(observations[1]); + expect(decide.mock.calls.length).toBeGreaterThan(0); + expect(report.variants[0]!.aggregate.jevAttempts).toBe( + decide.mock.calls.length, + ); + expect(report.variants[1]!.aggregate.jevAttempts).toBe(0); + expect(report.variants[0]!.aggregate.zeroPlanningAttempts).toBe(1); + expect(report.variants[1]!.aggregate.zeroPlanningAttempts).toBe(1); + expect(report.variants[0]!.aggregate.openRouterTokens).toEqual({ + prompt: 10, + completion: 4, + total: 14, + }); + expect(report.variants[0]!.aggregate.providerLatencyMs).toBeGreaterThan(0); + expect(report.variants[0]!.runs[0]!.ticks[0]!.openRouterTokens.total).toBe( + 14, + ); + expect( + report.variants[0]!.runs[0]!.ticks[0]!.actualOpenRouterCostCredits, + ).toBe('0.001'); + expect( + report.variants[0]!.runs[0]!.effort.actualOpenRouterCostCredits, + ).toBe('0.001'); + expect(report.variants[1]!.runs[0]!.effort.typeSafeCostCredits).toBeNull(); + }); +}); diff --git a/apps/game-api/src/live-swarm-comparison.ts b/apps/game-api/src/live-swarm-comparison.ts new file mode 100644 index 0000000..8568b13 --- /dev/null +++ b/apps/game-api/src/live-swarm-comparison.ts @@ -0,0 +1,775 @@ +import { + BrowserTestAgentProvider, + type ReflexProvider, + type SwarmPlanner, +} from '@hexzero/agent-runtime'; +import { assignBehavior, type CompatibleModel } from '@hexzero/shared'; +import { generateDeterministicRoster } from '@hexzero/world-engine'; +import { SimulationService } from './simulation-service'; +import type { CompiledReflexObservation } from './reflex-execution'; + +export type LiveComparisonVariant = + 'live-zero-jev' | 'live-zero-deterministic-workers'; + +/** All limits are deliberately small: this is an opt-in observation run. */ +export interface LiveComparisonConfig { + confirmed: true; + modelId: string; + seeds: readonly string[]; + tickCap: number; + agentCount: number; + providerAttemptLimit: number; + creditLimit: string; + reservationCreditsPerAttempt: string; +} + +export interface LiveComparisonProviders { + createPlanner(): SwarmPlanner; + createReflex(): ReflexProvider; +} + +export interface LiveComparisonTick { + tick: number; + activeWorkers: number; + captures: number; + patientZeroCaptured: boolean; + terminalOutcome: string | null; + infectedCells: number; + controlledCells: number; + territoryGained: number; + territoryLost: number; + disinfections: number; + workerStalls: number; + directiveProgress: Record; + workerReplanSignals: number; + zeroReplans: number; + zeroReplanReasons: readonly string[]; + zeroPlanningAttempts: number; + jevAttempts: number; + jevFallbacks: number; + jevConfidence: readonly number[]; + jevActionProbabilities: readonly Readonly>[]; + jevReplanProbabilities: readonly number[]; + providerLatencyMs: number; + openRouterTokens: { prompt: number; completion: number; total: number }; + typeSafeTokens: { input: number; output: number }; + actualOpenRouterCostCredits: string | null; + unknownProviderCostAttempts: number; + unknownTypeSafeCostAttempts: number; +} + +export interface LiveComparisonRun { + seed: string; + ticks: readonly LiveComparisonTick[]; + final: { + tick: number; + activeWorkers: number; + captures: number; + patientZeroCaptured: boolean; + terminalOutcome: string | null; + stoppedReason?: + 'tick-cap' | 'terminal' | 'budget-exhausted' | 'execution-error'; + infectedCells: number; + controlledCells: number; + }; + effort: { + zeroPlanningAttempts: number; + jevAttempts: number; + jevFallbacks: number; + openRouterTokens: { + prompt: number; + completion: number; + total: number; + unknownFields: readonly string[]; + }; + typeSafeTokens: { + input: number; + output: number; + unknownFields: readonly string[]; + }; + providerLatencyMs: number; + actualOpenRouterCostCredits: string | null; + unknownProviderCostAttempts: number; + typeSafeCostCredits: null; + unknownTypeSafeCostAttempts: number; + attemptsWithoutCommittedTick: number; + }; + attemptAccounting: { + started: number; + finalized: number; + exhausted: boolean; + exhaustionReason: string | null; + }; +} + +export interface LiveComparisonVariantReport { + variant: LiveComparisonVariant; + runs: readonly LiveComparisonRun[]; + aggregate: { + runCount: number; + captures: number; + patientZeroCapturedRuns: number; + terminalRuns: number; + finalActiveWorkers: number; + finalControlledCells: number; + disinfections: number; + territoryGained: number; + territoryLost: number; + workerStalls: number; + workerReplanSignals: number; + zeroReplans: number; + zeroPlanningAttempts: number; + jevAttempts: number; + jevFallbacks: number; + openRouterTokens: { prompt: number; completion: number; total: number }; + typeSafeTokens: { input: number; output: number }; + providerLatencyMs: number; + meanJevConfidence: number | null; + meanJevReplanProbability: number | null; + actualOpenRouterCostCredits: string | null; + unknownProviderCostAttempts: number; + unknownTypeSafeCostAttempts: number; + attemptsWithoutCommittedTick: number; + }; +} + +export interface LiveComparisonReport { + formatVersion: 1; + kind: 'live-zero-swarm-jev-ablation'; + configuration: { + modelId: string; + seeds: readonly string[]; + tickCap: number; + agentCount: number; + cognitionMode: 'zero-swarm-v1'; + trailHunterProfile: 'trail-hunter-v1'; + rosterSeed: 'worker-capture-roster'; + patientZeroRosterIndex: 1; + worldSeedPattern: 'offline-world-{seed}'; + planningPolicy: 'zero-swarm-v1-service-default'; + objectiveVersion: 'durable-influence-v3'; + executionLimits: Pick< + LiveComparisonConfig, + 'providerAttemptLimit' | 'creditLimit' | 'reservationCreditsPerAttempt' + >; + reproducibility: 'reproducible-input; provider output is nondeterministic'; + }; + providerCostNote: 'OpenRouter cost is provider-reported when available; TypeSafe monetary cost is unknown.'; + variants: readonly LiveComparisonVariantReport[]; +} + +const LIMITS = { + seeds: 5, + ticks: 30, + agents: 8, + attempts: 300, + credits: 10, + reservation: 0.5, +} as const; + +const selector = (compiled: CompiledReflexObservation): string => + (compiled.observation.candidates.find(({ description }) => + description.startsWith('Infect'), + ) ?? + compiled.observation.candidates.find(({ description }) => + description.includes('open territory'), + ) ?? + compiled.observation.candidates[0])!.id; + +const modelFor = (id: string): CompatibleModel => ({ + id, + name: id, + author: 'operator-selected', + contextLength: 16_384, + inputPricePerToken: '0', + outputPricePerToken: '0', + supportedParameters: [], + isFree: false, + reasoning: { mandatory: false, supportedEfforts: ['low'] }, +}); + +function decimal(value: string, label: string, maximum: number): void { + if ( + !/^(?:0|[1-9]\d*)(?:\.\d+)?$/u.test(value) || + value.length > 80 || + !/[1-9]/u.test(value) || + !Number.isFinite(Number(value)) || + Number(value) > maximum + ) + throw new Error(`${label} must be a decimal no greater than ${maximum}.`); +} + +function validate(config: LiveComparisonConfig): void { + if (config.confirmed !== true) + throw new Error( + 'Live comparison requires explicit provider-cost confirmation.', + ); + if ( + !/^[A-Za-z0-9._-]+\/[A-Za-z0-9._:-]+$/u.test(config.modelId) || + config.modelId.length > 200 + ) + throw new Error('Live comparison requires a safe explicit modelId.'); + if ( + !config.seeds.length || + config.seeds.length > LIMITS.seeds || + config.seeds.some( + (seed) => !/^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$/u.test(seed), + ) || + new Set(config.seeds).size !== config.seeds.length + ) + throw new Error( + `seeds must contain 1-${LIMITS.seeds} unique non-empty values.`, + ); + for (const [name, value, minimum, maximum] of [ + ['tickCap', config.tickCap, 1, LIMITS.ticks], + ['agentCount', config.agentCount, 2, LIMITS.agents], + ['providerAttemptLimit', config.providerAttemptLimit, 1, LIMITS.attempts], + ] as const) + if (!Number.isInteger(value) || value < minimum || value > maximum) + throw new Error( + `${name} must be an integer between ${minimum} and ${maximum}.`, + ); + decimal(config.creditLimit, 'creditLimit', LIMITS.credits); + decimal( + config.reservationCreditsPerAttempt, + 'reservationCreditsPerAttempt', + LIMITS.reservation, + ); +} + +function ids(prefix: string) { + let value = 0; + return () => + `00000000-0000-4000-8000-${prefix}${String(++value).padStart(11, '0')}`; +} + +function territory(snapshot: ReturnType) { + const infected = snapshot.world.hexes.filter( + (cell) => cell.state === 'infected', + ); + return { + infectedCells: infected.length, + controlledCells: infected.filter((cell) => cell.controllerAgentId !== null) + .length, + }; +} + +function tickSample( + snapshot: ReturnType, + zeroId: string, + baseline: boolean, + previousControlledCells: number, +): LiveComparisonTick { + const swarm = snapshot.swarmTicks?.at(-1); + const current = swarm?.tickNumber === snapshot.tickNumber ? swarm : undefined; + const events = snapshot.world.events.filter( + (event) => + 'originatingTick' in event && + event.originatingTick === snapshot.tickNumber, + ); + const workers = current?.workers ?? []; + const reflexes = workers.flatMap((worker) => + worker.reflexDecision ? [worker.reflexDecision] : [], + ); + const directiveProgress: Record = {}; + for (const worker of workers) { + const progress = worker.situation?.directiveProgress ?? 'unknown'; + directiveProgress[progress] = (directiveProgress[progress] ?? 0) + 1; + } + const metadata = [ + current?.plannerMetadata, + ...reflexes.map((decision) => ({ latencyMs: decision.latencyMs })), + ].filter(Boolean) as { latencyMs: number }[]; + return { + tick: snapshot.tickNumber, + activeWorkers: snapshot.world.agents.filter((agent) => agent.id !== zeroId) + .length, + captures: events.filter( + (event) => event.type === 'simulated-player-agent-captured', + ).length, + patientZeroCaptured: snapshot.status === 'patient-zero-captured', + terminalOutcome: ['patient-zero-captured', 'infection-eliminated'].includes( + snapshot.status, + ) + ? snapshot.status + : null, + ...territory(snapshot), + territoryGained: Math.max( + 0, + territory(snapshot).controlledCells - previousControlledCells, + ), + territoryLost: Math.max( + 0, + previousControlledCells - territory(snapshot).controlledCells, + ), + disinfections: events.filter((event) => event.type === 'hex-disinfected') + .length, + workerStalls: workers.filter( + (worker) => + worker.actionResult?.accepted === false || + (worker.action?.type === 'wait' && worker.directive.mission !== 'hold'), + ).length, + directiveProgress, + workerReplanSignals: current?.signals?.length ?? 0, + zeroReplans: + current?.planSource === 'zero-llm' && snapshot.tickNumber > 1 ? 1 : 0, + zeroReplanReasons: current?.replanReasons ?? [], + zeroPlanningAttempts: current?.planSource === 'zero-llm' ? 1 : 0, + jevAttempts: reflexes.length, + jevFallbacks: baseline + ? 0 + : workers.filter((worker) => worker.source !== 'jev-reflex').length, + jevConfidence: reflexes.map((decision) => decision.confidence), + jevActionProbabilities: reflexes.map((decision) => decision.probabilities), + jevReplanProbabilities: reflexes.flatMap((decision) => + decision.replanProbability === undefined + ? [] + : [decision.replanProbability], + ), + providerLatencyMs: metadata.reduce( + (total, entry) => total + entry.latencyMs, + 0, + ), + openRouterTokens: { prompt: 0, completion: 0, total: 0 }, + typeSafeTokens: { input: 0, output: 0 }, + actualOpenRouterCostCredits: null, + unknownProviderCostAttempts: 0, + unknownTypeSafeCostAttempts: 0, + }; +} + +function exportRequest() { + return { + agents: { mode: 'all' }, + turns: { mode: 'entire-retained' }, + outcomes: ['accepted', 'rejected', 'provider-error', 'operator-skipped'], + actions: ['move', 'infect', 'capture', 'wait'], + communications: { channel: 'all', status: 'all' }, + serialization: 'compact', + level: 'full-safe', + } as const; +} + +function addDecimal(left: string, right: string): string { + const [leftWhole, leftFraction = ''] = left.split('.'); + const [rightWhole, rightFraction = ''] = right.split('.'); + const scale = Math.max(leftFraction.length, rightFraction.length); + const integer = (whole: string, fraction: string) => + BigInt(`${whole}${fraction.padEnd(scale, '0')}`); + const raw = ( + integer(leftWhole!, leftFraction) + integer(rightWhole!, rightFraction) + ) + .toString() + .padStart(scale + 1, '0'); + return scale === 0 + ? raw + : `${raw.slice(0, -scale)}.${raw.slice(-scale)}` + .replace(/0+$/u, '') + .replace(/\.$/u, ''); +} + +async function runVariant( + variant: LiveComparisonVariant, + seed: string, + config: LiveComparisonConfig, + providers: LiveComparisonProviders, +): Promise { + const service = new SimulationService({ + // Zero-swarm never asks the legacy AgentProvider. Keeping this local test + // provider prevents an accidental legacy network call in the ablation. + provider: new BrowserTestAgentProvider(), + swarmPlanner: providers.createPlanner(), + reflexProvider: providers.createReflex(), + ...(variant === 'live-zero-deterministic-workers' + ? { deterministicWorkerCandidateSelector: selector } + : {}), + createEventId: ids('1'), + createExperimentId: ids('2'), + createAllianceId: ids('3'), + createProposalId: ids('4'), + }); + service.setCompatibleModels([modelFor(config.modelId)]); + const request = service.getDefaultWorldSetup(); + const roster = generateDeterministicRoster( + config.agentCount, + 'worker-capture-roster', + ); + const zeroAgentId = roster[1]!.id; + service.applyWorldSetup({ + ...request, + cognitionMode: 'zero-swarm-v1', + roster, + patientZeroAgentId: zeroAgentId, + worldSeed: `offline-world-${seed}`, + spawnSeed: seed, + objectiveVersion: 'durable-influence-v3', + capabilities: { ...request.capabilities, simulatedPlayerPressure: true }, + simulatedPlayer: { enabled: true, profile: 'trail-hunter-v1', seed }, + executionLimits: { + version: 'execution-limits-v2', + providerAttemptLimit: config.providerAttemptLimit, + creditLimit: config.creditLimit, + reservationCreditsPerAttempt: config.reservationCreditsPerAttempt, + }, + modelConfiguration: { + globalModelId: config.modelId, + globalReasoningProfile: 'low', + overrides: [], + locked: false, + }, + behaviorConfiguration: { + ...request.behaviorConfiguration, + assignments: assignBehavior( + roster.map(({ id }) => id), + `live-behavior-${seed}`, + 'balanced-random', + ), + }, + }); + const ticks: LiveComparisonTick[] = []; + let stoppedReason: LiveComparisonRun['final']['stoppedReason'] = 'tick-cap'; + for (let index = 0; index < config.tickCap; index += 1) { + const before = service.getSnapshot(); + if ( + ['patient-zero-captured', 'infection-eliminated'].includes(before.status) + ) { + stoppedReason = 'terminal'; + break; + } + if (before.status === 'budget-exhausted') { + stoppedReason = 'budget-exhausted'; + break; + } + try { + await service.executeNextTick(); + } catch { + stoppedReason = + service.getSnapshot().status === 'budget-exhausted' + ? 'budget-exhausted' + : 'execution-error'; + break; + } + ticks.push( + tickSample( + service.getSnapshot(), + zeroAgentId, + variant === 'live-zero-deterministic-workers', + territory(before).controlledCells, + ), + ); + } + const snapshot = service.getSnapshot(); + const document = service.generateExperimentExport(exportRequest()); + const attempts = document.providerAttempts ?? []; + // The roster fixes Zero's identity for the run. This classifies failed + // attempts too, where a provider may have returned no metadata at all. + const openRouter = attempts.filter( + (attempt) => attempt.agentId === zeroAgentId, + ); + const jev = attempts.filter((attempt) => attempt.agentId !== zeroAgentId); + const tokenSum = ( + items: typeof attempts, + field: 'promptTokens' | 'completionTokens' | 'totalTokens', + ) => items.reduce((total, item) => total + (item.provider?.[field] ?? 0), 0); + const unknown = ( + items: typeof attempts, + field: 'promptTokens' | 'completionTokens' | 'totalTokens', + ) => items.some((item) => item.provider?.[field] === undefined); + const costs = openRouter.flatMap((attempt) => + attempt.actualCostCredits === undefined ? [] : [attempt.actualCostCredits], + ); + const terminal = ['patient-zero-captured', 'infection-eliminated'].includes( + snapshot.status, + ) + ? snapshot.status + : null; + for (const tick of ticks) { + const perTick = attempts.filter( + (attempt) => attempt.intendedTickNumber === tick.tick, + ); + const zeroAttempts = perTick.filter( + (attempt) => attempt.agentId === zeroAgentId, + ); + const workerAttempts = perTick.filter( + (attempt) => attempt.agentId !== zeroAgentId, + ); + tick.zeroPlanningAttempts = zeroAttempts.length; + tick.jevAttempts = workerAttempts.length; + tick.providerLatencyMs = perTick.reduce( + (sum, attempt) => + sum + (attempt.provider?.latencyMs ?? attempt.failure?.latencyMs ?? 0), + 0, + ); + tick.openRouterTokens = { + prompt: tokenSum(zeroAttempts, 'promptTokens'), + completion: tokenSum(zeroAttempts, 'completionTokens'), + total: tokenSum(zeroAttempts, 'totalTokens'), + }; + tick.typeSafeTokens = { + input: workerAttempts.reduce( + (sum, attempt) => + sum + + (attempt.reflexDecision?.inputTokens ?? + attempt.provider?.promptTokens ?? + 0), + 0, + ), + output: workerAttempts.reduce( + (sum, attempt) => + sum + + (attempt.reflexDecision?.outputTokens ?? + attempt.provider?.completionTokens ?? + 0), + 0, + ), + }; + const tickCosts = zeroAttempts.flatMap((attempt) => + attempt.actualCostCredits === undefined + ? [] + : [attempt.actualCostCredits], + ); + tick.actualOpenRouterCostCredits = tickCosts.length + ? tickCosts.reduce(addDecimal, '0') + : null; + tick.unknownProviderCostAttempts = zeroAttempts.length - tickCosts.length; + tick.unknownTypeSafeCostAttempts = workerAttempts.length; + } + if (terminal) stoppedReason = 'terminal'; + else if (snapshot.status === 'budget-exhausted') + stoppedReason = 'budget-exhausted'; + return { + seed, + ticks, + final: { + tick: snapshot.tickNumber, + activeWorkers: snapshot.world.agents.filter( + (agent) => agent.id !== zeroAgentId, + ).length, + captures: ticks.reduce((sum, tick) => sum + tick.captures, 0), + patientZeroCaptured: snapshot.status === 'patient-zero-captured', + terminalOutcome: terminal, + stoppedReason, + ...territory(snapshot), + }, + effort: { + zeroPlanningAttempts: openRouter.length, + jevAttempts: jev.length, + jevFallbacks: + variant === 'live-zero-deterministic-workers' + ? 0 + : ticks.reduce((sum, tick) => sum + tick.jevFallbacks, 0), + openRouterTokens: { + prompt: tokenSum(openRouter, 'promptTokens'), + completion: tokenSum(openRouter, 'completionTokens'), + total: tokenSum(openRouter, 'totalTokens'), + unknownFields: [ + 'promptTokens', + 'completionTokens', + 'totalTokens', + ].filter((field) => unknown(openRouter, field as 'promptTokens')), + }, + typeSafeTokens: { + input: jev.reduce( + (sum, attempt) => + sum + + (attempt.reflexDecision?.inputTokens ?? + attempt.provider?.promptTokens ?? + 0), + 0, + ), + output: jev.reduce( + (sum, attempt) => + sum + + (attempt.reflexDecision?.outputTokens ?? + attempt.provider?.completionTokens ?? + 0), + 0, + ), + unknownFields: jev.some( + (attempt) => + attempt.reflexDecision === undefined && + (attempt.provider?.promptTokens === undefined || + attempt.provider?.completionTokens === undefined), + ) + ? ['inputTokens', 'outputTokens'] + : [], + }, + providerLatencyMs: attempts.reduce( + (sum, attempt) => + sum + + (attempt.provider?.latencyMs ?? attempt.failure?.latencyMs ?? 0), + 0, + ), + actualOpenRouterCostCredits: costs.length + ? costs.reduce(addDecimal, '0') + : null, + unknownProviderCostAttempts: openRouter.filter( + (attempt) => attempt.actualCostCredits === undefined, + ).length, + typeSafeCostCredits: null, + unknownTypeSafeCostAttempts: jev.length, + attemptsWithoutCommittedTick: attempts.filter( + (attempt) => + attempt.intendedTickNumber === undefined || + attempt.intendedTickNumber > snapshot.tickNumber, + ).length, + }, + attemptAccounting: { + started: snapshot.experiment.attemptAccounting.attemptsStarted, + finalized: snapshot.experiment.attemptAccounting.attemptsFinalized, + exhausted: snapshot.experiment.attemptAccounting.exhausted, + exhaustionReason: snapshot.experiment.attemptAccounting.exhaustionReason, + }, + }; +} + +function aggregate( + runs: readonly LiveComparisonRun[], +): LiveComparisonVariantReport['aggregate'] { + const ticks = runs.flatMap((run) => run.ticks); + const confidences = ticks.flatMap((tick) => tick.jevConfidence); + const replanProbabilities = ticks.flatMap( + (tick) => tick.jevReplanProbabilities, + ); + const costs = runs.flatMap((run) => + run.effort.actualOpenRouterCostCredits === null + ? [] + : [run.effort.actualOpenRouterCostCredits], + ); + return { + runCount: runs.length, + captures: runs.reduce((sum, run) => sum + run.final.captures, 0), + patientZeroCapturedRuns: runs.filter((run) => run.final.patientZeroCaptured) + .length, + terminalRuns: runs.filter((run) => run.final.terminalOutcome !== null) + .length, + finalActiveWorkers: runs.reduce( + (sum, run) => sum + run.final.activeWorkers, + 0, + ), + finalControlledCells: runs.reduce( + (sum, run) => sum + run.final.controlledCells, + 0, + ), + disinfections: runs + .flatMap((run) => run.ticks) + .reduce((sum, tick) => sum + tick.disinfections, 0), + territoryGained: runs + .flatMap((run) => run.ticks) + .reduce((sum, tick) => sum + tick.territoryGained, 0), + territoryLost: runs + .flatMap((run) => run.ticks) + .reduce((sum, tick) => sum + tick.territoryLost, 0), + workerStalls: runs + .flatMap((run) => run.ticks) + .reduce((sum, tick) => sum + tick.workerStalls, 0), + workerReplanSignals: runs + .flatMap((run) => run.ticks) + .reduce((sum, tick) => sum + tick.workerReplanSignals, 0), + zeroReplans: runs + .flatMap((run) => run.ticks) + .reduce((sum, tick) => sum + tick.zeroReplans, 0), + zeroPlanningAttempts: runs.reduce( + (sum, run) => sum + run.effort.zeroPlanningAttempts, + 0, + ), + jevAttempts: runs.reduce((sum, run) => sum + run.effort.jevAttempts, 0), + jevFallbacks: runs.reduce((sum, run) => sum + run.effort.jevFallbacks, 0), + openRouterTokens: { + prompt: runs.reduce( + (sum, run) => sum + run.effort.openRouterTokens.prompt, + 0, + ), + completion: runs.reduce( + (sum, run) => sum + run.effort.openRouterTokens.completion, + 0, + ), + total: runs.reduce( + (sum, run) => sum + run.effort.openRouterTokens.total, + 0, + ), + }, + typeSafeTokens: { + input: runs.reduce( + (sum, run) => sum + run.effort.typeSafeTokens.input, + 0, + ), + output: runs.reduce( + (sum, run) => sum + run.effort.typeSafeTokens.output, + 0, + ), + }, + providerLatencyMs: runs.reduce( + (sum, run) => sum + run.effort.providerLatencyMs, + 0, + ), + meanJevConfidence: confidences.length + ? confidences.reduce((sum, value) => sum + value, 0) / confidences.length + : null, + meanJevReplanProbability: replanProbabilities.length + ? replanProbabilities.reduce((sum, value) => sum + value, 0) / + replanProbabilities.length + : null, + actualOpenRouterCostCredits: costs.length + ? costs.reduce(addDecimal, '0') + : null, + unknownProviderCostAttempts: runs.reduce( + (sum, run) => sum + run.effort.unknownProviderCostAttempts, + 0, + ), + unknownTypeSafeCostAttempts: runs.reduce( + (sum, run) => sum + run.effort.unknownTypeSafeCostAttempts, + 0, + ), + attemptsWithoutCommittedTick: runs.reduce( + (sum, run) => sum + run.effort.attemptsWithoutCommittedTick, + 0, + ), + }; +} + +/** Runs the primary Jev ablation with real providers supplied by the caller. */ +export async function runLiveComparison( + config: LiveComparisonConfig, + providers: LiveComparisonProviders, +): Promise { + validate(config); + const variants: LiveComparisonVariant[] = [ + 'live-zero-jev', + 'live-zero-deterministic-workers', + ]; + const reports: LiveComparisonVariantReport[] = []; + for (const variant of variants) { + const runs: LiveComparisonRun[] = []; + for (const seed of config.seeds) + runs.push(await runVariant(variant, seed, config, providers)); + reports.push({ variant, runs, aggregate: aggregate(runs) }); + } + return { + formatVersion: 1, + kind: 'live-zero-swarm-jev-ablation', + configuration: { + modelId: config.modelId, + seeds: [...config.seeds], + tickCap: config.tickCap, + agentCount: config.agentCount, + cognitionMode: 'zero-swarm-v1', + trailHunterProfile: 'trail-hunter-v1', + rosterSeed: 'worker-capture-roster', + patientZeroRosterIndex: 1, + worldSeedPattern: 'offline-world-{seed}', + planningPolicy: 'zero-swarm-v1-service-default', + objectiveVersion: 'durable-influence-v3', + executionLimits: { + providerAttemptLimit: config.providerAttemptLimit, + creditLimit: config.creditLimit, + reservationCreditsPerAttempt: config.reservationCreditsPerAttempt, + }, + reproducibility: + 'reproducible-input; provider output is nondeterministic', + }, + providerCostNote: + 'OpenRouter cost is provider-reported when available; TypeSafe monetary cost is unknown.', + variants: reports, + }; +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f2182b7..df1532e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -49,6 +49,13 @@ for safe completed exports, but is legacy-turn-centric and does not substitute for the same-scenario swarm harness. See [Zero-swarm offline comparison](ZERO_SWARM_COMPARISON.md). +The separately opted-in live comparison runs real OpenRouter Agent Zero with +either Jev workers or a deterministic legal-candidate worker policy, using +identical scenario inputs and the same Zero model for both variants. It keeps +the production planning cadence and engine resolution, and reports safe +per-tick observations alongside the independent provider-attempt ledger. See +[Live swarm comparison](LIVE_SWARM_COMPARISON.md). + ## Simultaneous tick authority Before the frozen agent snapshot, the optional seeded simulated-player profile diff --git a/docs/LIVE_SWARM_COMPARISON.md b/docs/LIVE_SWARM_COMPARISON.md new file mode 100644 index 0000000..d252ab7 --- /dev/null +++ b/docs/LIVE_SWARM_COMPARISON.md @@ -0,0 +1,48 @@ +# Live swarm comparison + +`compare:live` is a small, opt-in provider experiment. It compares the same Agent Zero planning configuration, scenario, roster, stable seeds, trail-hunter profile, and tick cap across two primary variants: + +- `live-zero-jev`: OpenRouter Agent Zero planning plus live TypeSafe Jev worker cognition. +- `live-zero-deterministic-workers`: the same OpenRouter Agent Zero planning with the server's deterministic legal-candidate selector and no TypeSafe calls. + +This is a Jev ablation, not a general claim that zero-swarm is better than legacy multi-agent mode. Legacy is deliberately not included in the first live command because its provider-call pattern and cognition are different. + +## Run a bounded experiment + +Choose the Zero model explicitly and supply both keys in the invoking shell. The command does not load `.env` files. It makes no network call without `--confirm-provider-costs`. + +```bash +OPENROUTER_API_KEY='…' TYPESAFE_API_KEY='…' \ +pnpm --silent --filter @hexzero/game-api exec node --import tsx \ + src/live-swarm-comparison-cli.ts --confirm-provider-costs --model 'provider/model-id' \ + > /tmp/hexzero-live-swarm-comparison.json +``` + +The default is three stable seeds, a 20-tick cap, eight agents, and `trail-hunter-v1`, with 200 attempts, 3 credits admission, and 0.02 credits reserved per attempt. A roster must contain two to eight agents: Zero plus at least one worker. Inputs can be repeated with `--seeds seed-a,seed-b,seed-c`; this gives reproducible inputs, while provider output remains nondeterministic. Limits are deliberately bounded: at most five seeds, 30 ticks, eight agents, 300 provider attempts, 10 credits admission, and 0.5 credits reserved per attempt. The existing attempt and credit admission controls apply per run; they are not an upstream provider billing guarantee. + +Use the saved report without credentials or acknowledgement: + +```bash +pnpm --silent --filter @hexzero/game-api exec node --import tsx \ + src/live-swarm-comparison-cli.ts --summary-from /tmp/hexzero-live-swarm-comparison.json +``` + +The direct `pnpm exec` invocation writes JSON only to stdout and never archives or commits a result. The shorter `pnpm compare:live` script is also available for interactive use, but pnpm may prepend script output to redirected stdout. Reports exclude API keys, raw prompts, raw provider responses, private reasoning, and raw error bodies. + +## Reading the report + +Reports retain per-tick and aggregate active workers, captures including Patient Zero capture, terminal outcomes, controlled territory and territory retained/lost, disinfection, worker stalls, directive progress, worker replan signals, Zero replans and reasons, Zero planning attempts, Jev attempts/fallbacks/confidence/action and replan probabilities, token counts, latency, and provider accounting. + +OpenRouter cost is provider-reported cost. Missing provider-cost fields remain explicitly unknown. TypeSafe responses provide input/output tokens but no authoritative monetary bill. If a deployment chooses to include `estimatedTypeSafeCost`, it must retain the Jev model, configured price per million input tokens, and `estimate: true`; it is never actual cost. + +Do not reduce a run to final territory. Review these questions after the first baseline dataset, without threshold tuning beforehand: + +1. Does Jev materially change worker behavior from deterministic workers, and is that useful under hunter pressure? +2. Does it improve survival or captures, including keeping Zero alive, while considering territory retained or sacrificed? +3. Does it reduce stalls and use worker replan signals appropriately? Inspect false signals and missed replanning opportunities. +4. How often does Zero need generative replanning, and why? +5. What are the measured OpenRouter costs, Jev token volume, clearly labeled TypeSafe estimate if configured, and unknown cost fields? +6. Inspect low-confidence choices, high-confidence bad-looking choices, repeated stalls, capture-alert response, and high trail-hunter-pressure response. Does the narrow Jev state/question keep deterministic work in code and confidence routing meaningful? +7. Is Jev worth its complexity against deterministic workers, and does zero-swarm offer more compelling gameplay than legacy mode? + +Do not retire legacy systems from this experiment alone. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 5a8752c..4f92bcd 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -73,6 +73,14 @@ deterministic fallback and remains visible as unavailable to the operator. The development API has no authentication, rate limiting, or provider-account balance enforcement. It does enforce server-owned per-experiment provider-attempt and conservative credit-admission ceilings configured by World Setup. Credit reservations bound admission exposure but cannot guarantee the upstream bill, especially when reported cost exceeds the operator's reservation. It binds to loopback and its CORS allowlist is limited to the documented local World Lab origins. Do not deploy its cost-incurring tick endpoint to unauthenticated public traffic. +The separate live comparison CLI requires an explicit provider-cost +acknowledgement and an operator-selected Zero model before it constructs live +providers. Each run has hard attempt and credit-admission caps. Its JSON report +contains only allowlisted scenario, world, decision, and usage metrics; no keys, +raw provider payloads, prompts, or private reasoning. OpenRouter's returned +cost is factual when present. Missing costs and TypeSafe monetary cost remain +unknown; the command never launches from default tests or CI. + ## Model-provider isolation Simultaneous ticks retain the same provider isolation. Every job receives a diff --git a/docs/TESTING.md b/docs/TESTING.md index 5bea9b9..8d17bef 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -29,6 +29,12 @@ aggregate metrics, and has no provider, credential, or archive-write path. Run `pnpm compare:offline` (or bounded `--ticks` and `--seeds` options) to produce its JSON report. See [Zero-swarm offline comparison](ZERO_SWARM_COMPARISON.md). +The live comparison command is a separate paid experiment. Its tests inject +scripted providers and verify the acknowledgement gate, input parity, bounded +attempt admission, safe metrics, and a no-network Markdown summary path. +`pnpm validate`, Playwright, and GitHub CI never execute a paid run. See +[Live swarm comparison](LIVE_SWARM_COMPARISON.md). + The pre-live provider audit tests the Jev request's bounded capture-pressure projection, omission of hidden hunter state and empty capture noise, and the removal of the inert recent-facts field. Planner tests cover provider-reported diff --git a/package.json b/package.json index 2d142f2..e250c0c 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "scripts": { "build": "pnpm -r --if-present build", "compare:offline": "pnpm --filter @hexzero/game-api compare:offline", + "compare:live": "pnpm --filter @hexzero/game-api compare:live", "dev": "pnpm --parallel --filter @hexzero/game-api --filter @hexzero/world-lab dev", "dev:test-provider": "HEXZERO_PROVIDER=scripted pnpm dev", "dev:api": "pnpm --filter @hexzero/game-api dev",