Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ policies. The first comparison retains legacy mode pending real-provider trials;
see `docs/ZERO_SWARM_COMPARISON.md`. See ADRs 0028-0032. Historical milestones
below remain as implementation history.

PR #58 closes the pre-live Jev capture-context and Agent Zero cost-accounting
gaps. PR #59 adds an explicitly opted-in live comparison of Jev workers against
deterministic workers while holding the Zero model and scenario inputs constant.
Legacy retirement remains a later decision after measured live results.

## Experimental Patient Zero coordinator

The current focused slice requires one normal physical agent, provides a
Expand Down
1 change: 1 addition & 0 deletions apps/game-api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"scripts": {
"build": "esbuild src/server.ts --bundle --platform=node --format=esm --outfile=dist/server.js",
"compare:offline": "node --import tsx src/swarm-comparison-cli.ts",
"compare:live": "node --import tsx src/live-swarm-comparison-cli.ts",
"dev": "tsx watch src/server.ts",
"lint": "eslint .",
"start": "node dist/server.js",
Expand Down
139 changes: 139 additions & 0 deletions apps/game-api/src/live-swarm-comparison-cli.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
import { describe, expect, it, vi } from 'vitest';
import {
markdownSummary,
parseLiveComparisonArguments,
runLiveComparisonCli,
} from './live-swarm-comparison-cli';

describe('live swarm comparison CLI', () => {
it('refuses before constructing providers or calling the runner without cost acknowledgement', async () => {
const runComparison = vi.fn();
const providers = vi.fn();
await expect(
runLiveComparisonCli(['--model', 'openrouter/test'], {
runComparison,
providers,
readFile: vi.fn(),
env: { OPENROUTER_API_KEY: 'openrouter', TYPESAFE_API_KEY: 'typesafe' },
} as never),
).rejects.toThrow('--confirm-provider-costs');
expect(runComparison).not.toHaveBeenCalled();
expect(providers).not.toHaveBeenCalled();
});

it('requires an explicit model after acknowledgement without invoking providers', async () => {
const runComparison = vi.fn();
const providers = vi.fn();
await expect(
runLiveComparisonCli(['--confirm-provider-costs'], {
runComparison,
providers,
readFile: vi.fn(),
env: { OPENROUTER_API_KEY: 'openrouter', TYPESAFE_API_KEY: 'typesafe' },
} as never),
).rejects.toThrow('--model is required');
expect(runComparison).not.toHaveBeenCalled();
expect(providers).not.toHaveBeenCalled();
});

it('reads a saved report for a summary without acknowledgement, credentials, or providers', async () => {
const runComparison = vi.fn();
const providers = vi.fn();
const report = {
configuration: { modelId: 'openrouter/test', seeds: ['a'], tickCap: 20 },
variants: [
{
variant: 'live-zero-jev',
aggregate: {
totalCaptures: 1,
totalWorkerStalls: 2,
openRouterTokens: { prompt: 10, completion: 4, total: 14 },
},
},
],
};
await expect(
runLiveComparisonCli(['--summary-from', '/tmp/report.json'], {
runComparison,
providers,
readFile: vi.fn().mockResolvedValue(JSON.stringify(report)),
env: {},
} as never),
).resolves.toContain('live-zero-jev');
expect(runComparison).not.toHaveBeenCalled();
expect(providers).not.toHaveBeenCalled();
});

it('rejects unsafe aggregate text in a hand-edited saved report', async () => {
await expect(
runLiveComparisonCli(['--summary-from', '/tmp/report.json'], {
runComparison: vi.fn(),
providers: vi.fn(),
readFile: vi.fn().mockResolvedValue(
JSON.stringify({
configuration: {
modelId: 'openrouter/test',
seeds: ['a'],
tickCap: 20,
},
variants: [
{
variant: 'live-zero-jev',
aggregate: { captures: '| injected Markdown |' },
},
],
}),
),
env: {},
} as never),
).rejects.toThrow('invalid shape');
});

it('uses the bounded defaults and rejects oversized live inputs', () => {
expect(
parseLiveComparisonArguments([
'--confirm-provider-costs',
'--model',
'openrouter/test',
]),
).toMatchObject({
seeds: ['worker-capture-spawn', 'comparison-seed-b', 'comparison-seed-c'],
ticks: 20,
agents: 8,
attempts: 200,
creditLimit: '3',
reservation: '0.02',
});
expect(() =>
parseLiveComparisonArguments([
'--model',
'openrouter/test',
'--ticks',
'31',
]),
).toThrow('between 1 and 30');
expect(() =>
parseLiveComparisonArguments([
'--model',
'openrouter/test',
'--agents',
'1',
]),
).toThrow('between 2 and 8');
});

it('keeps survival, territory, stalls, replans, and cost separate in summaries', () => {
expect(
markdownSummary({
configuration: {
modelId: 'openrouter/test',
seeds: ['a'],
tickCap: 20,
},
variants: [
{ variant: 'live-zero-jev', aggregate: { totalCaptures: 1 } },
],
} as never),
).toContain('This summary does not select a winner.');
});
});
Loading
Loading