From 5cdc240138abaa65751483430d18864b94f51d9c Mon Sep 17 00:00:00 2001 From: Christopher Nelson Date: Thu, 20 Aug 2026 04:03:57 -0400 Subject: [PATCH] feat: add local experiment archive --- .gitignore | 2 + .prettierignore | 1 + README.md | 19 +- ROADMAP.md | 2 + docs/ARCHITECTURE.md | 2 + docs/EXPERIMENT_ARCHIVE.md | 78 ++ docs/SECURITY.md | 4 +- docs/TESTING.md | 2 + package.json | 1 + packages/experiment-archive/package.json | 25 + .../experiment-archive/src/archive.test.ts | 413 ++++++++ packages/experiment-archive/src/cli.ts | 297 ++++++ packages/experiment-archive/src/database.ts | 112 +++ packages/experiment-archive/src/importer.ts | 795 +++++++++++++++ packages/experiment-archive/src/index.ts | 4 + packages/experiment-archive/src/migrations.ts | 268 +++++ packages/experiment-archive/src/notes.ts | 213 ++++ .../experiment-archive/src/query-service.ts | 921 ++++++++++++++++++ packages/experiment-archive/tsconfig.json | 4 + pnpm-lock.yaml | 19 + 20 files changed, 3173 insertions(+), 9 deletions(-) create mode 100644 docs/EXPERIMENT_ARCHIVE.md create mode 100644 packages/experiment-archive/package.json create mode 100644 packages/experiment-archive/src/archive.test.ts create mode 100644 packages/experiment-archive/src/cli.ts create mode 100644 packages/experiment-archive/src/database.ts create mode 100644 packages/experiment-archive/src/importer.ts create mode 100644 packages/experiment-archive/src/index.ts create mode 100644 packages/experiment-archive/src/migrations.ts create mode 100644 packages/experiment-archive/src/notes.ts create mode 100644 packages/experiment-archive/src/query-service.ts create mode 100644 packages/experiment-archive/tsconfig.json diff --git a/.gitignore b/.gitignore index 9b3b53e..6c6bb28 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ test-results/ *.tsbuildinfo .DS_Store *.log +logs/ +.agentborne/ diff --git a/.prettierignore b/.prettierignore index 8558ccf..7a7f2cf 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,2 @@ **/next-env.d.ts +logs/ diff --git a/README.md b/README.md index 2461d4e..ee26ff0 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,14 @@ rules as every other agent. The universal flat provider contract is ## Workspace -| Path | Responsibility | -| ------------------------ | ---------------------------------------------------------------- | -| `apps/world-lab` | Next.js developer/admin map, controls, inspector, and event log | -| `apps/game-api` | Hono HTTP boundary and in-memory simulation service | -| `packages/world-engine` | Pure world validation and consequence application | -| `packages/agent-runtime` | OpenRouter provider boundary and explicit scripted testing seams | -| `packages/shared` | Runtime-validated schemas and inferred domain types | +| Path | Responsibility | +| ----------------------------- | ---------------------------------------------------------------- | +| `apps/world-lab` | Next.js developer/admin map, controls, inspector, and event log | +| `apps/game-api` | Hono HTTP boundary and in-memory simulation service | +| `packages/world-engine` | Pure world validation and consequence application | +| `packages/agent-runtime` | OpenRouter provider boundary and explicit scripted testing seams | +| `packages/shared` | Runtime-validated schemas and inferred domain types | +| `packages/experiment-archive` | Durable SQLite imports and bounded research queries | ## Local development @@ -52,7 +53,7 @@ The command reads only `OPENROUTER_API_KEY` from the repository-root `.env`; its The compatible default centers on Toledo, Ohio (`41.6528, -83.5379`) at H3 resolution 9 and renders the same deterministic radius-six disk of exactly 127 cells with eight fixed perimeter starts. World Setup previews and applies resolution 8–11 scenarios with 1–32 agents, radius at most 40, at most 5,000 actual generated cells, and a 12 km default physical communication range. Schema-v9 exports preserve the authoritative scenario and `durable-influence-v2` attribution. MapLibre uses CARTO Dark Matter's tokenless raster tiles with `© OpenStreetMap contributors © CARTO` attribution. -Alliance leadership, merging, custom metadata, combat systems, relationship scores, group chat, persistent memory, player mechanics, persistence, and autonomous scheduling remain deferred. +Alliance leadership, merging, custom metadata, combat systems, relationship scores, group chat, persistent memory, player mechanics, restartable world persistence, and autonomous scheduling remain deferred. When every development cell is infected, World Lab automatically pauses playback and disables Start to avoid accidental provider calls. Reset and export remain available, and Single turn remains an explicitly manual diagnostic action. @@ -63,3 +64,5 @@ The persistent operator shell keeps execution controls, run target, playback spe The agent roster defaults to browser-local **Follow turn** behavior: the inspector follows the active request, or the next scheduled agent while paused. Selecting an agent manually disables following without hiding the roster's textual Acting/Next marker; the preference remains in that browser and is never exported. Public world chat and the event log are newest-first bounded feeds, and the shared Model and Export dialogs keep their headers/actions fixed while their bodies scroll within the viewport. See [Testing](docs/TESTING.md), [Architecture](docs/ARCHITECTURE.md), [Security](docs/SECURITY.md), the accepted future [Gameplay Foundation](docs/GAMEPLAY_FOUNDATION.md), and the [Roadmap](ROADMAP.md). + +Completed schema-v9 exports can be imported into an ignored local SQLite archive and queried without repeatedly loading full JSON artifacts. See [Local experiment archive](docs/EXPERIMENT_ARCHIVE.md). diff --git a/ROADMAP.md b/ROADMAP.md index 0006045..01607b9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -75,6 +75,8 @@ Twelfth focused slice: add scenario-owned physical communication range, private ## PR 5 — Goals and memory +Pre-PR-5 observability slice: add a local append-only SQLite experiment archive, transactional schema-v9 export import, bounded human/Codex queries, normalized comparisons, and FTS-searchable curated notes. The in-memory engine remains authoritative. Crash recovery, restartable simulation state, MCP, embeddings, vector search, and a database browser remain deferred. + Persistent short- and long-term objectives, compact memories, plan revision, summaries, and longer simulation runs. ## PR 6 — Persistent autonomous world diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index dfca911..bb20376 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -16,6 +16,8 @@ World Lab → Game API / simulation service → agent runtime → one decision The Game API also owns one process-local experiment record. Each completed safe turn is captured once, independently from the browser snapshot, and server-side export levels filter that record without affecting provider requests. +Completed schema-v9 exports may cross a separate offline archive boundary into `packages/experiment-archive`. Node's built-in SQLite stores normalized immutable research records through versioned migrations, foreign keys, prepared statements, and transactional idempotent imports. This downstream observability archive is never consulted by turn execution and cannot recover, resume, or mutate the active world. Its bounded query service is application-independent so a future read-only MCP adapter can reuse it without exposing arbitrary SQL. + World Setup uses `world-scenario-v1`. Pure preview computes the actual H3 disk, exact count, summed cell area, deterministic roster/spawns, feasibility, and warnings. Apply recomputes and atomically replaces world and experiment state. Reset reconstructs the current scenario; the Toledo default preserves legacy starts. Explicit location search crosses a replaceable server-owned adapter with no autocomplete, a one-request-per-second Nominatim limit, bounded cache/timeout, normalized results, and OpenStreetMap attribution. Manual coordinates bypass that network boundary. World Lab may opt into browser-driven unattended recovery for continuous Start or Run-to sessions. After the existing initial call and at most one internal repair/transport retry, the browser serially requests one-call unattended retries up to a local limit of one through three, then commits one attributed unattended skip. Every mutation has a fresh idempotency key and uses the authoritative pending turn. Pausing, cancellation, ineligible failures, or failed reconciliation stops the loop. The tab must remain open; this is not scheduling, simultaneous ticks, or a production recovery service. diff --git a/docs/EXPERIMENT_ARCHIVE.md b/docs/EXPERIMENT_ARCHIVE.md new file mode 100644 index 0000000..4569c19 --- /dev/null +++ b/docs/EXPERIMENT_ARCHIVE.md @@ -0,0 +1,78 @@ +# Local experiment archive + +The experiment archive is a durable, local research surface for completed or partially retained exports. It does not participate in an active simulation: the Game API's in-memory engine remains authoritative, and an archive write cannot change an accepted game outcome. This slice imports schema-v9 JSON exports; it is not crash recovery, restartable simulation state, or a scheduler. + +## Storage and configuration + +`@agentborne/experiment-archive` uses SQLite built into the pinned Node 24 runtime. Versioned migrations create strict tables with foreign keys and indexes. File-backed databases enable WAL and a five-second busy timeout; imports use prepared statements inside one transaction. Tests use in-memory or temporary databases. + +The default database is `.agentborne/experiments.sqlite`, an ignored development path. Override it with `--db ` or `AGENTBORNE_EXPERIMENT_DB`; the flag takes precedence. + +The schema normalizes experiments, source exports, roster/assignments, topology, turns, model attempts, communications and recipients, diplomacy attempts, alliance events, world events, configuration changes, and research notes. Stable source IDs are retained; deterministic experiment/turn/attempt IDs make imports idempotent. Source aggregate metrics remain audit evidence, while summaries derive canonical metrics from stored records. + +Retention is never silently upgraded. A filtered import, missing optional observation, or truncated source remains visible as incomplete or missing data. Import validates the complete export and rejects prohibited credential/private-reasoning fields before beginning its transaction. Persistence errors roll back and surface explicitly. + +## Commands + +```bash +pnpm experiment:db import ./exports/run.json +pnpm experiment:db list +pnpm experiment:db summary +pnpm experiment:db compare +pnpm experiment:db turns --agent --from-turn 40 --to-turn 80 +pnpm experiment:db communications --channel direct --recipient +pnpm experiment:db alliance-events --reason expired +pnpm experiment:db patient-zero --from-turn 1 --to-turn 100 +pnpm experiment:db failures --reason invalid-json +``` + +Commands support `--format table|json|markdown`; table is the concise default. Detail queries default to 50 rows and clamp limits to 500. Filters are exact, ordering is deterministic, and arbitrary SQL is not exposed. + +Summary covers scenario/roster attribution, outcomes and retries, action distribution, territory, communication channels, alliance lifecycle/rejections, Patient Zero behavior, usage, size trends, retention, and inconsistencies. Comparison reports absolute totals plus per-turn, per-agent-turn, per-active-agent, and per-Patient-Zero-turn rates. + +`directionChangesAfterCommunication` has one canonical definition: for each agent independently, count an accepted move whose direction differs from that agent's previous accepted move when the retained observation contains an inbound direct or alliance message after the previous move and no later than the current turn. Aggregate is exactly the sum of per-agent counts. Original imported values remain unchanged in source-metric audit JSON, and disagreements are reported. + +Patient Zero output distinguishes: + +- `message-to-patient-zero`: an accepted direct message without a qualifying earlier directive. +- `reply-after-directive`: an accepted direct message after a Zero directive addressed to that sender and before a later directive to that sender. +- `observable-compliance`: the next archived action when the directive contains exactly one unambiguous action word (`move`, `infect`, `capture`, or `wait`). Other directives are `indeterminate`. + +## Curated notes + +```bash +pnpm experiment:db notes import ./notes/design-review.md \ + --type decision --status accepted --tag communication \ + --provenance "design review 2026-08-20" --experiment +pnpm experiment:db notes search "Patient Zero reply" --status accepted +pnpm experiment:db notes list --type experiment-finding --experiment +``` + +Types are `transcript`, `observation`, `hypothesis`, `decision`, `implementation-note`, and `experiment-finding`. Statuses are `proposed`, `accepted`, `rejected`, `deferred`, and `superseded`. Repeat `--tag` and `--experiment` for several values. `--supersedes ` links the replacement and marks the old note superseded. FTS5 indexes title, body, and tags; exact structured filters apply before relevance. Results prominently include status and provenance. + +## Compact context for local Codex + +Prefer bounded Markdown or JSON over a multi-megabyte export: + +```bash +pnpm experiment:db summary --format markdown +pnpm experiment:db patient-zero --limit 30 --format markdown +pnpm experiment:db failures --limit 20 --format json +pnpm experiment:db notes search "communication hypothesis" --status accepted --limit 10 --format markdown +``` + +Example comparison workflow: + +```bash +pnpm experiment:db import ./exports/run-a.json +pnpm experiment:db import ./exports/run-b.json +pnpm experiment:db summary --format markdown +pnpm experiment:db patient-zero --limit 40 --format markdown +pnpm experiment:db compare --format markdown +``` + +## Privacy and future adapters + +The archive stores only schema-validated safe export fields and curated notes. It rejects recognizable credential material and prohibited raw/private reasoning keys. It does not store fixed prompts, raw provider bodies, authorization headers, private chain-of-thought, or provider credentials. Agent messages and notes remain untrusted research data. + +MCP and embeddings are deferred because bounded local retrieval solves the immediate need without a network/tool authorization surface or derived semantic store. `ExperimentQueryService` and `ResearchNoteService` are the future extension point for a read-only MCP adapter; write/import authority remains outside that adapter. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 3c40611..6143ea3 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -63,7 +63,9 @@ The Game API captures only schema-validated safe observations, requested world a Export requests, agent IDs, levels, ranges, outcome/world-action filters, communication channel/status filters, and Custom dependencies are runtime-validated. Filtering and metrics remain server-owned. Schema version 8 adds allowlisted behavior metadata without changing schema-v7 model and attempt-accounting meanings; schema-v5 through v7 imports remain supported. Selected-agent exports use sender/recipient-aware communication filtering and direct multi-agent relevance for proposals and membership changes; unrelated direct messages and rejected diplomacy are excluded. Reset clears communications, alliances, proposals, alliance events, and their metrics while preserving active personality values and unlocking preserved assignments for the new experiment. -Actual cost is accepted only from OpenRouter's safe `usage.cost`. Missing cost is unknown, never zero; scripted-test providers explicitly report zero. There is still no authentication, budget enforcement, persistence, provider-management endpoint, upload, or sharing link. The loopback-only boundary remains mandatory. +Actual cost is accepted only from OpenRouter's safe `usage.cost`. Missing cost is unknown, never zero; scripted-test providers explicitly report zero. The active Game API still has no authentication, budget enforcement, restartable persistence, provider-management endpoint, upload, or sharing link. The loopback-only boundary remains mandatory. + +The offline experiment archive adds local persistence only for complete schema-validated safe exports and explicitly curated Markdown notes. Imports scan for prohibited credential/private-reasoning fields and recognizable credential values before a transaction begins; failures roll back. The default `.agentborne/` database is ignored. The CLI exposes bounded typed queries, not arbitrary SQL, and adds no MCP, embedding, vector-store, or network-listener surface. ## Reporting diff --git a/docs/TESTING.md b/docs/TESTING.md index 28333b2..b31270c 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -103,3 +103,5 @@ consumption, cancellation during backoff, and the unchanged two-call ceiling. The separately opted-in `pnpm smoke:openrouter -- ` command makes one bounded OpenRouter call and validates the decision. It reads only `OPENROUTER_API_KEY` from `.env`; the model is an explicit argument. It is intentionally absent from default validation, Playwright, and CI and must not be run without deliberate authorization because it incurs third-party cost. World Lab also offers an explicit “Test selected model” probe. It uses the production text/flat-JSON contract and selected reasoning profile, does not advance or mutate the world, may incur a small charge, and is cached by model plus profile plus contract version. It is never invoked by deterministic validation or CI. + +Experiment-archive tests generate current schema-v9 exports through the deterministic scripted provider and cover migration/reopen, transactional rollback, stable idempotent identifiers, normalized round trips, bounded filters, summary/comparison calculations, canonical metric agreement with preserved source inconsistencies, Patient Zero classifications, retention completeness, FTS notes/supersession, explicit persistence failures, and credential/private-reasoning rejection. They use only in-memory or temporary SQLite databases and make no live provider or geocoding requests. diff --git a/package.json b/package.json index a2ad1d3..591786a 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "dev": "pnpm --parallel --filter @agentborne/game-api --filter @agentborne/world-lab dev", "dev:test-provider": "AGENTBORNE_PROVIDER=scripted pnpm dev", "dev:api": "pnpm --filter @agentborne/game-api dev", + "experiment:db": "pnpm --filter @agentborne/experiment-archive cli", "format": "prettier --write .", "format:check": "prettier --check .", "lint": "pnpm -r --if-present lint", diff --git a/packages/experiment-archive/package.json b/packages/experiment-archive/package.json new file mode 100644 index 0000000..816ebfc --- /dev/null +++ b/packages/experiment-archive/package.json @@ -0,0 +1,25 @@ +{ + "name": "@agentborne/experiment-archive", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "cli": "tsx src/cli.ts", + "lint": "eslint .", + "test": "vitest run", + "typecheck": "tsc -p tsconfig.json" + }, + "dependencies": { + "@agentborne/shared": "workspace:*" + }, + "devDependencies": { + "@agentborne/agent-runtime": "workspace:*", + "@agentborne/world-engine": "workspace:*", + "tsx": "4.23.12", + "vitest": "4.1.10" + } +} diff --git a/packages/experiment-archive/src/archive.test.ts b/packages/experiment-archive/src/archive.test.ts new file mode 100644 index 0000000..451f04b --- /dev/null +++ b/packages/experiment-archive/src/archive.test.ts @@ -0,0 +1,413 @@ +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { + AgentProvider, + ProviderDecision, +} from '@agentborne/agent-runtime'; +import { + experimentExportDocumentSchema, + type AgentObservation, + type ExperimentExportDocument, +} from '@agentborne/shared'; +import { defaultWorldSetupRequest } from '@agentborne/world-engine'; +import { describe, expect, it } from 'vitest'; + +import { SimulationService } from '../../../apps/game-api/src/simulation-service.js'; +import { + ArchiveDatabase, + ExperimentImportError, + ExperimentQueryService, + ResearchNoteService, + importExperimentExport, +} from './index.js'; + +const EXPERIMENT_ID = '10000000-0000-4000-8000-000000000001'; +const NOW = '2026-08-20T12:00:00.000Z'; + +function nextUuid() { + let counter = 10; + return () => `20000000-0000-4000-8000-${String(counter++).padStart(12, '0')}`; +} + +async function currentExport(): Promise { + let firstOtherAgent: string | undefined; + const provider: AgentProvider = { + mode: 'scripted-test', + model: 'archive-test-model', + configured: true, + async decide(observation: AgentObservation): Promise { + let worldAction: ProviderDecision['decision']['worldAction'] = { + type: 'wait', + }; + let communication: ProviderDecision['decision']['communication']; + if (observation.patientZero.isPatientZero) { + communication = { + channel: 'zero', + message: 'Infect your current cell.', + }; + } else if (!firstOtherAgent) { + firstOtherAgent = observation.agentId; + worldAction = { type: 'infect' }; + communication = { + channel: 'direct', + recipientId: observation.patientZero.agentId!, + message: 'I will infect now.', + }; + } + return { + decision: { + worldAction, + communication, + summary: 'Deterministic archive fixture.', + }, + metadata: { + provider: 'scripted-test', + model: 'archive-test-model', + latencyMs: 12, + promptTokens: 100, + completionTokens: 10, + totalTokens: 110, + costCredits: 0, + }, + }; + }, + }; + const eventId = nextUuid(); + const simulation = new SimulationService({ + provider, + now: () => NOW, + createEventId: eventId, + createExperimentId: () => EXPERIMENT_ID, + createAllianceId: eventId, + createProposalId: eventId, + }); + const snapshot = simulation.getSnapshot(); + const setup = defaultWorldSetupRequest(); + simulation.applyWorldSetup({ + ...setup, + patientZeroAgentId: setup.roster[0]!.id, + modelConfiguration: snapshot.modelConfiguration, + behaviorConfiguration: snapshot.behaviorConfiguration, + }); + for (let index = 0; index < 10; index += 1) + await simulation.executeNextTurn(); + return simulation.generateExperimentExport({ + agents: { mode: 'all' }, + turns: { mode: 'entire-retained' }, + outcomes: ['accepted', 'rejected', 'provider-error', 'operator-skipped'], + actions: ['move', 'infect', 'capture', 'wait'], + communications: { channel: 'all', status: 'all' }, + level: 'full-safe', + serialization: 'compact', + }); +} + +function temporaryPath(name: string): string { + return join(mkdtempSync(join(tmpdir(), 'agentborne-archive-')), name); +} + +describe('experiment archive', () => { + it('migrates a fresh database and reopens it without replaying migrations', () => { + const path = temporaryPath('archive.sqlite'); + const first = new ArchiveDatabase({ path, clock: () => new Date(NOW) }); + expect( + first.database + .prepare('SELECT COUNT(*) AS count FROM schema_migrations') + .get(), + ).toEqual({ count: 1 }); + expect(first.database.prepare('PRAGMA journal_mode').get()).toEqual({ + journal_mode: 'wal', + }); + first.close(); + const reopened = new ArchiveDatabase({ path, clock: () => new Date(NOW) }); + expect(reopened.database.prepare('PRAGMA foreign_keys').get()).toEqual({ + foreign_keys: 1, + }); + expect( + reopened.database + .prepare('SELECT COUNT(*) AS count FROM schema_migrations') + .get(), + ).toEqual({ count: 1 }); + reopened.close(); + }); + + it('imports current JSON exports transactionally, round-trips structured records, and is idempotent', async () => { + const document = await currentExport(); + const path = temporaryPath('export.json'); + writeFileSync(path, JSON.stringify(document)); + const archive = new ArchiveDatabase({ + path: ':memory:', + clock: () => new Date(NOW), + }); + const first = importExperimentExport(archive, path); + const second = importExperimentExport(archive, path); + expect(first.inserted).toBeGreaterThan(10); + expect(first.rejected).toBe(0); + expect(second).toMatchObject({ inserted: 0, existing: 1, rejected: 0 }); + expect( + archive.database.prepare('SELECT COUNT(*) AS count FROM turns').get(), + ).toEqual({ count: 10 }); + expect( + archive.database + .prepare('SELECT COUNT(DISTINCT id) AS count FROM communications') + .get(), + ).toEqual({ count: 3 }); + expect( + experimentExportDocumentSchema.parse( + JSON.parse(readFileSync(path, 'utf8')), + ), + ).toEqual(document); + archive.close(); + }); + + it('rolls back malformed and unsupported exports without leaving partial state', async () => { + const archive = new ArchiveDatabase({ path: ':memory:' }); + const malformed = temporaryPath('malformed.json'); + writeFileSync(malformed, '{ nope'); + expect(() => importExperimentExport(archive, malformed)).toThrow( + ExperimentImportError, + ); + const unsupported = temporaryPath('unsupported.json'); + writeFileSync( + unsupported, + JSON.stringify({ ...(await currentExport()), schemaVersion: 8 }), + ); + expect(() => importExperimentExport(archive, unsupported)).toThrow( + ExperimentImportError, + ); + expect( + archive.database + .prepare('SELECT COUNT(*) AS count FROM experiments') + .get(), + ).toEqual({ count: 0 }); + archive.close(); + }); + + it('uses stable record IDs and prevents duplicates across differently serialized source files', async () => { + const document = await currentExport(); + const archive = new ArchiveDatabase({ path: ':memory:' }); + importExperimentExport(archive, document); + const pretty = temporaryPath('pretty.json'); + writeFileSync(pretty, JSON.stringify(document, null, 2)); + const report = importExperimentExport(archive, pretty); + expect(report.existing).toBeGreaterThan(10); + expect( + archive.database.prepare('SELECT COUNT(*) AS count FROM turns').get(), + ).toEqual({ count: 10 }); + expect( + archive.database + .prepare('SELECT id FROM turns ORDER BY turn_number LIMIT 1') + .get(), + ).toEqual({ id: `${EXPERIMENT_ID}:turn:1` }); + archive.close(); + }); + + it('applies deterministic query filters, ordering, and enforced limits', async () => { + const archive = new ArchiveDatabase({ path: ':memory:' }); + const document = await currentExport(); + importExperimentExport(archive, document); + const queries = new ExperimentQueryService(archive); + const agent = document.agents[0]!.id; + const filtered = queries.turns(EXPERIMENT_ID, { + agent, + fromTurn: 1, + toTurn: 9, + action: 'wait', + outcome: 'accepted', + limit: 1, + }); + expect(filtered.limit).toBe(1); + expect(filtered.rows).toHaveLength(1); + expect(filtered.rows[0]).toMatchObject({ + agent, + action: 'wait', + outcome: 'accepted', + }); + expect(queries.turns(EXPERIMENT_ID, { limit: 50_000 }).limit).toBe(500); + expect( + queries.communications(EXPERIMENT_ID, { + channel: 'direct', + recipient: document.experiment.scenario!.patientZeroAgentId!, + }).rows, + ).toEqual([ + expect.objectContaining({ channel: 'direct', status: 'accepted' }), + ]); + const ordered = queries.turns(EXPERIMENT_ID).rows.map(({ turn }) => turn); + expect(ordered).toEqual( + [...ordered].sort((left, right) => Number(left) - Number(right)), + ); + archive.close(); + }); + + it('derives summaries and normalized comparisons without loading observations', async () => { + const archive = new ArchiveDatabase({ path: ':memory:' }); + importExperimentExport(archive, await currentExport()); + const queries = new ExperimentQueryService(archive); + const summary = queries.summary(EXPERIMENT_ID); + expect(summary).toMatchObject({ + turns: { requested: 10, accepted: 10, failed: 0, lost: 0 }, + retention: { complete: true, archivedTurns: 10 }, + usage: { aggregate: { modelAttempts: 10, totalTokens: 1100 } }, + }); + const comparison = queries.compare(EXPERIMENT_ID, EXPERIMENT_ID); + expect(comparison.delta).toEqual( + expect.objectContaining({ communicationsPerTurn: 0, acceptedPerTurn: 0 }), + ); + archive.close(); + }); + + it('defines canonical direction changes as an agreeing sum while retaining inconsistent source evidence', async () => { + const document = await currentExport(); + document.metrics!.aggregate.directionChangesAfterCommunication = 7; + document.metrics!.byAgent[0]!.metrics.directionChangesAfterCommunication = 2; + const archive = new ArchiveDatabase({ path: ':memory:' }); + importExperimentExport(archive, document); + const summary = new ExperimentQueryService(archive).summary(EXPERIMENT_ID); + expect(summary.directionChangesAfterCommunication).toMatchObject({ + aggregate: 0, + agreement: true, + }); + expect(summary.inconsistencies).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + metric: 'directionChangesAfterCommunication', + aggregate: 7, + perAgentSum: 2, + }), + ]), + ); + const stored = archive.database + .prepare('SELECT source_metrics_json AS metrics FROM experiments') + .get() as { metrics: string }; + expect( + JSON.parse(stored.metrics).aggregate.directionChangesAfterCommunication, + ).toBe(7); + archive.close(); + }); + + it('distinguishes messages to Patient Zero, replies after directives, and observable compliance', async () => { + const archive = new ArchiveDatabase({ path: ':memory:' }); + importExperimentExport(archive, await currentExport()); + const rows = new ExperimentQueryService(archive).patientZero( + EXPERIMENT_ID, + { limit: 100 }, + ).rows; + expect(rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'directive' }), + expect.objectContaining({ kind: 'reply-after-directive' }), + expect.objectContaining({ + kind: 'observable-compliance', + classification: 'compliant', + observedAction: 'infect', + }), + ]), + ); + archive.close(); + }); + + it('supports FTS note search, exact status filters, experiment links, and supersession', async () => { + const archive = new ArchiveDatabase({ + path: ':memory:', + clock: () => new Date(NOW), + }); + importExperimentExport(archive, await currentExport()); + const notes = new ResearchNoteService(archive); + const oldPath = temporaryPath('old.md'); + writeFileSync( + oldPath, + '# Communication idea\nTry a noisy broadcast experiment.', + ); + const old = notes.import(oldPath, { + type: 'hypothesis', + status: 'proposed', + tags: ['communication'], + experiments: [EXPERIMENT_ID], + provenance: 'design chat 12', + }); + const newPath = temporaryPath('new.md'); + writeFileSync( + newPath, + '# Communication decision\nUse bounded direct communication.', + ); + const replacement = notes.import(newPath, { + type: 'decision', + status: 'accepted', + tags: ['communication'], + experiments: [EXPERIMENT_ID], + provenance: 'ADR review', + supersedes: old.id, + }); + expect(notes.search('communication', { status: 'accepted' }).rows).toEqual([ + expect.objectContaining({ + id: replacement.id, + status: 'accepted', + provenance: 'ADR review', + }), + ]); + expect( + notes.list({ status: 'superseded', experiment: EXPERIMENT_ID }).rows, + ).toEqual([expect.objectContaining({ id: old.id, status: 'superseded' })]); + archive.close(); + }); + + it('reports incomplete retention and missing selected history explicitly', async () => { + const document = await currentExport(); + document.retention = { + ...document.retention, + totalCompletedTurns: 20, + droppedRecords: 10, + complete: false, + requestedRangeExtendsBeyondRetention: true, + }; + const archive = new ArchiveDatabase({ path: ':memory:' }); + importExperimentExport(archive, document); + expect( + new ExperimentQueryService(archive).summary(EXPERIMENT_ID), + ).toMatchObject({ + retention: { + complete: false, + droppedRecords: 10, + requestedRangeExtendsBeyondRetention: true, + }, + }); + archive.close(); + }); + + it('surfaces persistence failures and never reports a successful import', async () => { + const archive = new ArchiveDatabase({ path: ':memory:' }); + const document = await currentExport(); + archive.close(); + expect(() => importExperimentExport(archive, document)).toThrow( + ExperimentImportError, + ); + }); + + it('rejects credentials and private-reasoning fields from exports and notes', async () => { + const archive = new ArchiveDatabase({ path: ':memory:' }); + const unsafe = { + ...(await currentExport()), + apiKey: 'sk-or-v1-not-allowed-credential', + } as unknown as ExperimentExportDocument; + expect(() => importExperimentExport(archive, unsafe)).toThrow( + /prohibited|Credential/i, + ); + const notePath = temporaryPath('unsafe.md'); + writeFileSync( + notePath, + '# Secret\nBearer abcdefghijklmnopqrstuvwxyz012345', + ); + expect(() => + new ResearchNoteService(archive).import(notePath, { + type: 'transcript', + status: 'proposed', + }), + ).toThrow(/Credential/i); + expect( + archive.database.prepare('SELECT COUNT(*) AS count FROM notes').get(), + ).toEqual({ count: 0 }); + archive.close(); + }); +}); diff --git a/packages/experiment-archive/src/cli.ts b/packages/experiment-archive/src/cli.ts new file mode 100644 index 0000000..b45ee70 --- /dev/null +++ b/packages/experiment-archive/src/cli.ts @@ -0,0 +1,297 @@ +#!/usr/bin/env node +import { + ArchiveDatabase, + ExperimentImportError, + ExperimentQueryService, + NOTE_STATUSES, + NOTE_TYPES, + ResearchNoteService, + importExperimentExport, + type DetailFilters, + type NoteFilters, + type NoteStatus, + type NoteType, +} from './index.js'; + +type OutputFormat = 'table' | 'json' | 'markdown'; + +interface Arguments { + positional: string[]; + flags: Map; +} + +function parseArguments(input: string[]): Arguments { + const positional: string[] = []; + const flags = new Map(); + for (let index = 0; index < input.length; index += 1) { + const value = input[index]!; + if (!value.startsWith('--')) { + positional.push(value); + continue; + } + const key = value.slice(2); + const next = input[index + 1]; + if (!next || next.startsWith('--')) + throw new Error(`Option --${key} requires a value.`); + flags.set(key, [...(flags.get(key) ?? []), next]); + index += 1; + } + return { positional, flags }; +} + +function flag(args: Arguments, key: string): string | undefined { + return args.flags.get(key)?.at(-1); +} + +function integerFlag(args: Arguments, key: string): number | undefined { + const value = flag(args, key); + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isInteger(parsed)) + throw new Error(`--${key} must be an integer.`); + return parsed; +} + +function detailFilters(args: Arguments): DetailFilters { + return { + agent: flag(args, 'agent'), + fromTurn: integerFlag(args, 'from-turn'), + toTurn: integerFlag(args, 'to-turn'), + action: flag(args, 'action'), + outcome: flag(args, 'outcome'), + channel: flag(args, 'channel'), + sender: flag(args, 'sender'), + recipient: flag(args, 'recipient'), + reason: flag(args, 'reason'), + limit: integerFlag(args, 'limit'), + }; +} + +function noteFilters(args: Arguments): NoteFilters { + const type = flag(args, 'type'); + const status = flag(args, 'status'); + if (type && !NOTE_TYPES.includes(type as NoteType)) + throw new Error(`Unsupported note type: ${type}`); + if (status && !NOTE_STATUSES.includes(status as NoteStatus)) + throw new Error(`Unsupported note status: ${status}`); + return { + type: type as NoteType | undefined, + status: status as NoteStatus | undefined, + tag: flag(args, 'tag'), + experiment: flag(args, 'experiment'), + limit: integerFlag(args, 'limit'), + }; +} + +function format(args: Arguments): OutputFormat { + const value = flag(args, 'format') ?? 'table'; + if (value !== 'table' && value !== 'json' && value !== 'markdown') + throw new Error('--format must be table, json, or markdown.'); + return value; +} + +function scalar(value: unknown): string { + if (value === null || value === undefined) return ''; + if (typeof value === 'object') return JSON.stringify(value); + return String(value); +} + +function rowsFor(value: unknown): Array> { + if (Array.isArray(value)) return value as Array>; + if (typeof value === 'object' && value !== null && 'rows' in value) + return (value as { rows: Array> }).rows; + if (typeof value === 'object' && value !== null) + return flatten(value as Record); + return [{ value }]; +} + +function flatten( + value: Record, + prefix = '', +): Array> { + const rows: Array> = []; + for (const [key, entry] of Object.entries(value)) { + const path = prefix ? `${prefix}.${key}` : key; + if (Array.isArray(entry)) { + if (entry.length === 0) rows.push({ field: path, value: '[]' }); + else + entry.forEach((item, index) => { + if (typeof item === 'object' && item !== null) + rows.push( + ...flatten(item as Record, `${path}[${index}]`), + ); + else rows.push({ field: `${path}[${index}]`, value: item }); + }); + } else if (typeof entry === 'object' && entry !== null) + rows.push(...flatten(entry as Record, path)); + else rows.push({ field: path, value: entry }); + } + return rows; +} + +function render(value: unknown, outputFormat: OutputFormat): string { + if (outputFormat === 'json') return JSON.stringify(value, null, 2); + const rows = rowsFor(value); + if (rows.length === 0) + return outputFormat === 'markdown' ? '_No results._' : 'No results.'; + const columns = [...new Set(rows.flatMap((row) => Object.keys(row)))]; + if (outputFormat === 'markdown') { + const escape = (entry: unknown) => + scalar(entry).replaceAll('|', '\\|').replaceAll('\n', ' '); + return [ + `| ${columns.join(' | ')} |`, + `| ${columns.map(() => '---').join(' | ')} |`, + ...rows.map( + (row) => + `| ${columns.map((column) => escape(row[column])).join(' | ')} |`, + ), + ].join('\n'); + } + const widths = columns.map((column) => + Math.min( + 80, + Math.max(column.length, ...rows.map((row) => scalar(row[column]).length)), + ), + ); + const line = (row: Record) => + columns + .map((column, index) => + scalar(row[column]).slice(0, widths[index]!).padEnd(widths[index]!), + ) + .join(' ') + .trimEnd(); + return [ + line(Object.fromEntries(columns.map((column) => [column, column]))), + line( + Object.fromEntries( + columns.map((column, index) => [column, '-'.repeat(widths[index]!)]), + ), + ), + ...rows.map(line), + ].join('\n'); +} + +function requirePositional( + args: Arguments, + index: number, + description: string, +): string { + const value = args.positional[index]; + if (!value) throw new Error(`Missing ${description}.`); + return value; +} + +const HELP = `Usage: + pnpm experiment:db import [--db path] [--format table|json|markdown] + pnpm experiment:db list [--limit n] [--format ...] + pnpm experiment:db summary [--format ...] + pnpm experiment:db compare [--format ...] + pnpm experiment:db turns [--agent id] [--from-turn n] [--to-turn n] [--action action] [--outcome outcome] [--limit n] + pnpm experiment:db communications [--channel channel] [--sender id] [--recipient id] [--reason reason] [--limit n] + pnpm experiment:db alliance-events [--agent id] [--from-turn n] [--to-turn n] [--reason reason] [--limit n] + pnpm experiment:db patient-zero [--agent id] [--from-turn n] [--to-turn n] [--limit n] + pnpm experiment:db failures [--agent id] [--reason code] [--limit n] + pnpm experiment:db notes import --type --status [--tag tag] [--experiment id] [--provenance text] [--supersedes note-id] + pnpm experiment:db notes search [--type type] [--status status] [--tag tag] [--experiment id] [--limit n] + pnpm experiment:db notes list [filters] + +Database path defaults to .agentborne/experiments.sqlite and can also be set with AGENTBORNE_EXPERIMENT_DB. +Detail commands default to 50 rows and enforce a maximum of 500. Arbitrary SQL is not supported.`; + +async function main(): Promise { + const args = parseArguments(process.argv.slice(2)); + const command = args.positional[0]; + if (!command || command === 'help') { + process.stdout.write(`${HELP}\n`); + return; + } + const archive = new ArchiveDatabase({ path: flag(args, 'db') }); + try { + const queries = new ExperimentQueryService(archive); + const notes = new ResearchNoteService(archive); + let result: unknown; + if (command === 'import') { + result = importExperimentExport( + archive, + requirePositional(args, 1, 'export JSON path'), + ); + } else if (command === 'list') { + result = queries.list({ limit: integerFlag(args, 'limit') }); + } else if (command === 'summary') { + result = queries.summary(requirePositional(args, 1, 'experiment ID')); + } else if (command === 'compare') { + result = queries.compare( + requirePositional(args, 1, 'first experiment ID'), + requirePositional(args, 2, 'second experiment ID'), + ); + } else if (command === 'turns') { + result = queries.turns( + requirePositional(args, 1, 'experiment ID'), + detailFilters(args), + ); + } else if (command === 'communications') { + result = queries.communications( + requirePositional(args, 1, 'experiment ID'), + detailFilters(args), + ); + } else if (command === 'alliance-events') { + result = queries.allianceEvents( + requirePositional(args, 1, 'experiment ID'), + detailFilters(args), + ); + } else if (command === 'patient-zero') { + result = queries.patientZero( + requirePositional(args, 1, 'experiment ID'), + detailFilters(args), + ); + } else if (command === 'failures') { + result = queries.failures( + requirePositional(args, 1, 'experiment ID'), + detailFilters(args), + ); + } else if (command === 'notes') { + const operation = requirePositional(args, 1, 'notes operation'); + if (operation === 'import') { + const type = flag(args, 'type'); + const status = flag(args, 'status'); + if (!type || !NOTE_TYPES.includes(type as NoteType)) + throw new Error( + '--type is required and must be a supported note type.', + ); + if (!status || !NOTE_STATUSES.includes(status as NoteStatus)) + throw new Error( + '--status is required and must be a supported note status.', + ); + result = notes.import(requirePositional(args, 2, 'Markdown path'), { + type: type as NoteType, + status: status as NoteStatus, + title: flag(args, 'title'), + tags: args.flags.get('tag'), + provenance: flag(args, 'provenance'), + experiments: args.flags.get('experiment'), + supersedes: flag(args, 'supersedes'), + }); + } else if (operation === 'search') { + result = notes.search( + requirePositional(args, 2, 'search query'), + noteFilters(args), + ); + } else if (operation === 'list') result = notes.list(noteFilters(args)); + else throw new Error(`Unknown notes operation: ${operation}`); + } else throw new Error(`Unknown command: ${command}`); + process.stdout.write(`${render(result, format(args))}\n`); + } finally { + archive.close(); + } +} + +main().catch((error: unknown) => { + const details = + error instanceof ExperimentImportError + ? ` ${JSON.stringify(error.report)}` + : ''; + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}${details}\n`, + ); + process.exitCode = 1; +}); diff --git a/packages/experiment-archive/src/database.ts b/packages/experiment-archive/src/database.ts new file mode 100644 index 0000000..eacb391 --- /dev/null +++ b/packages/experiment-archive/src/database.ts @@ -0,0 +1,112 @@ +import { mkdirSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +import { migrations } from './migrations.js'; + +export const DEFAULT_DATABASE_PATH = '.agentborne/experiments.sqlite'; +export const DATABASE_PATH_ENV = 'AGENTBORNE_EXPERIMENT_DB'; + +export function archiveInvocationRoot(): string { + return process.env.INIT_CWD?.trim() || process.cwd(); +} + +export interface OpenArchiveOptions { + path?: string; + clock?: () => Date; +} + +export class ArchivePersistenceError extends Error { + constructor( + message: string, + readonly cause?: unknown, + ) { + super(message); + this.name = 'ArchivePersistenceError'; + } +} + +export class ArchiveDatabase { + readonly database: DatabaseSync; + readonly path: string; + readonly clock: () => Date; + + constructor(options: OpenArchiveOptions = {}) { + const configured = + options.path ?? process.env[DATABASE_PATH_ENV] ?? DEFAULT_DATABASE_PATH; + this.path = + configured === ':memory:' + ? configured + : resolve(archiveInvocationRoot(), configured); + this.clock = options.clock ?? (() => new Date()); + try { + if (this.path !== ':memory:') + mkdirSync(dirname(this.path), { recursive: true }); + this.database = new DatabaseSync(this.path); + this.database.exec( + 'PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;', + ); + if (this.path !== ':memory:') + this.database.exec('PRAGMA journal_mode = WAL;'); + this.migrate(); + } catch (error) { + throw new ArchivePersistenceError( + `Could not open experiment archive at ${this.path}.`, + error, + ); + } + } + + close(): void { + this.database.close(); + } + + transaction(operation: () => T): T { + try { + this.database.exec('BEGIN IMMEDIATE'); + const result = operation(); + this.database.exec('COMMIT'); + return result; + } catch (error) { + try { + this.database.exec('ROLLBACK'); + } catch { + // Preserve the original persistence failure. + } + if (error instanceof ArchivePersistenceError) throw error; + throw new ArchivePersistenceError( + 'Experiment archive transaction failed and was rolled back.', + error, + ); + } + } + + private migrate(): void { + this.database.exec(` + CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + description TEXT NOT NULL, + applied_at TEXT NOT NULL + ) STRICT; + `); + const rows = this.database + .prepare('SELECT version FROM schema_migrations ORDER BY version') + .all() as Array<{ version: number }>; + const applied = new Set(rows.map(({ version }) => version)); + for (const migration of migrations) { + if (applied.has(migration.version)) continue; + this.transaction(() => { + this.database.exec(migration.sql); + this.database + .prepare( + 'INSERT INTO schema_migrations(version, description, applied_at) VALUES (?, ?, ?)', + ) + .run( + migration.version, + migration.description, + this.clock().toISOString(), + ); + }); + } + } +} diff --git a/packages/experiment-archive/src/importer.ts b/packages/experiment-archive/src/importer.ts new file mode 100644 index 0000000..a786c6c --- /dev/null +++ b/packages/experiment-archive/src/importer.ts @@ -0,0 +1,795 @@ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { + AGENT_DECISION_CONTRACT_VERSION, + experimentExportDocumentSchema, + exportedCommunicationSchema, + type ExperimentExportDocument, +} from '@agentborne/shared'; + +import { + ArchiveDatabase, + ArchivePersistenceError, + archiveInvocationRoot, +} from './database.js'; + +export interface ImportReport { + experimentId: string; + sourceSha256: string; + inserted: number; + existing: number; + skipped: number; + rejected: number; +} + +export class ExperimentImportError extends Error { + readonly report: Omit; + + constructor( + message: string, + readonly cause?: unknown, + ) { + super(message); + this.name = 'ExperimentImportError'; + this.report = { inserted: 0, existing: 0, skipped: 0, rejected: 1 }; + } +} + +const prohibitedKey = + /^(authorization|cookie|api[-_]?key|secret|password|raw[-_]?(reasoning|response|request)|chain[-_]?of[-_]?thought|private[-_]?reasoning)$/i; +const credentialValue = /\b(?:sk-or-v1|sk-proj|Bearer\s+[A-Za-z0-9._~-]{16,})/i; + +export function assertSafeArchiveValue(value: unknown, path = '$'): void { + if (typeof value === 'string' && credentialValue.test(value)) + throw new ExperimentImportError( + `Credential-like content is prohibited at ${path}.`, + ); + if (Array.isArray(value)) { + value.forEach((entry, index) => + assertSafeArchiveValue(entry, `${path}[${index}]`), + ); + return; + } + if (typeof value !== 'object' || value === null) return; + for (const [key, entry] of Object.entries(value)) { + if (prohibitedKey.test(key)) + throw new ExperimentImportError( + `Prohibited private or credential field at ${path}.${key}.`, + ); + assertSafeArchiveValue(entry, `${path}.${key}`); + } +} + +function json(value: unknown): string | null { + return value === undefined ? null : JSON.stringify(value); +} + +function stableId(...parts: readonly unknown[]): string { + return createHash('sha256') + .update(parts.map((part) => String(part)).join('\u001f')) + .digest('hex'); +} + +function readExport(input: string | ExperimentExportDocument): { + document: ExperimentExportDocument; + sourceText: string; + sourcePath: string; +} { + try { + if (typeof input !== 'string') { + assertSafeArchiveValue(input); + const document = experimentExportDocumentSchema.parse(input); + return { + document, + sourceText: JSON.stringify(input), + sourcePath: '', + }; + } + const sourcePath = resolve(archiveInvocationRoot(), input); + const sourceText = readFileSync(sourcePath, 'utf8'); + const parsed: unknown = JSON.parse(sourceText); + assertSafeArchiveValue(parsed); + const document = experimentExportDocumentSchema.parse(parsed); + return { document, sourceText, sourcePath }; + } catch (error) { + if (error instanceof ExperimentImportError) throw error; + throw new ExperimentImportError( + `Export validation failed${typeof input === 'string' ? ` for ${input}` : ''}; nothing was imported.`, + error, + ); + } +} + +function runInsert( + statement: ReturnType, + values: readonly (string | number | null)[], + report: ImportReport, +): void { + const result = statement.run(...values); + if (result.changes > 0) report.inserted += 1; + else report.existing += 1; +} + +function sourceMetricInconsistencies(document: ExperimentExportDocument) { + const metrics = document.metrics; + if (!metrics) return []; + const perAgent = metrics.byAgent.reduce( + (sum, entry) => sum + entry.metrics.directionChangesAfterCommunication, + 0, + ); + return metrics.aggregate.directionChangesAfterCommunication === perAgent + ? [] + : [ + { + metric: 'directionChangesAfterCommunication', + aggregate: metrics.aggregate.directionChangesAfterCommunication, + perAgentSum: perAgent, + canonical: + 'sum of per-agent chronological direction changes after an observed inbound direct or alliance communication since that agent previous move', + }, + ]; +} + +export function importExperimentExport( + archive: ArchiveDatabase, + input: string | ExperimentExportDocument, +): ImportReport { + // Parse the entire document and inspect it for prohibited data before BEGIN. + const { document, sourceText, sourcePath } = readExport(input); + const sourceSha256 = createHash('sha256').update(sourceText).digest('hex'); + const experimentId = document.experiment.id; + const report: ImportReport = { + experimentId, + sourceSha256, + inserted: 0, + existing: 0, + skipped: 0, + rejected: 0, + }; + + try { + return archive.transaction(() => { + const db = archive.database; + const alreadyImported = db + .prepare('SELECT 1 AS found FROM source_exports WHERE sha256 = ?') + .get(sourceSha256); + if (alreadyImported) { + report.existing += 1; + return report; + } + + const scenario = document.experiment.scenario; + const experimentInsert = db.prepare(` + INSERT OR IGNORE INTO experiments( + id, schema_version, started_at, provider_mode, imported_at, + scenario_json, model_configuration_json, behavior_configuration_json, + objective_version, decision_contract_version, observation_contract_version, + retention_limit, total_completed_turns, retained_turns, + first_retained_turn, last_retained_turn, dropped_records, + retention_complete, requested_range_extends_beyond_retention, + source_metrics_json, source_territory_json, source_alliances_json, + metric_inconsistencies_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + runInsert( + experimentInsert, + [ + experimentId, + document.schemaVersion, + document.experiment.startedAt, + document.experiment.providerMode, + archive.clock().toISOString(), + json(scenario), + json(document.experiment.modelConfiguration), + json(document.experiment.behaviorConfiguration), + scenario?.objectiveVersion ?? null, + AGENT_DECISION_CONTRACT_VERSION, + `experiment-export-schema-v${document.schemaVersion}`, + document.retention.limit, + document.retention.totalCompletedTurns, + document.retention.retainedTurns, + document.retention.firstRetainedTurn ?? null, + document.retention.lastRetainedTurn ?? null, + document.retention.droppedRecords, + Number(document.retention.complete), + Number(document.retention.requestedRangeExtendsBeyondRetention), + json(document.metrics), + json(document.currentTerritory), + json(document.currentAlliances), + json(sourceMetricInconsistencies(document))!, + ], + report, + ); + + // A later, broader export may fill immutable source metadata omitted by an earlier filtered export. + db.prepare( + ` + UPDATE experiments SET + scenario_json = COALESCE(scenario_json, ?), + model_configuration_json = COALESCE(model_configuration_json, ?), + behavior_configuration_json = COALESCE(behavior_configuration_json, ?), + objective_version = COALESCE(objective_version, ?), + source_metrics_json = COALESCE(?, source_metrics_json), + source_territory_json = COALESCE(?, source_territory_json), + source_alliances_json = COALESCE(?, source_alliances_json), + retention_limit = MAX(retention_limit, ?), + total_completed_turns = MAX(total_completed_turns, ?), + retained_turns = MAX(retained_turns, ?), + dropped_records = MAX(dropped_records, ?), + retention_complete = retention_complete AND ?, + requested_range_extends_beyond_retention = requested_range_extends_beyond_retention OR ? + WHERE id = ? + `, + ).run( + json(scenario), + json(document.experiment.modelConfiguration), + json(document.experiment.behaviorConfiguration), + scenario?.objectiveVersion ?? null, + json(document.metrics), + json(document.currentTerritory), + json(document.currentAlliances), + document.retention.limit, + document.retention.totalCompletedTurns, + document.retention.retainedTurns, + document.retention.droppedRecords, + Number(document.retention.complete), + Number(document.retention.requestedRangeExtendsBeyondRetention), + experimentId, + ); + + const sourceInsert = db.prepare(` + INSERT INTO source_exports( + sha256, experiment_id, source_path, generated_at, imported_at, + filters_json, selection_json, source_metrics_json, retention_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + runInsert( + sourceInsert, + [ + sourceSha256, + experimentId, + sourcePath, + document.generatedAt, + archive.clock().toISOString(), + json(document.filters)!, + json(document.selection)!, + json(document.metrics), + json(document.retention)!, + ], + report, + ); + + importAgents(archive, document, report); + importMap(archive, document, report); + importTurns(archive, document, report); + importCommunications(archive, document, report); + importAllianceEvents(archive, document, report); + importWorldEvents(archive, document, report); + importConfigurationEvents(archive, document, report); + return report; + }); + } catch (error) { + if (error instanceof ExperimentImportError) throw error; + if (error instanceof ArchivePersistenceError) + throw new ExperimentImportError( + `Archive persistence failed for experiment ${experimentId}; the import was rolled back.`, + error, + ); + throw error; + } +} + +function importAgents( + archive: ArchiveDatabase, + document: ExperimentExportDocument, + report: ImportReport, +): void { + const statement = archive.database.prepare(` + INSERT OR IGNORE INTO agents( + experiment_id, agent_id, name, color, model_id, reasoning_profile, + personality_id, strategy_id, personality, is_patient_zero, + initial_agent_json, current_agent_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const behavior = new Map( + document.experiment.behaviorConfiguration?.assignments.map((entry) => [ + entry.agentId, + entry, + ]), + ); + const models = document.experiment.modelConfiguration; + const initial = new Map( + document.experiment.initialAgents?.map((agent) => [agent.id, agent]), + ); + const current = new Map( + document.currentWorld?.agents.map((agent) => [agent.id, agent]), + ); + const profiles = new Map< + ExperimentExportDocument['agents'][number]['id'], + { + id: ExperimentExportDocument['agents'][number]['id']; + name: string; + color: string; + personality?: string; + currentCell?: string; + } + >(); + document.experiment.scenario?.roster.forEach((agent, index) => + profiles.set(agent.id, { + ...agent, + currentCell: document.experiment.scenario?.startingCells[index], + }), + ); + for (const agent of document.agents) + profiles.set(agent.id, { ...(profiles.get(agent.id) ?? {}), ...agent }); + for (const agent of profiles.values()) { + const assignment = behavior.get(agent.id); + const override = models?.overrides.find( + ({ agentId }) => agentId === agent.id, + ); + runInsert( + statement, + [ + document.experiment.id, + agent.id, + agent.name, + agent.color, + override?.modelId ?? models?.globalModelId ?? null, + override?.reasoningProfile ?? models?.globalReasoningProfile ?? null, + assignment?.personalityId ?? null, + assignment?.strategyId ?? null, + agent.personality ?? initial.get(agent.id)?.personality ?? null, + Number(document.experiment.scenario?.patientZeroAgentId === agent.id), + json(initial.get(agent.id)), + json(current.get(agent.id)), + ], + report, + ); + } +} + +function importMap( + archive: ArchiveDatabase, + document: ExperimentExportDocument, + report: ImportReport, +): void { + const initial = new Map( + document.initialWorld?.hexes.map((hex) => [hex.cell, hex]), + ); + const current = new Map( + document.currentWorld?.hexes.map((hex) => [hex.cell, hex]), + ); + const ids = [...new Set([...initial.keys(), ...current.keys()])]; + if (ids.length === 0) { + report.skipped += 1; + return; + } + const statement = archive.database.prepare(` + INSERT OR IGNORE INTO map_cells( + experiment_id, cell_id, ordinal, initial_state, initial_controller_agent_id, + current_state, current_controller_agent_id, initial_json, current_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + ids.forEach((cellId, ordinal) => { + const before = initial.get(cellId); + const after = current.get(cellId); + runInsert( + statement, + [ + document.experiment.id, + cellId, + ordinal, + before?.state ?? null, + before?.state === 'infected' ? before.controllerAgentId : null, + after?.state ?? null, + after?.state === 'infected' ? after.controllerAgentId : null, + json(before), + json(after), + ], + report, + ); + }); +} + +function importTurns( + archive: ArchiveDatabase, + document: ExperimentExportDocument, + report: ImportReport, +): void { + const movement = movementAttribution(document); + const turnStatement = archive.database.prepare(` + INSERT OR IGNORE INTO turns( + id, experiment_id, turn_number, agent_id, started_at, completed_at, + outcome, action, action_accepted, action_reason, summary, + world_action_summary, communication_summary, diplomacy_summary, + position_before, position_after, move_direction, + inbound_communication_since_previous_move, model_id, reasoning_profile, + personality_id, strategy_id, latency_ms, prompt_tokens, + completion_tokens, total_tokens, reasoning_tokens, cached_read_tokens, + cache_write_tokens, cost_credits, observation_size_bytes, + observation_json, world_action_json, world_action_result_json, + communication_result_json, diplomacy_result_json, failure_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const attemptStatement = archive.database.prepare(` + INSERT OR IGNORE INTO model_attempts( + id, turn_id, experiment_id, turn_number, agent_id, attempt_number, + kind, started_at, completed_at, model_id, reasoning_profile, accepted, + failure_code, failure_message, validation_codes_json, latency_ms, + prompt_tokens, completion_tokens, total_tokens, reasoning_tokens, + cached_read_tokens, cache_write_tokens, cost_credits + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const diplomacyStatement = archive.database.prepare(` + INSERT OR IGNORE INTO diplomacy_attempts( + id, experiment_id, turn_number, agent_id, type, recipient_agent_id, + proposal_id, accepted, rejection_reason, rejection_details, source_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + + for (const turn of document.turns) { + const turnId = `${document.experiment.id}:turn:${turn.turnNumber}`; + const result = turn.worldActionResult; + const positionBefore = turn.observation?.currentCell?.cell ?? null; + const positionAfter = + result?.accepted && result.event.type === 'agent-moved' + ? result.event.toCell + : positionBefore; + const provider = turn.provider; + const lastAttempt = turn.modelAttempts.at(-1); + const movementEntry = movement.get(turn.turnNumber); + runInsert( + turnStatement, + [ + turnId, + document.experiment.id, + turn.turnNumber, + turn.agentId, + turn.startedAt, + turn.completedAt, + turn.outcome, + turn.worldAction?.type ?? null, + result ? Number(result.accepted) : null, + result && !result.accepted ? result.reason : null, + turn.summary ?? null, + turn.worldActionSummary ?? null, + turn.communicationSummary ?? null, + turn.diplomacySummary ?? null, + positionBefore, + positionAfter, + movementEntry?.direction ?? null, + movementEntry === undefined + ? null + : Number(movementEntry.inboundSincePreviousMove), + provider?.model ?? lastAttempt?.modelId ?? null, + lastAttempt?.reasoningProfile ?? null, + turn.behavior?.personalityId ?? null, + turn.behavior?.strategyId ?? null, + provider?.latencyMs ?? null, + provider?.promptTokens ?? null, + provider?.completionTokens ?? null, + provider?.totalTokens ?? null, + provider?.reasoningTokens ?? null, + provider?.cachedReadTokens ?? null, + provider?.cacheWriteTokens ?? null, + provider?.costCredits ?? null, + turn.observation + ? Buffer.byteLength(JSON.stringify(turn.observation)) + : null, + json(turn.observation), + json(turn.worldAction), + json(turn.worldActionResult), + json(turn.communicationResult), + json(turn.diplomacyResult), + json(turn.failure), + ], + report, + ); + if (!turn.observation) report.skipped += 1; + + for (const attempt of turn.modelAttempts) { + const attemptProvider = attempt.provider; + runInsert( + attemptStatement, + [ + `${turnId}:attempt:${attempt.attemptNumber}`, + turnId, + document.experiment.id, + turn.turnNumber, + turn.agentId, + attempt.attemptNumber, + attempt.kind, + attempt.startedAt, + attempt.completedAt, + attempt.modelId, + attempt.reasoningProfile, + Number(!attempt.failure), + attempt.failure?.code ?? null, + attempt.failure?.message ?? null, + json(attempt.failure?.validationCodes), + attemptProvider?.latencyMs ?? attempt.failure?.latencyMs ?? null, + attemptProvider?.promptTokens ?? null, + attemptProvider?.completionTokens ?? null, + attemptProvider?.totalTokens ?? null, + attemptProvider?.reasoningTokens ?? null, + attemptProvider?.cachedReadTokens ?? null, + attemptProvider?.cacheWriteTokens ?? null, + attemptProvider?.costCredits ?? null, + ], + report, + ); + } + + const diplomacy = turn.diplomacyResult; + if (diplomacy?.requested) { + const value = diplomacy.accepted ? diplomacy.intent : diplomacy.attempt; + runInsert( + diplomacyStatement, + [ + `${turnId}:diplomacy`, + document.experiment.id, + turn.turnNumber, + turn.agentId, + value.type, + 'recipientId' in value ? (value.recipientId ?? null) : null, + 'proposalId' in value ? (value.proposalId ?? null) : null, + Number(diplomacy.accepted), + diplomacy.accepted ? null : diplomacy.reason, + diplomacy.accepted ? null : diplomacy.details, + json(diplomacy)!, + ], + report, + ); + } + } +} + +function movementAttribution(document: ExperimentExportDocument) { + const result = new Map< + number, + { direction: string; inboundSincePreviousMove: boolean } + >(); + for (const actingAgentId of [ + ...new Set(document.turns.map(({ agentId }) => agentId)), + ]) { + let previousMoveAt: string | undefined; + for (const turn of document.turns + .filter(({ agentId }) => agentId === actingAgentId) + .toSorted((left, right) => left.turnNumber - right.turnNumber)) { + if ( + turn.outcome !== 'accepted' || + turn.worldActionResult?.accepted !== true || + turn.worldActionResult.event.type !== 'agent-moved' + ) + continue; + const destination = turn.worldActionResult.event.toCell; + const option = turn.observation?.actionAvailability?.moveOptions.find( + ({ targetCell }) => targetCell === destination, + ); + if (!option) continue; + const messages = [ + ...(turn.observation?.recentDirectMessages ?? []), + ...(turn.observation?.recentAllianceMessages ?? []), + ]; + const inboundSincePreviousMove = + previousMoveAt !== undefined && + messages.some((message) => { + const inbound = + ('direction' in message && message.direction === 'inbound') || + ('senderId' in message && message.senderId !== turn.agentId); + return ( + inbound && + message.occurredAt > previousMoveAt! && + message.occurredAt <= turn.completedAt + ); + }); + result.set(turn.turnNumber, { + direction: option.direction, + inboundSincePreviousMove, + }); + previousMoveAt = turn.completedAt; + } + } + return result; +} + +function derivedCommunications(document: ExperimentExportDocument) { + const byId = new Map( + (document.communications ?? []).map((entry) => [entry.id, entry]), + ); + for (const turn of document.turns) { + const result = turn.communicationResult; + if (!result?.requested) continue; + const source = result.accepted ? result.event : result.attempt; + if (byId.has(source.id)) continue; + byId.set( + source.id, + exportedCommunicationSchema.parse({ + id: source.id, + agentId: source.agentId, + channel: source.channel, + ...('recipientId' in source ? { recipientId: source.recipientId } : {}), + ...('recipientIds' in source + ? { recipientIds: source.recipientIds } + : {}), + message: source.message, + ...('distance' in source ? { distance: source.distance } : {}), + occurredAt: source.occurredAt, + originatingTurn: turn.turnNumber, + status: result.accepted ? 'accepted' : 'rejected', + ...(!result.accepted + ? { rejectionReason: result.reason, rejectionDetails: result.details } + : {}), + }), + ); + } + return [...byId.values()]; +} + +function importCommunications( + archive: ArchiveDatabase, + document: ExperimentExportDocument, + report: ImportReport, +): void { + const statement = archive.database.prepare(` + INSERT OR IGNORE INTO communications( + id, experiment_id, turn_number, sender_agent_id, channel, + recipient_agent_id, message, distance, occurred_at, status, + rejection_reason, rejection_details, source_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const recipientStatement = archive.database.prepare( + 'INSERT OR IGNORE INTO communication_recipients(communication_id, recipient_agent_id) VALUES (?, ?)', + ); + for (const communication of derivedCommunications(document)) { + runInsert( + statement, + [ + communication.id, + document.experiment.id, + communication.originatingTurn, + communication.agentId, + communication.channel, + communication.recipientId ?? null, + communication.message, + communication.distance ?? null, + communication.occurredAt, + communication.status, + communication.rejectionReason ?? null, + communication.rejectionDetails ?? null, + json(communication)!, + ], + report, + ); + for (const recipientId of communication.recipientIds ?? []) + runInsert(recipientStatement, [communication.id, recipientId], report); + } +} + +function importAllianceEvents( + archive: ArchiveDatabase, + document: ExperimentExportDocument, + report: ImportReport, +): void { + const statement = archive.database.prepare(` + INSERT OR IGNORE INTO alliance_events( + id, experiment_id, turn_number, occurred_at, agent_id, type, reason, source_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `); + const events = new Map( + (document.allianceEvents ?? []).map((event) => [event.id, event]), + ); + for (const turn of document.turns) + if (turn.diplomacyResult?.requested && turn.diplomacyResult.accepted) + for (const event of turn.diplomacyResult.events) + events.set(event.id, event); + for (const event of events.values()) + runInsert( + statement, + [ + event.id, + document.experiment.id, + event.turnNumber, + event.occurredAt, + event.agentId, + event.type, + 'reason' in event ? event.reason : null, + json(event)!, + ], + report, + ); +} + +function importWorldEvents( + archive: ArchiveDatabase, + document: ExperimentExportDocument, + report: ImportReport, +): void { + const statement = archive.database.prepare(` + INSERT OR IGNORE INTO world_events( + id, experiment_id, turn_number, occurred_at, agent_id, type, + cell_id, previous_controller_agent_id, source_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const events = new Map< + string, + { + event: NonNullable[number]; + turn: number; + } + >(); + for (const event of document.worldEvents ?? []) { + const originatingTurn = document.turns.find( + (candidate) => + candidate.worldActionResult?.accepted && + candidate.worldActionResult.event.id === event.id, + )?.turnNumber; + if (originatingTurn) events.set(event.id, { event, turn: originatingTurn }); + } + for (const event of document.controlChanges ?? []) + events.set(event.id, { event, turn: event.originatingTurn }); + for (const turn of document.turns) { + const result = turn.worldActionResult; + if (result?.accepted) + events.set(result.event.id, { + event: result.event, + turn: turn.turnNumber, + }); + } + for (const { event, turn } of events.values()) + runInsert( + statement, + [ + event.id, + document.experiment.id, + turn, + event.occurredAt, + event.agentId, + event.type, + 'cell' in event ? event.cell : null, + 'previousControllerAgentId' in event + ? event.previousControllerAgentId + : null, + json(event)!, + ], + report, + ); +} + +function importConfigurationEvents( + archive: ArchiveDatabase, + document: ExperimentExportDocument, + report: ImportReport, +): void { + const statement = archive.database.prepare(` + INSERT OR IGNORE INTO configuration_events( + id, experiment_id, occurred_at, type, agent_id, effective_turn, source_json + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `); + for (const event of document.configurationEvents ?? []) { + const type = + 'type' in event ? event.type : `personality-${event.operation}`; + const id = stableId( + document.experiment.id, + type, + event.timestamp, + 'agentId' in event ? event.agentId : '', + json(event), + ); + runInsert( + statement, + [ + id, + document.experiment.id, + event.timestamp, + type, + 'agentId' in event ? (event.agentId ?? null) : null, + 'effectiveTurn' in event ? event.effectiveTurn : null, + json(event)!, + ], + report, + ); + } +} diff --git a/packages/experiment-archive/src/index.ts b/packages/experiment-archive/src/index.ts new file mode 100644 index 0000000..6fdb9c3 --- /dev/null +++ b/packages/experiment-archive/src/index.ts @@ -0,0 +1,4 @@ +export * from './database.js'; +export * from './importer.js'; +export * from './notes.js'; +export * from './query-service.js'; diff --git a/packages/experiment-archive/src/migrations.ts b/packages/experiment-archive/src/migrations.ts new file mode 100644 index 0000000..ec271ea --- /dev/null +++ b/packages/experiment-archive/src/migrations.ts @@ -0,0 +1,268 @@ +export interface Migration { + version: number; + description: string; + sql: string; +} + +export const migrations: readonly Migration[] = [ + { + version: 1, + description: 'initial durable experiment archive', + sql: ` + CREATE TABLE experiments ( + id TEXT PRIMARY KEY, + schema_version INTEGER NOT NULL, + started_at TEXT NOT NULL, + provider_mode TEXT NOT NULL, + imported_at TEXT NOT NULL, + scenario_json TEXT, + model_configuration_json TEXT, + behavior_configuration_json TEXT, + objective_version TEXT, + decision_contract_version TEXT NOT NULL, + observation_contract_version TEXT NOT NULL, + retention_limit INTEGER NOT NULL, + total_completed_turns INTEGER NOT NULL, + retained_turns INTEGER NOT NULL, + first_retained_turn INTEGER, + last_retained_turn INTEGER, + dropped_records INTEGER NOT NULL, + retention_complete INTEGER NOT NULL CHECK (retention_complete IN (0, 1)), + requested_range_extends_beyond_retention INTEGER NOT NULL CHECK (requested_range_extends_beyond_retention IN (0, 1)), + source_metrics_json TEXT, + source_territory_json TEXT, + source_alliances_json TEXT, + metric_inconsistencies_json TEXT NOT NULL DEFAULT '[]' + ) STRICT; + + CREATE TABLE source_exports ( + sha256 TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + source_path TEXT NOT NULL, + generated_at TEXT NOT NULL, + imported_at TEXT NOT NULL, + filters_json TEXT NOT NULL, + selection_json TEXT NOT NULL, + source_metrics_json TEXT, + retention_json TEXT NOT NULL + ) STRICT; + CREATE INDEX source_exports_experiment_idx ON source_exports(experiment_id, imported_at); + + CREATE TABLE agents ( + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + agent_id TEXT NOT NULL, + name TEXT NOT NULL, + color TEXT NOT NULL, + model_id TEXT, + reasoning_profile TEXT, + personality_id TEXT, + strategy_id TEXT, + personality TEXT, + is_patient_zero INTEGER NOT NULL DEFAULT 0 CHECK (is_patient_zero IN (0, 1)), + initial_agent_json TEXT, + current_agent_json TEXT, + PRIMARY KEY (experiment_id, agent_id) + ) STRICT; + CREATE INDEX agents_patient_zero_idx ON agents(experiment_id, is_patient_zero); + + CREATE TABLE map_cells ( + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + cell_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, + initial_state TEXT, + initial_controller_agent_id TEXT, + current_state TEXT, + current_controller_agent_id TEXT, + initial_json TEXT, + current_json TEXT, + PRIMARY KEY (experiment_id, cell_id) + ) STRICT; + CREATE INDEX map_cells_current_controller_idx ON map_cells(experiment_id, current_controller_agent_id); + + CREATE TABLE turns ( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + turn_number INTEGER NOT NULL, + agent_id TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT NOT NULL, + outcome TEXT NOT NULL, + action TEXT, + action_accepted INTEGER, + action_reason TEXT, + summary TEXT, + world_action_summary TEXT, + communication_summary TEXT, + diplomacy_summary TEXT, + position_before TEXT, + position_after TEXT, + move_direction TEXT, + inbound_communication_since_previous_move INTEGER CHECK (inbound_communication_since_previous_move IN (0, 1)), + model_id TEXT, + reasoning_profile TEXT, + personality_id TEXT, + strategy_id TEXT, + latency_ms INTEGER, + prompt_tokens INTEGER, + completion_tokens INTEGER, + total_tokens INTEGER, + reasoning_tokens INTEGER, + cached_read_tokens INTEGER, + cache_write_tokens INTEGER, + cost_credits REAL, + observation_size_bytes INTEGER, + observation_json TEXT, + world_action_json TEXT, + world_action_result_json TEXT, + communication_result_json TEXT, + diplomacy_result_json TEXT, + failure_json TEXT, + UNIQUE (experiment_id, turn_number) + ) STRICT; + CREATE INDEX turns_experiment_agent_idx ON turns(experiment_id, agent_id, turn_number); + CREATE INDEX turns_experiment_outcome_idx ON turns(experiment_id, outcome, turn_number); + CREATE INDEX turns_experiment_action_idx ON turns(experiment_id, action, turn_number); + + CREATE TABLE model_attempts ( + id TEXT PRIMARY KEY, + turn_id TEXT NOT NULL REFERENCES turns(id) ON DELETE CASCADE, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + turn_number INTEGER NOT NULL, + agent_id TEXT NOT NULL, + attempt_number INTEGER NOT NULL, + kind TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT NOT NULL, + model_id TEXT NOT NULL, + reasoning_profile TEXT NOT NULL, + accepted INTEGER NOT NULL CHECK (accepted IN (0, 1)), + failure_code TEXT, + failure_message TEXT, + validation_codes_json TEXT, + latency_ms INTEGER, + prompt_tokens INTEGER, + completion_tokens INTEGER, + total_tokens INTEGER, + reasoning_tokens INTEGER, + cached_read_tokens INTEGER, + cache_write_tokens INTEGER, + cost_credits REAL, + UNIQUE (turn_id, attempt_number) + ) STRICT; + CREATE INDEX model_attempts_experiment_failure_idx ON model_attempts(experiment_id, failure_code, turn_number); + + CREATE TABLE communications ( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + turn_number INTEGER NOT NULL, + sender_agent_id TEXT NOT NULL, + channel TEXT NOT NULL, + recipient_agent_id TEXT, + message TEXT NOT NULL, + distance REAL, + occurred_at TEXT NOT NULL, + status TEXT NOT NULL, + rejection_reason TEXT, + rejection_details TEXT, + source_json TEXT NOT NULL + ) STRICT; + CREATE INDEX communications_experiment_channel_idx ON communications(experiment_id, channel, turn_number, id); + CREATE INDEX communications_sender_idx ON communications(experiment_id, sender_agent_id, turn_number); + CREATE INDEX communications_recipient_idx ON communications(experiment_id, recipient_agent_id, turn_number); + + CREATE TABLE communication_recipients ( + communication_id TEXT NOT NULL REFERENCES communications(id) ON DELETE CASCADE, + recipient_agent_id TEXT NOT NULL, + PRIMARY KEY (communication_id, recipient_agent_id) + ) STRICT; + + CREATE TABLE diplomacy_attempts ( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + turn_number INTEGER NOT NULL, + agent_id TEXT NOT NULL, + type TEXT NOT NULL, + recipient_agent_id TEXT, + proposal_id TEXT, + accepted INTEGER NOT NULL CHECK (accepted IN (0, 1)), + rejection_reason TEXT, + rejection_details TEXT, + source_json TEXT NOT NULL + ) STRICT; + CREATE INDEX diplomacy_attempts_experiment_idx ON diplomacy_attempts(experiment_id, turn_number, type, accepted); + CREATE INDEX diplomacy_attempts_rejection_idx ON diplomacy_attempts(experiment_id, rejection_reason, turn_number); + + CREATE TABLE alliance_events ( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + turn_number INTEGER NOT NULL, + occurred_at TEXT NOT NULL, + agent_id TEXT NOT NULL, + type TEXT NOT NULL, + reason TEXT, + source_json TEXT NOT NULL + ) STRICT; + CREATE INDEX alliance_events_experiment_type_idx ON alliance_events(experiment_id, type, turn_number, id); + CREATE INDEX alliance_events_reason_idx ON alliance_events(experiment_id, reason, turn_number); + + CREATE TABLE world_events ( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + turn_number INTEGER NOT NULL, + occurred_at TEXT NOT NULL, + agent_id TEXT NOT NULL, + type TEXT NOT NULL, + cell_id TEXT, + previous_controller_agent_id TEXT, + source_json TEXT NOT NULL + ) STRICT; + CREATE INDEX world_events_experiment_type_idx ON world_events(experiment_id, type, turn_number, id); + + CREATE TABLE configuration_events ( + id TEXT PRIMARY KEY, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + occurred_at TEXT NOT NULL, + type TEXT NOT NULL, + agent_id TEXT, + effective_turn INTEGER, + source_json TEXT NOT NULL + ) STRICT; + CREATE INDEX configuration_events_experiment_idx ON configuration_events(experiment_id, occurred_at, id); + + CREATE TABLE notes ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + body TEXT NOT NULL, + type TEXT NOT NULL CHECK (type IN ('transcript', 'observation', 'hypothesis', 'decision', 'implementation-note', 'experiment-finding')), + status TEXT NOT NULL CHECK (status IN ('proposed', 'accepted', 'rejected', 'deferred', 'superseded')), + provenance TEXT NOT NULL, + superseded_note_id TEXT REFERENCES notes(id), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) STRICT; + CREATE INDEX notes_status_type_idx ON notes(status, type, updated_at DESC, id); + + CREATE TABLE note_tags ( + note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE, + tag TEXT NOT NULL, + PRIMARY KEY (note_id, tag) + ) STRICT; + CREATE INDEX note_tags_tag_idx ON note_tags(tag, note_id); + + CREATE TABLE note_experiments ( + note_id TEXT NOT NULL REFERENCES notes(id) ON DELETE CASCADE, + experiment_id TEXT NOT NULL REFERENCES experiments(id) ON DELETE CASCADE, + PRIMARY KEY (note_id, experiment_id) + ) STRICT; + CREATE INDEX note_experiments_experiment_idx ON note_experiments(experiment_id, note_id); + + CREATE VIRTUAL TABLE notes_fts USING fts5( + note_id UNINDEXED, + title, + body, + tags, + tokenize = 'unicode61' + ); + `, + }, +] as const; diff --git a/packages/experiment-archive/src/notes.ts b/packages/experiment-archive/src/notes.ts new file mode 100644 index 0000000..e15aa0d --- /dev/null +++ b/packages/experiment-archive/src/notes.ts @@ -0,0 +1,213 @@ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { basename, resolve } from 'node:path'; + +import { ArchiveDatabase, archiveInvocationRoot } from './database.js'; +import { assertSafeArchiveValue } from './importer.js'; +import { + DEFAULT_DETAIL_LIMIT, + MAX_DETAIL_LIMIT, + type QueryPage, +} from './query-service.js'; + +export const NOTE_TYPES = [ + 'transcript', + 'observation', + 'hypothesis', + 'decision', + 'implementation-note', + 'experiment-finding', +] as const; +export const NOTE_STATUSES = [ + 'proposed', + 'accepted', + 'rejected', + 'deferred', + 'superseded', +] as const; +export type NoteType = (typeof NOTE_TYPES)[number]; +export type NoteStatus = (typeof NOTE_STATUSES)[number]; + +export interface NoteImportOptions { + type: NoteType; + status: NoteStatus; + title?: string; + tags?: string[]; + provenance?: string; + experiments?: string[]; + supersedes?: string; +} + +export interface NoteFilters { + type?: NoteType; + status?: NoteStatus; + tag?: string; + experiment?: string; + limit?: number; +} + +function normalizeList(values: readonly string[] = []): string[] { + return [ + ...new Set( + values.map((value) => value.trim().toLowerCase()).filter(Boolean), + ), + ].sort(); +} + +function limit(value?: number): number { + if (value === undefined) return DEFAULT_DETAIL_LIMIT; + if (!Number.isInteger(value) || value < 1) + throw new Error('Limit must be a positive integer.'); + return Math.min(value, MAX_DETAIL_LIMIT); +} + +function extractTitle(body: string, fallback: string): string { + const heading = body.match(/^#\s+(.+)$/m)?.[1]?.trim(); + return heading || fallback.replace(/\.md$/i, ''); +} + +export class ResearchNoteService { + constructor(readonly archive: ArchiveDatabase) {} + + import( + file: string, + options: NoteImportOptions, + ): { id: string; inserted: boolean } { + if (!NOTE_TYPES.includes(options.type)) + throw new Error(`Unsupported note type: ${options.type}`); + if (!NOTE_STATUSES.includes(options.status)) + throw new Error(`Unsupported note status: ${options.status}`); + const sourcePath = resolve(archiveInvocationRoot(), file); + const body = readFileSync(sourcePath, 'utf8').trim(); + if (!body) throw new Error('A research note cannot be empty.'); + assertSafeArchiveValue(body, '$.note.body'); + const title = (options.title ?? extractTitle(body, basename(file))).trim(); + if (!title) throw new Error('A research note requires a title.'); + const tags = normalizeList(options.tags); + const experiments = normalizeList(options.experiments); + const provenance = options.provenance?.trim() || basename(sourcePath); + assertSafeArchiveValue({ title, body, tags, provenance }, '$.note'); + const id = createHash('sha256') + .update(JSON.stringify({ title, body, type: options.type, provenance })) + .digest('hex'); + const timestamp = this.archive.clock().toISOString(); + return this.archive.transaction(() => { + const result = this.archive.database + .prepare( + ` + INSERT OR IGNORE INTO notes( + id, title, body, type, status, provenance, superseded_note_id, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ) + .run( + id, + title, + body, + options.type, + options.status, + provenance, + options.supersedes ?? null, + timestamp, + timestamp, + ); + if (result.changes === 0) return { id, inserted: false }; + const tagStatement = this.archive.database.prepare( + 'INSERT INTO note_tags(note_id, tag) VALUES (?, ?)', + ); + for (const tag of tags) tagStatement.run(id, tag); + const experimentStatement = this.archive.database.prepare( + 'INSERT INTO note_experiments(note_id, experiment_id) VALUES (?, ?)', + ); + for (const experiment of experiments) + experimentStatement.run(id, experiment); + this.archive.database + .prepare( + 'INSERT INTO notes_fts(note_id, title, body, tags) VALUES (?, ?, ?, ?)', + ) + .run(id, title, body, tags.join(' ')); + if (options.supersedes) + this.archive.database + .prepare( + "UPDATE notes SET status = 'superseded', updated_at = ? WHERE id = ?", + ) + .run(timestamp, options.supersedes); + return { id, inserted: true }; + }); + } + + list(filters: NoteFilters = {}): QueryPage> { + return this.query(undefined, filters); + } + + search( + query: string, + filters: NoteFilters = {}, + ): QueryPage> { + const terms = query.trim().split(/\s+/).filter(Boolean); + if (terms.length === 0) + throw new Error('A note search query cannot be empty.'); + const safeQuery = terms + .map((term) => `"${term.replaceAll('"', '""')}"`) + .join(' AND '); + return this.query(safeQuery, filters); + } + + private query( + search: string | undefined, + filters: NoteFilters, + ): QueryPage> { + const bounded = limit(filters.limit); + const clauses: string[] = []; + const values: Array = []; + if (filters.type) { + clauses.push('n.type = ?'); + values.push(filters.type); + } + if (filters.status) { + clauses.push('n.status = ?'); + values.push(filters.status); + } + if (filters.tag) { + clauses.push( + 'EXISTS (SELECT 1 FROM note_tags nt WHERE nt.note_id = n.id AND nt.tag = ?)', + ); + values.push(filters.tag.toLowerCase()); + } + if (filters.experiment) { + clauses.push( + 'EXISTS (SELECT 1 FROM note_experiments ne WHERE ne.note_id = n.id AND ne.experiment_id = ?)', + ); + values.push(filters.experiment); + } + if (search) { + clauses.push('notes_fts MATCH ?'); + values.push(search); + } + const rows = this.archive.database + .prepare( + ` + SELECT n.id, n.title, n.type, n.status, n.provenance, + n.superseded_note_id AS supersedes, + n.created_at AS createdAt, n.updated_at AS updatedAt, + (SELECT GROUP_CONCAT(tag) FROM note_tags listed_tags + WHERE listed_tags.note_id = n.id) AS tags, + ${search ? 'ROUND(bm25(notes_fts), 6)' : 'NULL'} AS relevance + FROM notes n + ${search ? 'JOIN notes_fts ON notes_fts.note_id = n.id' : ''} + WHERE 1 = 1 ${clauses.map((clause) => `AND ${clause}`).join('\n')} + ORDER BY ${search ? 'relevance ASC,' : ''} n.updated_at DESC, n.id ASC + LIMIT ? + `, + ) + .all(...values, bounded + 1) as Array>; + for (const row of rows) + row.tags = typeof row.tags === 'string' ? row.tags.split(',').sort() : []; + return { + rows: rows.slice(0, bounded), + limit: bounded, + truncated: rows.length > bounded, + }; + } +} diff --git a/packages/experiment-archive/src/query-service.ts b/packages/experiment-archive/src/query-service.ts new file mode 100644 index 0000000..29526c3 --- /dev/null +++ b/packages/experiment-archive/src/query-service.ts @@ -0,0 +1,921 @@ +import type { DatabaseSync } from 'node:sqlite'; + +import { ArchiveDatabase } from './database.js'; + +export const DEFAULT_DETAIL_LIMIT = 50; +export const MAX_DETAIL_LIMIT = 500; + +export interface DetailFilters { + agent?: string; + fromTurn?: number; + toTurn?: number; + action?: string; + outcome?: string; + channel?: string; + sender?: string; + recipient?: string; + reason?: string; + limit?: number; +} + +export interface QueryPage { + rows: T[]; + limit: number; + truncated: boolean; +} + +interface SqlFilter { + clauses: string[]; + values: Array; +} + +function boundedLimit(limit?: number): number { + if (limit === undefined) return DEFAULT_DETAIL_LIMIT; + if (!Number.isInteger(limit) || limit < 1) + throw new Error('Limit must be a positive integer.'); + return Math.min(limit, MAX_DETAIL_LIMIT); +} + +function page(rows: T[], limit: number): QueryPage { + return { rows: rows.slice(0, limit), limit, truncated: rows.length > limit }; +} + +function turnFilter(filters: DetailFilters, alias = ''): SqlFilter { + const prefix = alias ? `${alias}.` : ''; + const clauses: string[] = []; + const values: Array = []; + const add = ( + column: string, + value: string | number | undefined, + operator = '=', + ) => { + if (value === undefined) return; + clauses.push(`${prefix}${column} ${operator} ?`); + values.push(value); + }; + add('agent_id', filters.agent); + add('turn_number', filters.fromTurn, '>='); + add('turn_number', filters.toTurn, '<='); + add('action', filters.action); + add('outcome', filters.outcome); + return { clauses, values }; +} + +function asObject(value: unknown): Record | null { + return typeof value === 'object' && value !== null + ? (value as Record) + : null; +} + +function parseJson(value: unknown, fallback: T): T { + if (typeof value !== 'string') return fallback; + try { + return JSON.parse(value) as T; + } catch { + return fallback; + } +} + +export class ExperimentQueryService { + readonly #db: DatabaseSync; + + constructor(archive: ArchiveDatabase) { + this.#db = archive.database; + } + + list( + filters: Pick = {}, + ): QueryPage> { + const limit = boundedLimit(filters.limit); + const rows = this.#db + .prepare( + ` + SELECT e.id, e.started_at AS startedAt, e.provider_mode AS providerMode, + e.total_completed_turns AS totalCompletedTurns, + COUNT(DISTINCT t.id) AS archivedTurns, + COUNT(DISTINCT a.agent_id) AS rosterSize, + e.retention_complete AS retentionComplete, + e.dropped_records AS droppedRecords, + e.imported_at AS importedAt + FROM experiments e + LEFT JOIN turns t ON t.experiment_id = e.id + LEFT JOIN agents a ON a.experiment_id = e.id + GROUP BY e.id + ORDER BY e.started_at DESC, e.id ASC + LIMIT ? + `, + ) + .all(limit + 1) as Array>; + return page(rows, limit); + } + + turns( + experimentId: string, + filters: DetailFilters = {}, + ): QueryPage> { + const limit = boundedLimit(filters.limit); + const built = turnFilter(filters); + const rows = this.#db + .prepare( + ` + SELECT turn_number AS turn, agent_id AS agent, outcome, action, + action_accepted AS actionAccepted, action_reason AS reason, + position_before AS positionBefore, position_after AS positionAfter, + model_id AS model, reasoning_profile AS reasoning, + latency_ms AS latencyMs, total_tokens AS totalTokens, + cost_credits AS costCredits, summary + FROM turns + WHERE experiment_id = ? + ${built.clauses.map((clause) => `AND ${clause}`).join('\n')} + ORDER BY turn_number ASC, id ASC + LIMIT ? + `, + ) + .all(experimentId, ...built.values, limit + 1) as Array< + Record + >; + return page(rows, limit); + } + + communications( + experimentId: string, + filters: DetailFilters = {}, + ): QueryPage> { + const limit = boundedLimit(filters.limit); + const clauses: string[] = []; + const values: Array = []; + const add = ( + column: string, + value: string | number | undefined, + operator = '=', + ) => { + if (value === undefined) return; + clauses.push(`${column} ${operator} ?`); + values.push(value); + }; + add('turn_number', filters.fromTurn, '>='); + add('turn_number', filters.toTurn, '<='); + add('channel', filters.channel); + add('sender_agent_id', filters.sender ?? filters.agent); + if (filters.recipient !== undefined) { + clauses.push(`( + recipient_agent_id = ? OR EXISTS ( + SELECT 1 FROM communication_recipients cr + WHERE cr.communication_id = communications.id AND cr.recipient_agent_id = ? + ) + )`); + values.push(filters.recipient, filters.recipient); + } + add('rejection_reason', filters.reason); + add('status', filters.outcome); + const rows = this.#db + .prepare( + ` + SELECT id, turn_number AS turn, occurred_at AS occurredAt, + sender_agent_id AS sender, channel, + recipient_agent_id AS recipient, status, + rejection_reason AS reason, message + FROM communications + WHERE experiment_id = ? + ${clauses.map((clause) => `AND ${clause}`).join('\n')} + ORDER BY turn_number ASC, occurred_at ASC, id ASC + LIMIT ? + `, + ) + .all(experimentId, ...values, limit + 1) as Array< + Record + >; + return page(rows, limit); + } + + allianceEvents( + experimentId: string, + filters: DetailFilters = {}, + ): QueryPage> { + const limit = boundedLimit(filters.limit); + const clauses: string[] = []; + const values: Array = []; + const add = ( + column: string, + value: string | number | undefined, + operator = '=', + ) => { + if (value === undefined) return; + clauses.push(`${column} ${operator} ?`); + values.push(value); + }; + add('agent_id', filters.agent); + add('turn_number', filters.fromTurn, '>='); + add('turn_number', filters.toTurn, '<='); + add('type', filters.action ?? filters.outcome); + add('reason', filters.reason); + const rows = this.#db + .prepare( + ` + SELECT id, turn_number AS turn, occurred_at AS occurredAt, + agent_id AS agent, type, reason, source_json AS source + FROM alliance_events + WHERE experiment_id = ? + ${clauses.map((clause) => `AND ${clause}`).join('\n')} + ORDER BY turn_number ASC, occurred_at ASC, id ASC + LIMIT ? + `, + ) + .all(experimentId, ...values, limit + 1) as Array< + Record + >; + for (const row of rows) row.source = parseJson(row.source, null); + return page(rows, limit); + } + + failures( + experimentId: string, + filters: DetailFilters = {}, + ): QueryPage> { + const limit = boundedLimit(filters.limit); + const built: SqlFilter = { clauses: [], values: [] }; + if (filters.agent !== undefined) { + built.clauses.push('m.agent_id = ?'); + built.values.push(filters.agent); + } + if (filters.fromTurn !== undefined) { + built.clauses.push('m.turn_number >= ?'); + built.values.push(filters.fromTurn); + } + if (filters.toTurn !== undefined) { + built.clauses.push('m.turn_number <= ?'); + built.values.push(filters.toTurn); + } + if (filters.reason !== undefined) { + built.clauses.push('m.failure_code = ?'); + built.values.push(filters.reason); + } + const rows = this.#db + .prepare( + ` + SELECT m.id, m.turn_number AS turn, m.agent_id AS agent, + m.attempt_number AS attempt, m.kind, m.model_id AS model, + m.failure_code AS code, m.failure_message AS message, + m.validation_codes_json AS validationCodes, + m.latency_ms AS latencyMs + FROM model_attempts m + WHERE m.experiment_id = ? AND m.failure_code IS NOT NULL + ${built.clauses.map((clause) => `AND ${clause}`).join('\n')} + ORDER BY m.turn_number ASC, m.attempt_number ASC, m.id ASC + LIMIT ? + `, + ) + .all(experimentId, ...built.values, limit + 1) as Array< + Record + >; + for (const row of rows) + row.validationCodes = parseJson(row.validationCodes, []); + return page(rows, limit); + } + + patientZero( + experimentId: string, + filters: DetailFilters = {}, + ): QueryPage> { + const limit = boundedLimit(filters.limit); + const patientZero = this.#db + .prepare( + 'SELECT agent_id AS agentId FROM agents WHERE experiment_id = ? AND is_patient_zero = 1', + ) + .get(experimentId) as { agentId: string } | undefined; + if (!patientZero) return page([], limit); + const communications = this.#db + .prepare( + ` + SELECT c.id, c.turn_number AS turn, c.occurred_at AS occurredAt, + c.sender_agent_id AS sender, c.channel, c.recipient_agent_id AS recipient, + c.message, c.status + FROM communications c + WHERE c.experiment_id = ? AND c.status = 'accepted' + ORDER BY c.turn_number ASC, c.occurred_at ASC, c.id ASC + `, + ) + .all(experimentId) as Array>; + const turns = this.#db + .prepare( + ` + SELECT turn_number AS turn, agent_id AS agent, action, outcome, completed_at AS completedAt + FROM turns WHERE experiment_id = ? ORDER BY turn_number ASC, id ASC + `, + ) + .all(experimentId) as Array>; + const recipients = this.#db + .prepare( + ` + SELECT cr.communication_id AS communicationId, cr.recipient_agent_id AS recipient + FROM communication_recipients cr + JOIN communications c ON c.id = cr.communication_id + WHERE c.experiment_id = ? + `, + ) + .all(experimentId) as Array<{ + communicationId: string; + recipient: string; + }>; + const recipientMap = new Map(); + for (const entry of recipients) + recipientMap.set(entry.communicationId, [ + ...(recipientMap.get(entry.communicationId) ?? []), + entry.recipient, + ]); + const directives = communications.filter( + ({ channel, sender }) => + channel === 'zero' && sender === patientZero.agentId, + ); + const rows: Array> = []; + for (const directive of directives) { + const addressed = recipientMap.get(String(directive.id)) ?? []; + rows.push({ + kind: 'directive', + id: directive.id, + turn: directive.turn, + agent: directive.sender, + message: directive.message, + recipients: addressed, + }); + for (const agent of addressed) { + const reply = communications.find( + (entry) => + entry.channel === 'direct' && + entry.sender === agent && + entry.recipient === patientZero.agentId && + Number(entry.turn) > Number(directive.turn) && + !directives.some( + (later) => + Number(later.turn) > Number(directive.turn) && + Number(later.turn) < Number(entry.turn) && + (recipientMap.get(String(later.id)) ?? []).includes(agent), + ), + ); + if (reply) + rows.push({ + kind: 'reply-after-directive', + id: reply.id, + turn: reply.turn, + agent, + directiveId: directive.id, + message: reply.message, + }); + const nextTurn = turns.find( + (turn) => + turn.agent === agent && Number(turn.turn) > Number(directive.turn), + ); + if (nextTurn) { + const requested = directiveAction(String(directive.message)); + rows.push({ + kind: 'observable-compliance', + id: `${String(directive.id)}:${agent}`, + turn: nextTurn.turn, + agent, + directiveId: directive.id, + requestedAction: requested, + observedAction: nextTurn.action, + classification: + requested === null + ? 'indeterminate' + : requested === nextTurn.action && + nextTurn.outcome === 'accepted' + ? 'compliant' + : 'noncompliant', + }); + } + } + } + for (const message of communications.filter( + (entry) => + entry.channel === 'direct' && entry.recipient === patientZero.agentId, + )) + if (!rows.some(({ id }) => id === message.id)) + rows.push({ + kind: 'message-to-patient-zero', + id: message.id, + turn: message.turn, + agent: message.sender, + message: message.message, + }); + const filtered = rows + .filter( + (row) => filters.agent === undefined || row.agent === filters.agent, + ) + .filter( + (row) => + filters.fromTurn === undefined || + Number(row.turn) >= filters.fromTurn, + ) + .filter( + (row) => + filters.toTurn === undefined || Number(row.turn) <= filters.toTurn, + ) + .sort( + (a, b) => + Number(a.turn) - Number(b.turn) || + String(a.kind).localeCompare(String(b.kind)) || + String(a.id).localeCompare(String(b.id)), + ); + return page(filtered.slice(0, limit + 1), limit); + } + + summary(experimentId: string): Record { + const experiment = this.#db + .prepare('SELECT * FROM experiments WHERE id = ?') + .get(experimentId) as Record | undefined; + if (!experiment) throw new Error(`Unknown experiment: ${experimentId}`); + const roster = this.#db + .prepare( + ` + SELECT agent_id AS id, name, model_id AS model, reasoning_profile AS reasoning, + personality_id AS personality, strategy_id AS strategy, + is_patient_zero AS patientZero + FROM agents WHERE experiment_id = ? ORDER BY agent_id ASC + `, + ) + .all(experimentId); + const sourceExports = ( + this.#db + .prepare( + ` + SELECT sha256, source_path AS sourcePath, generated_at AS generatedAt, + filters_json AS filters, retention_json AS retention + FROM source_exports WHERE experiment_id = ? + ORDER BY generated_at ASC, sha256 ASC + `, + ) + .all(experimentId) as Array> + ).map((source) => ({ + ...source, + filters: parseJson(source.filters, null), + retention: parseJson(source.retention, null), + })); + const turnMetrics = this.#db + .prepare( + ` + SELECT COUNT(*) AS requested, + SUM(outcome = 'accepted') AS accepted, + SUM(outcome = 'rejected') AS rejected, + SUM(outcome = 'provider-error') AS failed, + SUM(outcome = 'operator-skipped') AS lost, + SUM((SELECT COUNT(*) FROM model_attempts m WHERE m.turn_id = turns.id) > 1) AS retried + FROM turns WHERE experiment_id = ? + `, + ) + .get(experimentId); + const actions = this.#db + .prepare( + ` + SELECT COALESCE(action, 'none') AS action, COUNT(*) AS count + FROM turns WHERE experiment_id = ? GROUP BY action ORDER BY action ASC + `, + ) + .all(experimentId); + const communications = this.#db + .prepare( + ` + SELECT channel, status, COUNT(*) AS count FROM communications + WHERE experiment_id = ? GROUP BY channel, status ORDER BY channel, status + `, + ) + .all(experimentId); + const allianceLifecycle = this.#db + .prepare( + ` + SELECT type, COALESCE(reason, '') AS reason, COUNT(*) AS count + FROM alliance_events WHERE experiment_id = ? + GROUP BY type, reason ORDER BY type, reason + `, + ) + .all(experimentId); + const diplomacyRejections = this.#db + .prepare( + ` + SELECT rejection_reason AS reason, COUNT(*) AS count + FROM diplomacy_attempts WHERE experiment_id = ? AND accepted = 0 + GROUP BY rejection_reason ORDER BY rejection_reason + `, + ) + .all(experimentId); + const diplomacyOutcomes = this.#db + .prepare( + ` + SELECT type, accepted, COUNT(*) AS count + FROM diplomacy_attempts WHERE experiment_id = ? + GROUP BY type, accepted ORDER BY type, accepted DESC + `, + ) + .all(experimentId); + const usageByAgent = this.#db + .prepare( + ` + SELECT agent_id AS agent, + COUNT(*) AS modelAttempts, + SUM(latency_ms) AS latencyTotalMs, + SUM(latency_ms IS NOT NULL) AS attemptsWithKnownLatency, + ROUND(AVG(latency_ms), 2) AS averageLatencyMs, + SUM(prompt_tokens) AS promptTokens, + SUM(completion_tokens) AS completionTokens, + SUM(total_tokens) AS totalTokens, + ROUND(SUM(cost_credits), 8) AS knownCostCredits, + SUM(cost_credits IS NULL) AS attemptsWithUnknownCost + FROM model_attempts WHERE experiment_id = ? + GROUP BY agent_id ORDER BY agent_id + `, + ) + .all(experimentId) as Array>; + const usageAggregate = aggregateUsage(usageByAgent); + const sizeTrends = this.#db + .prepare( + ` + SELECT MIN(observation_size_bytes) AS minObservationBytes, + ROUND(AVG(observation_size_bytes), 2) AS averageObservationBytes, + MAX(observation_size_bytes) AS maxObservationBytes, + MIN(prompt_tokens) AS minPromptTokens, + ROUND(AVG(prompt_tokens), 2) AS averagePromptTokens, + MAX(prompt_tokens) AS maxPromptTokens + FROM turns WHERE experiment_id = ? + `, + ) + .get(experimentId); + const territoryRows = this.#db + .prepare( + ` + SELECT current_controller_agent_id AS agent, COUNT(*) AS controlledCells + FROM map_cells + WHERE experiment_id = ? AND current_state = 'infected' + GROUP BY current_controller_agent_id ORDER BY current_controller_agent_id + `, + ) + .all(experimentId); + const territoryChanges = this.#db + .prepare( + ` + SELECT type, COUNT(*) AS count + FROM world_events + WHERE experiment_id = ? AND type IN ('hex-infected', 'hex-captured') + GROUP BY type ORDER BY type + `, + ) + .all(experimentId); + const sourceTerritory = parseJson(experiment.source_territory_json, []); + const directions = canonicalDirectionChanges(this.#db, experimentId); + const patientZero = this.patientZero(experimentId, { + limit: MAX_DETAIL_LIMIT, + }).rows; + const sourceInconsistencies = parseJson>>( + experiment.metric_inconsistencies_json, + [], + ); + const missing: string[] = []; + if (!Boolean(experiment.retention_complete)) + missing.push( + `source retention is incomplete (${String(experiment.dropped_records)} dropped records)`, + ); + if ( + Number(turnMetrics && asObject(turnMetrics)?.requested) < + Number(experiment.retained_turns) + ) + missing.push('not all retained turns are present in imported selections'); + if (!experiment.scenario_json) + missing.push('scenario configuration absent'); + if (sizeTrends && asObject(sizeTrends)?.minObservationBytes === null) + missing.push('retained observations absent'); + const availableSections = sourceCompleteness(sourceExports); + for (const [section, available] of Object.entries(availableSections)) + if (!available) + missing.push(`${section} excluded by every imported export`); + return { + experiment: { + id: experimentId, + startedAt: experiment.started_at, + providerMode: experiment.provider_mode, + scenario: parseJson(experiment.scenario_json, null), + decisionContractVersion: experiment.decision_contract_version, + observationContractVersion: experiment.observation_contract_version, + }, + roster, + sourceExports, + turns: turnMetrics, + actions, + territory: { + current: territoryRows.length > 0 ? territoryRows : sourceTerritory, + changes: territoryChanges, + }, + communications, + alliances: { + lifecycle: allianceLifecycle, + attempts: diplomacyOutcomes, + rejections: diplomacyRejections, + }, + patientZero: { + directives: patientZero.filter(({ kind }) => kind === 'directive') + .length, + messagesToPatientZero: + patientZero.filter(({ kind }) => kind === 'message-to-patient-zero') + .length + + patientZero.filter(({ kind }) => kind === 'reply-after-directive') + .length, + repliesAfterDirective: patientZero.filter( + ({ kind }) => kind === 'reply-after-directive', + ).length, + observableCompliance: countBy( + patientZero.filter(({ kind }) => kind === 'observable-compliance'), + 'classification', + ), + }, + usage: { aggregate: usageAggregate, byAgent: usageByAgent }, + promptAndObservationSizeTrends: sizeTrends, + directionChangesAfterCommunication: directions, + retention: { + limit: experiment.retention_limit, + totalCompletedTurns: experiment.total_completed_turns, + retainedTurns: experiment.retained_turns, + archivedTurns: asObject(turnMetrics)?.requested ?? 0, + droppedRecords: experiment.dropped_records, + complete: Boolean(experiment.retention_complete), + requestedRangeExtendsBeyondRetention: Boolean( + experiment.requested_range_extends_beyond_retention, + ), + availableSections, + }, + inconsistencies: [ + ...sourceInconsistencies, + ...missing.map((detail) => ({ metric: 'missing-data', detail })), + ], + }; + } + + compare(leftId: string, rightId: string): Record { + const left = comparisonMetrics(this.#db, leftId); + const right = comparisonMetrics(this.#db, rightId); + return { + normalization: { + perTurn: 'total / archived turn', + perAgentTurn: 'total / archived agent-turn', + perActiveAgent: 'total / agent with at least one archived turn', + perPatientZeroTurn: 'Patient Zero total / archived Patient Zero turn', + }, + left, + right, + delta: metricDelta(left, right), + }; + } +} + +function directiveAction(message: string): string | null { + const matches = ['move', 'infect', 'capture', 'wait'].filter((action) => + new RegExp(`\\b${action}\\b`, 'i').test(message), + ); + return matches.length === 1 ? matches[0]! : null; +} + +function countBy(rows: Array>, key: string) { + const counts: Record = {}; + for (const row of rows) { + const value = String(row[key]); + counts[value] = (counts[value] ?? 0) + 1; + } + return counts; +} + +interface UsageTotals { + modelAttempts: number; + latencyTotalMs: number; + attemptsWithKnownLatency: number; + promptTokens: number; + completionTokens: number; + totalTokens: number; + knownCostCredits: number; + attemptsWithUnknownCost: number; +} + +function aggregateUsage(rows: Array>) { + const totals = rows.reduce( + (aggregate, row) => ({ + modelAttempts: aggregate.modelAttempts + Number(row.modelAttempts ?? 0), + latencyTotalMs: + aggregate.latencyTotalMs + Number(row.latencyTotalMs ?? 0), + attemptsWithKnownLatency: + aggregate.attemptsWithKnownLatency + + Number(row.attemptsWithKnownLatency ?? 0), + promptTokens: aggregate.promptTokens + Number(row.promptTokens ?? 0), + completionTokens: + aggregate.completionTokens + Number(row.completionTokens ?? 0), + totalTokens: aggregate.totalTokens + Number(row.totalTokens ?? 0), + knownCostCredits: + aggregate.knownCostCredits + Number(row.knownCostCredits ?? 0), + attemptsWithUnknownCost: + aggregate.attemptsWithUnknownCost + + Number(row.attemptsWithUnknownCost ?? 0), + }), + { + modelAttempts: 0, + latencyTotalMs: 0, + attemptsWithKnownLatency: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + knownCostCredits: 0, + attemptsWithUnknownCost: 0, + }, + ); + return { + ...totals, + averageLatencyMs: + totals.attemptsWithKnownLatency === 0 + ? null + : Number( + (totals.latencyTotalMs / totals.attemptsWithKnownLatency).toFixed( + 2, + ), + ), + }; +} + +function sourceCompleteness(sources: Array>) { + const filters = sources + .map(({ filters }) => asObject(filters)) + .filter((value): value is Record => value !== null); + const includes = ( + standardLevels: readonly string[], + customKey: string, + selection: 'turns' | 'communications' | 'none' = 'none', + ): boolean => + filters.some((filter) => { + const level = filter.level; + const included = + (typeof level === 'string' && standardLevels.includes(level)) || + (level === 'custom' && asObject(filter.custom)?.[customKey] === true); + if (!included) return false; + if (selection === 'none') return true; + const agents = asObject(filter.agents); + const turns = asObject(filter.turns); + const outcomes = Array.isArray(filter.outcomes) ? filter.outcomes : []; + const actions = Array.isArray(filter.actions) ? filter.actions : []; + const completeTurns = + agents?.mode === 'all' && + turns?.mode === 'entire-retained' && + outcomes.length === 4 && + actions.length === 4; + if (selection === 'turns') return completeTurns; + const communication = asObject(filter.communications); + return ( + completeTurns && + communication?.channel === 'all' && + communication.status === 'all' + ); + }); + return { + observations: includes( + ['standard', 'full-safe'], + 'turnObservations', + 'turns', + ), + metrics: includes( + ['minimal', 'standard', 'full-safe'], + 'computedMetrics', + 'turns', + ), + communications: includes( + ['minimal', 'standard', 'full-safe'], + 'communications', + 'communications', + ), + currentWorld: includes(['full-safe'], 'currentWorldState'), + initialWorld: includes(['full-safe'], 'initialWorldState'), + }; +} + +interface DirectionTurn { + turn: number; + agent: string; + direction: string; + inboundSincePreviousMove: number; +} + +function canonicalDirectionChanges(db: DatabaseSync, experimentId: string) { + const rows = db + .prepare( + ` + SELECT turn_number AS turn, agent_id AS agent, move_direction AS direction, + inbound_communication_since_previous_move AS inboundSincePreviousMove + FROM turns + WHERE experiment_id = ? AND move_direction IS NOT NULL + ORDER BY agent_id, turn_number, id + `, + ) + .all(experimentId) as unknown as DirectionTurn[]; + const byAgent: Array<{ agent: string; count: number }> = []; + for (const agent of [...new Set(rows.map(({ agent }) => agent))].sort()) { + let previousDirection: string | null = null; + let count = 0; + for (const row of rows.filter((entry) => entry.agent === agent)) { + if ( + previousDirection && + row.direction !== previousDirection && + Boolean(row.inboundSincePreviousMove) + ) + count += 1; + previousDirection = row.direction; + } + byAgent.push({ agent, count }); + } + return { + canonicalDefinition: + 'For each agent independently, count an accepted move whose direction differs from that agent previous accepted move when the retained observation contains an inbound direct or alliance message after the previous move and no later than the current turn. Aggregate is the sum of per-agent counts.', + aggregate: byAgent.reduce((sum, entry) => sum + entry.count, 0), + byAgent, + agreement: true, + }; +} + +function comparisonMetrics(db: DatabaseSync, experimentId: string) { + const base = db + .prepare( + ` + SELECT COUNT(*) AS turns, + COUNT(DISTINCT agent_id) AS activeAgents, + SUM(outcome = 'accepted') AS accepted, + SUM(outcome = 'provider-error') AS failed, + SUM(outcome = 'operator-skipped') AS lost + FROM turns WHERE experiment_id = ? + `, + ) + .get(experimentId) as Record; + const communicationCount = db + .prepare( + "SELECT COUNT(*) AS count FROM communications WHERE experiment_id = ? AND status = 'accepted'", + ) + .get(experimentId) as { count: number }; + const patientZeroTurns = db + .prepare( + ` + SELECT COUNT(*) AS count FROM turns t JOIN agents a + ON a.experiment_id = t.experiment_id AND a.agent_id = t.agent_id + WHERE t.experiment_id = ? AND a.is_patient_zero = 1 + `, + ) + .get(experimentId) as { count: number }; + const patientZeroMessages = db + .prepare( + ` + SELECT COUNT(*) AS count FROM communications c JOIN agents a + ON a.experiment_id = c.experiment_id AND a.agent_id = c.sender_agent_id + WHERE c.experiment_id = ? AND a.is_patient_zero = 1 AND c.status = 'accepted' + `, + ) + .get(experimentId) as { count: number }; + const turns = Number(base.turns); + const activeAgents = Number(base.activeAgents); + const messages = Number(communicationCount.count); + const zeroTurns = Number(patientZeroTurns.count); + return { + experimentId, + absolute: { + ...base, + communications: messages, + patientZeroMessages: patientZeroMessages.count, + patientZeroTurns: zeroTurns, + }, + normalized: { + communicationsPerTurn: rate(messages, turns), + communicationsPerAgentTurn: rate(messages, turns), + communicationsPerActiveAgent: rate(messages, activeAgents), + patientZeroMessagesPerPatientZeroTurn: rate( + patientZeroMessages.count, + zeroTurns, + ), + acceptedPerTurn: rate(Number(base.accepted), turns), + failedOrLostPerTurn: rate(Number(base.failed) + Number(base.lost), turns), + }, + }; +} + +function rate(value: number, denominator: number): number | null { + return denominator === 0 ? null : Number((value / denominator).toFixed(6)); +} + +function metricDelta( + left: Record, + right: Record, +) { + const leftNormalized = asObject(left.normalized) ?? {}; + const rightNormalized = asObject(right.normalized) ?? {}; + return Object.fromEntries( + Object.keys(leftNormalized) + .sort() + .map((key) => [ + key, + typeof leftNormalized[key] === 'number' && + typeof rightNormalized[key] === 'number' + ? Number( + ( + Number(rightNormalized[key]) - Number(leftNormalized[key]) + ).toFixed(6), + ) + : null, + ]), + ); +} diff --git a/packages/experiment-archive/tsconfig.json b/packages/experiment-archive/tsconfig.json new file mode 100644 index 0000000..9e25e6e --- /dev/null +++ b/packages/experiment-archive/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d714346..0255ea2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -128,6 +128,25 @@ importers: specifier: 4.1.10 version: 4.1.10(@types/node@24.13.3)(jsdom@30.0.1)(vite@8.2.1(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)) + packages/experiment-archive: + dependencies: + '@agentborne/shared': + specifier: workspace:* + version: link:../shared + devDependencies: + '@agentborne/agent-runtime': + specifier: workspace:* + version: link:../agent-runtime + '@agentborne/world-engine': + specifier: workspace:* + version: link:../world-engine + tsx: + specifier: 4.23.12 + version: 4.23.12 + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@24.13.3)(jsdom@30.0.1)(vite@8.2.1(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)) + packages/shared: dependencies: zod: