diff --git a/bun.lock b/bun.lock index 4fa759b..9c14794 100644 --- a/bun.lock +++ b/bun.lock @@ -12,6 +12,13 @@ "typescript": "^5.5.0", }, }, + "packages/aep-export": { + "name": "@fresharena/aep-export", + "version": "0.1.0", + "dependencies": { + "@fresharena/faep-schema": "workspace:*", + }, + }, "packages/cli": { "name": "@fresharena/cli", "version": "0.1.0", @@ -135,6 +142,8 @@ "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@1.9.4", "", { "os": "win32", "cpu": "x64" }, "sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA=="], + "@fresharena/aep-export": ["@fresharena/aep-export@workspace:packages/aep-export"], + "@fresharena/cli": ["@fresharena/cli@workspace:packages/cli"], "@fresharena/core": ["@fresharena/core@workspace:packages/core"], diff --git a/packages/aep-export/package.json b/packages/aep-export/package.json new file mode 100644 index 0000000..bff0acd --- /dev/null +++ b/packages/aep-export/package.json @@ -0,0 +1,33 @@ +{ + "name": "@fresharena/aep-export", + "version": "0.1.0", + "description": "Export FreshArena FAEP evaluation records as AEP evidence bundles for open-agent-audit integration", + "license": "Apache-2.0", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": ["dist"], + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "bun test ./src || (ls src 2>/dev/null | grep -q \"\\.test\\.ts\\|\\.spec\\.ts\" && exit 1 || exit 0)", + "clean": "rm -rf dist .turbo" + }, + "dependencies": { + "@fresharena/faep-schema": "workspace:*" + }, + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/WasmAgent/fresharena.git", + "directory": "packages/aep-export" + } +} diff --git a/packages/aep-export/src/index.test.ts b/packages/aep-export/src/index.test.ts new file mode 100644 index 0000000..76537fb --- /dev/null +++ b/packages/aep-export/src/index.test.ts @@ -0,0 +1,140 @@ +import { expect, test } from 'bun:test'; +import type { FaepRecord } from '@fresharena/faep-schema'; +import type { AepEvidenceBundle } from './index.js'; +import { + exportAsAepBundle, + exportAsAepBundles, + serializeBundle, + serializeBundles, +} from './index.js'; + +function makeFaepRecord(overrides?: Partial): FaepRecord { + const zeros = '0'.repeat(64); + const ones = '1'.repeat(64); + const twos = '2'.repeat(64); + const threes = '3'.repeat(64); + const fours = '4'.repeat(64); + const fives = '5'.repeat(64); + const sixes = '6'.repeat(64); + const sevens = '7'.repeat(64); + + return { + schema_version: '0.1.0', + run_id: 'run-test-1', + task: { + id: 'json_transform.normalize.v0|seed-1', + family: 'json_transform.normalize.v0', + family_version: '0.1.0', + seed_hash: zeros, + spec_hash: ones, + }, + solver: { + id: 'reference', + track: 'non_llm', + model_metadata_hash: twos, + workflow_hash: threes, + artifact_hash: fours, + }, + generator: { + id: 'random-baseline', + version: '0.1.0', + seed_hash: zeros, + }, + tester: { + id: 'closed-semantics', + version: '0.1.0', + tests_hash: fives, + }, + verifier: { + package: 'json_transform_verifier', + version: '0.1.0', + result_hash: sixes, + }, + environment: { + os: 'linux', + runtime: 'bun-1.3.14', + container_hash: sevens, + }, + score: { + canonical_pass: true, + hidden_pass: true, + adversarial_pass: true, + immunity_pass: true, + cost: {}, + score_vector: {}, + }, + replay: { + command: 'bun run fresharena replay --run-id run-test-1', + log_hash: zeros, + }, + ...overrides, + } as FaepRecord; +} + +test('exportAsAepBundle produces a valid AEP evidence envelope', () => { + const record = makeFaepRecord(); + const bundle = exportAsAepBundle(record); + + expect(bundle.aep_version).toBe('0.1.0'); + expect(bundle.source.system).toBe('fresharena'); + expect(bundle.source.component).toBe('aep-export'); + expect(bundle.source.version).toBe('0.1.0'); + expect(bundle.evidence).toEqual(record); + expect(bundle.produced_at).toBeTruthy(); + expect(bundle.evidence_hash).toBeTruthy(); + expect(bundle.evidence_hash).toHaveLength(64); +}); + +test('exportAsAepBundle accepts a custom version string', () => { + const record = makeFaepRecord(); + const bundle = exportAsAepBundle(record, '2.0.0'); + + expect(bundle.source.version).toBe('2.0.0'); +}); + +test('exportAsAepBundles converts multiple records', () => { + const records = [makeFaepRecord(), makeFaepRecord({ run_id: 'run-test-2' })]; + const bundles = exportAsAepBundles(records); + + expect(bundles).toHaveLength(2); + expect(bundles[0].evidence.run_id).toBe('run-test-1'); + expect(bundles[1].evidence.run_id).toBe('run-test-2'); + // Each bundle gets a unique evidence_hash based on its content. + expect(bundles[0].evidence_hash).not.toBe(bundles[1].evidence_hash); +}); + +test('serializeBundle produces valid JSON', () => { + const record = makeFaepRecord(); + const bundle = exportAsAepBundle(record); + const json = serializeBundle(bundle); + + const parsed = JSON.parse(json) as AepEvidenceBundle; + expect(parsed.aep_version).toBe('0.1.0'); + expect(parsed.source.system).toBe('fresharena'); + expect(parsed.evidence.run_id).toBe('run-test-1'); +}); + +test('serializeBundles produces JSONL (one JSON object per line)', () => { + const records = [makeFaepRecord(), makeFaepRecord({ run_id: 'run-test-2' })]; + const bundles = exportAsAepBundles(records); + const jsonl = serializeBundles(bundles); + + const lines = jsonl.split('\n'); + expect(lines).toHaveLength(2); + + for (const line of lines) { + const parsed = JSON.parse(line) as AepEvidenceBundle; + expect(parsed.aep_version).toBe('0.1.0'); + } +}); + +test('exportAsAepBundle preserves failure_diff annotation when present', () => { + const record = makeFaepRecord({ + failure_diff: { added: true, removed: false }, + failure_diff_hash: 'a'.repeat(64), + }); + const bundle = exportAsAepBundle(record); + + expect(bundle.evidence.failure_diff).toEqual({ added: true, removed: false }); + expect(bundle.evidence.failure_diff_hash).toBe('a'.repeat(64)); +}); diff --git a/packages/aep-export/src/index.ts b/packages/aep-export/src/index.ts new file mode 100644 index 0000000..8f3727e --- /dev/null +++ b/packages/aep-export/src/index.ts @@ -0,0 +1,100 @@ +/** + * @fresharena/aep-export + * + * Exports FreshArena FAEP evaluation records as AEP (Audit Evidence Protocol) + * evidence bundles compatible with the open-agent-audit integration. + * + * The AEP envelope wraps each FAEP record with provenance metadata so that + * downstream audit pipelines (trace-pipeline, open-agent-audit) can ingest + * FreshArena evaluations without coupling to FAEP internals. + */ + +import { createHash } from 'node:crypto'; +import type { FaepRecord } from '@fresharena/faep-schema'; + +// ─── AEP envelope schema ──────────────────────────────────────────────────── +// +// Minimal envelope that open-agent-audit expects. New fields are additive — +// existing consumers must ignore unknown keys. + +export interface AepEvidenceBundle { + /** AEP schema version this envelope targets. */ + readonly aep_version: '0.1.0'; + /** Provenance: which system produced this evidence. */ + readonly source: { + readonly system: 'fresharena'; + readonly component: 'aep-export'; + readonly version: string; + }; + /** The original FAEP record, embedded verbatim. */ + readonly evidence: FaepRecord; + /** ISO-8601 timestamp when this bundle was produced. */ + readonly produced_at: string; + /** SHA-256 of the canonical JSON serialization of `evidence`. */ + readonly evidence_hash: string; +} + +// ─── Export function ────────────────────────────────────────────────────────── + +/** + * Convert a single FAEP record into an AEP evidence bundle. + * + * @param record A validated FAEP v0.1 evaluation record. + * @param version Optional exporter version string (defaults to package version). + * @returns An `AepEvidenceBundle` ready for JSON serialization. + */ +export function exportAsAepBundle(record: FaepRecord, version = '0.1.0'): AepEvidenceBundle { + const evidenceJson = JSON.stringify(record); + const evidence_hash = sha256Hex(evidenceJson); + + return { + aep_version: '0.1.0', + source: { + system: 'fresharena', + component: 'aep-export', + version, + }, + evidence: record, + produced_at: new Date().toISOString(), + evidence_hash, + }; +} + +/** + * Convert an array of FAEP records into AEP evidence bundles. + * + * @param records Array of validated FAEP v0.1 evaluation records. + * @param version Optional exporter version string. + * @returns Array of `AepEvidenceBundle` objects. + */ +export function exportAsAepBundles( + records: readonly FaepRecord[], + version = '0.1.0', +): AepEvidenceBundle[] { + return records.map((record) => exportAsAepBundle(record, version)); +} + +/** + * Serialize an AEP evidence bundle to a JSONL line (no trailing newline). + * Suitable for streaming writes to `.jsonl` audit logs. + */ +export function serializeBundle(bundle: AepEvidenceBundle): string { + return JSON.stringify(bundle); +} + +/** + * Serialize multiple bundles as a JSONL string (one JSON object per line). + */ +export function serializeBundles(bundles: readonly AepEvidenceBundle[]): string { + return bundles.map(serializeBundle).join('\n'); +} + +// ─── SHA-256 helper ───────────────────────────────────────────────────────── + +function sha256Hex(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} + +// ─── Re-exports for convenience ──────────────────────────────────────────────── + +export type { FaepRecord } from '@fresharena/faep-schema'; diff --git a/packages/aep-export/tsconfig.json b/packages/aep-export/tsconfig.json new file mode 100644 index 0000000..5285d28 --- /dev/null +++ b/packages/aep-export/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src"] +}