From 8c3a00e011d6df3c40ef8cf80ff7efac5d65f692 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sun, 19 Jul 2026 18:38:47 +0800 Subject: [PATCH 1/2] Fix #117: Fix WasmAgent/fresharena#114 ([milestone Milestone 4 ] Hosted arena: `arena/` service accepts solver submissions via API --- arena/package.json | 36 +++ arena/src/evaluator.ts | 174 +++++++++++ arena/src/index.test.ts | 575 +++++++++++++++++++++++++++++++++++++ arena/src/index.ts | 55 ++++ arena/src/server.ts | 282 ++++++++++++++++++ arena/src/store.ts | 99 +++++++ arena/src/types.ts | 91 ++++++ arena/tsconfig.json | 8 + bun.lock | 12 + package.json | 1 + packages/core/package.json | 8 + 11 files changed, 1341 insertions(+) create mode 100644 arena/package.json create mode 100644 arena/src/evaluator.ts create mode 100644 arena/src/index.test.ts create mode 100644 arena/src/index.ts create mode 100644 arena/src/server.ts create mode 100644 arena/src/store.ts create mode 100644 arena/src/types.ts create mode 100644 arena/tsconfig.json diff --git a/arena/package.json b/arena/package.json new file mode 100644 index 0000000..291037f --- /dev/null +++ b/arena/package.json @@ -0,0 +1,36 @@ +{ + "name": "@fresharena/arena", + "version": "0.1.0", + "description": "FreshArena hosted arena service: accepts solver submissions via API, runs evaluations, and publishes leaderboard", + "license": "Apache-2.0", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": ["dist"], + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "bun test ./src || (ls src 2>/dev/null | grep -q \"\\.test\\.ts\\|\\.spec\\.ts\" && exit 1 || exit 0)", + "clean": "rm -rf dist .turbo" + }, + "dependencies": { + "@fresharena/faep-schema": "workspace:*", + "@fresharena/core": "workspace:*", + "@fresharena/verifier-runtime": "workspace:*", + "zod": "^3.23.0" + }, + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/WasmAgent/fresharena.git", + "directory": "arena" + } +} diff --git a/arena/src/evaluator.ts b/arena/src/evaluator.ts new file mode 100644 index 0000000..f711e9d --- /dev/null +++ b/arena/src/evaluator.ts @@ -0,0 +1,174 @@ +import type { TaskSpec } from '@fresharena/faep-schema'; +import { + verify, + sha256OfString, +} from '@fresharena/verifier-runtime'; +import type { EvaluationResponse } from './types.js'; +import { ArenaStore } from './store.js'; + +/** + * Run a single evaluation end-to-end: + * + * 1. Generate fresh tasks via `@fresharena/core/generator` + * 2. Execute the solver on each task's example input + * 3. Verify outputs via the deterministic verifier + * 4. Build per-task audit records + * 5. Persist results to the store + * + * Returns the completed `EvaluationResponse`. + */ +export async function runEvaluation(params: { + evaluationId: string; + submissionId: string; + solverId: string; + track: string; + family: string; + taskCount: number; + rootSeed: string; + store: ArenaStore; +}): Promise { + const { + evaluationId, + submissionId, + solverId, + track, + family, + taskCount, + rootSeed, + store, + } = params; + + const records: Array> = []; + let passed = 0; + let failed = 0; + let errors = 0; + + // Mark evaluation as running + store.updateEvaluation(evaluationId, { status: 'running' }); + + try { + // Import generator (lazy to avoid circular deps at module level) + const { generateTasks } = await import( + '@fresharena/core/generator' + ); + + const generateOutput = generateTasks({ + family: family as TaskSpec['family'], + count: taskCount, + rootSeed, + }); + + // Import built-in solver registry + const { getSolver } = await import( + '@fresharena/core/solvers' + ); + const solverEntry = getSolver(solverId); + + for (let i = 0; i < generateOutput.tasks.length; i++) { + const task = generateOutput.tasks[i]; + const taskSeed = generateOutput.taskSeeds[i]; + const admissibility = generateOutput.admissibilityResults[i]; + + try { + const input = task.examples[0]?.input ?? {}; + const start = performance.now(); + + // Run the solver + const actualOutput = await solverEntry.fn(input, task); + const durationMs = performance.now() - start; + + // Verify against the deterministic reference + const result = verify({ + taskId: task.id, + input, + output: actualOutput, + constraints: task.operation_spec.constraints, + }); + + if (result.passed) { + passed++; + } else { + failed++; + } + + records.push({ + schema_version: '0.1.0', + task_id: task.id, + task_family: task.family, + seed_hash: sha256OfString(taskSeed), + solver_id: solverId, + track, + verdict: result.passed ? 'pass' : 'fail', + duration_ms: Math.round(durationMs), + output_hash: result.actual_hash, + expected_hash: result.expected_hash, + error: null, + admissibility, + gen_duration_ms: generateOutput.genDurationsMs[i] ?? 0, + generator_id: 'random-baseline', + generator_version: '0.1.0', + verifier_package: task.verifier.package, + verifier_version: task.verifier.version, + }); + } catch (err) { + errors++; + records.push({ + schema_version: '0.1.0', + task_id: task.id, + task_family: task.family, + solver_id: solverId, + track, + verdict: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } + } + + const now = new Date().toISOString(); + const total = generateOutput.tasks.length; + + const evaluation: EvaluationResponse = { + evaluation_id: evaluationId, + submission_id: submissionId, + solver_id: solverId, + track, + family, + task_count: taskCount, + status: 'completed', + passed, + failed, + errors, + total, + records, + created_at: now, + completed_at: now, + }; + + store.updateEvaluation(evaluationId, evaluation); + return evaluation; + } catch (err) { + const now = new Date().toISOString(); + const errorMsg = err instanceof Error ? err.message : String(err); + + const evaluation: EvaluationResponse = { + evaluation_id: evaluationId, + submission_id: submissionId, + solver_id: solverId, + track, + family, + task_count: taskCount, + status: 'error', + passed: 0, + failed: 0, + errors: 0, + total: 0, + records: [], + created_at: now, + completed_at: now, + error: errorMsg, + }; + + store.updateEvaluation(evaluationId, evaluation); + return evaluation; + } +} diff --git a/arena/src/index.test.ts b/arena/src/index.test.ts new file mode 100644 index 0000000..6d373e2 --- /dev/null +++ b/arena/src/index.test.ts @@ -0,0 +1,575 @@ +import { afterEach, beforeAll, describe, expect, test } from 'bun:test'; +import { createArenaServer, ArenaStore } from './index.js'; +import type { ArenaServer } from './index.js'; + +/** Base URL for all test requests. */ +let BASE = ''; +let arena: ArenaServer; + +beforeAll(() => { + arena = createArenaServer({ port: 0, store: new ArenaStore() }); + BASE = arena.url; +}); + +afterEach(() => { + // Clear store between tests for isolation + for (const key of [ + ...arena.store.getEvaluation('x') ? ['x'] : [], + ]) { + // Store only exposes get methods; tests create fresh submissions + } +}); + +// ── Helpers ────────────────────────────────────────────────────────────────── + +async function get(path: string) { + return fetch(`${BASE}${path}`); +} + +async function post( + path: string, + body?: unknown, +): Promise { + return fetch(`${BASE}${path}`, { + method: 'POST', + headers: body !== undefined + ? { 'Content-Type': 'application/json' } + : {}, + body: body !== undefined ? JSON.stringify(body) : undefined, + }); +} + +async function getJson(path: string) { + const res = await get(path); + return { status: res.status, body: (await res.json()) as Record }; +} + +async function postJson( + path: string, + body?: unknown, +) { + const res = await post(path, body); + return { status: res.status, body: (await res.json()) as Record }; +} + +// ── Health endpoint ────────────────────────────────────────────────────────── + +describe('GET /api/v1/health', () => { + test('returns 200 with ok status', async () => { + const { status, body } = await getJson('/api/v1/health'); + + expect(status).toBe(200); + expect(body.status).toBe('ok'); + expect(body.version).toBe('0.1.0'); + expect(typeof body.uptime_seconds).toBe('number'); + }); +}); + +// ── Solvers endpoint ───────────────────────────────────────────────────────── + +describe('GET /api/v1/solvers', () => { + test('returns list of known solvers', async () => { + const { status, body } = await getJson('/api/v1/solvers'); + + expect(status).toBe(200); + const solvers = body.solvers as Array>; + expect(Array.isArray(solvers)).toBe(true); + expect(solvers.length).toBeGreaterThanOrEqual(5); + + // Check known solver IDs are present + const ids = solvers.map((s) => s.id as string); + expect(ids).toContain('reference'); + expect(ids).toContain('weak'); + expect(ids).toContain('buggy-A'); + expect(ids).toContain('buggy-B'); + expect(ids).toContain('buggy-C'); + }); +}); + +// ── Submission endpoint ──────────────────────────────────────────────────── + +describe('POST /api/v1/submissions', () => { + test('creates a submission with valid solver_id', async () => { + const { status, body } = await postJson('/api/v1/submissions', { + solver_id: 'reference', + }); + + expect(status).toBe(201); + expect(body.submission_id).toBeDefined(); + expect(body.solver_id).toBe('reference'); + expect(body.track).toBe('non_llm'); + expect(body.status).toBe('pending'); + }); + + test('creates a submission with explicit track', async () => { + const { status, body } = await postJson('/api/v1/submissions', { + solver_id: 'reference', + track: 'model_fixed', + }); + + expect(status).toBe(201); + expect(body.track).toBe('model_fixed'); + }); + + test('rejects missing solver_id', async () => { + const { status, body } = await postJson('/api/v1/submissions', {}); + + expect(status).toBe(400); + expect(body.error).toBeDefined(); + }); + + test('rejects empty solver_id', async () => { + const { status, body } = await postJson('/api/v1/submissions', { + solver_id: '', + }); + + expect(status).toBe(400); + expect(body.error).toBeDefined(); + }); + + test('rejects invalid JSON', async () => { + const res = await fetch(`${BASE}/api/v1/submissions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: 'not json', + }); + + expect(res.status).toBe(400); + const body = (await res.json()) as Record; + expect(body.error).toBe('Invalid JSON body'); + }); +}); + +// ── Get submission endpoint ───────────────────────────────────────────────── + +describe('GET /api/v1/submissions/:id', () => { + test('returns submission by id', async () => { + const created = await postJson('/api/v1/submissions', { + solver_id: 'reference', + }); + const id = created.body.submission_id as string; + + const { status, body } = await getJson( + `/api/v1/submissions/${id}`, + ); + + expect(status).toBe(200); + expect(body.submission_id).toBe(id); + expect(body.solver_id).toBe('reference'); + }); + + test('returns 404 for unknown submission', async () => { + const { status, body } = await getJson( + '/api/v1/submissions/nonexistent', + ); + + expect(status).toBe(404); + expect(body.error).toBe('Submission not found'); + }); +}); + +// ── Evaluation endpoint ───────────────────────────────────────────────────── + +describe('POST /api/v1/submissions/:id/evaluate', () => { + test('triggers evaluation and returns 202', async () => { + // Create submission first + const created = await postJson('/api/v1/submissions', { + solver_id: 'reference', + }); + const submissionId = created.body.submission_id as string; + + // Trigger evaluation + const { status, body } = await postJson( + `/api/v1/submissions/${submissionId}/evaluate`, + { task_count: 5, root_seed: 'test-seed-123' }, + ); + + expect(status).toBe(202); + expect(body.evaluation_id).toBeDefined(); + expect(body.submission_id).toBe(submissionId); + expect(body.solver_id).toBe('reference'); + expect(body.status).toBe('pending'); + expect(body.task_count).toBe(5); + expect(body.family).toBe('json_transform.normalize.v0'); + }); + + test('returns 404 for unknown submission', async () => { + const { status, body } = await postJson( + '/api/v1/submissions/nonexistent/evaluate', + { task_count: 5 }, + ); + + expect(status).toBe(404); + expect(body.error).toBe('Submission not found'); + }); + + test('uses default values when body is empty', async () => { + const created = await postJson('/api/v1/submissions', { + solver_id: 'weak', + }); + const submissionId = created.body.submission_id as string; + + const { status, body } = await postJson( + `/api/v1/submissions/${submissionId}/evaluate`, + ); + + expect(status).toBe(202); + expect(body.task_count).toBe(20); + expect(body.family).toBe('json_transform.normalize.v0'); + }); +}); + +// ── Get evaluation endpoint ──────────────────────────────────────────────── + +describe('GET /api/v1/evaluations/:id', () => { + test('returns evaluation results after completion', async () => { + // Create and evaluate + const created = await postJson('/api/v1/submissions', { + solver_id: 'reference', + }); + const submissionId = created.body.submission_id as string; + const evalResp = await postJson( + `/api/v1/submissions/${submissionId}/evaluate`, + { task_count: 5, root_seed: 'eval-test-seed' }, + ); + const evaluationId = evalResp.body.evaluation_id as string; + + // Wait for async evaluation to complete + await Bun.sleep(1000); + + const { status, body } = await getJson( + `/api/v1/evaluations/${evaluationId}`, + ); + + expect(status).toBe(200); + expect(body.evaluation_id).toBe(evaluationId); + expect(body.solver_id).toBe('reference'); + expect(body.status).toBe('completed'); + expect(body.total).toBeGreaterThan(0); + // Reference solver should pass all tasks + expect(body.passed).toBe(body.total); + }); + + test('returns 404 for unknown evaluation', async () => { + const { status, body } = await getJson( + '/api/v1/evaluations/nonexistent', + ); + + expect(status).toBe(404); + expect(body.error).toBe('Evaluation not found'); + }); +}); + +// ── Leaderboard endpoint ──────────────────────────────────────────────────── + +describe('GET /api/v1/leaderboard', () => { + test('returns empty leaderboard initially', async () => { + const { status, body } = await getJson('/api/v1/leaderboard'); + + expect(status).toBe(200); + expect(body.track).toBe('non_llm'); + expect(body.family).toBe('json_transform.normalize.v0'); + const entries = body.entries as unknown[]; + expect(Array.isArray(entries)).toBe(true); + }); + + test('returns ranked entries after evaluations complete', async () => { + // Use a unique store for this test + const testStore = new ArenaStore(); + const testArena = createArenaServer({ + port: 0, + store: testStore, + }); + const testBase = testArena.url; + + // Submit and evaluate reference solver + const refSub = await ( + await fetch(`${testBase}/api/v1/submissions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ solver_id: 'reference' }), + }) + ).json(); + const refEval = await ( + await fetch( + `${testBase}/api/v1/submissions/${(refSub as Record).submission_id}/evaluate`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + task_count: 5, + root_seed: 'leaderboard-test-ref', + }), + }, + ) + ).json(); + + // Submit and evaluate weak solver + const weakSub = await ( + await fetch(`${testBase}/api/v1/submissions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ solver_id: 'weak' }), + }) + ).json(); + const weakEval = await ( + await fetch( + `${testBase}/api/v1/submissions/${(weakSub as Record).submission_id}/evaluate`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + task_count: 5, + root_seed: 'leaderboard-test-weak', + }), + }, + ) + ).json(); + + // Wait for async evaluations + await Bun.sleep(1000); + + const lbRes = await fetch(`${testBase}/api/v1/leaderboard`); + const lbBody = (await lbRes.json()) as Record; + + expect(lbRes.status).toBe(200); + const entries = lbBody.entries as Array>; + expect(entries.length).toBe(2); + + // Reference should rank higher than weak + const refEntry = entries.find( + (e) => e.solver_id === 'reference', + ); + const weakEntry = entries.find( + (e) => e.solver_id === 'weak', + ); + expect(refEntry).toBeDefined(); + expect(weakEntry).toBeDefined(); + expect(refEntry!.rank as number).toBeLessThan(weakEntry!.rank as number); + + testArena.stop(); + }); + + test('filters by track and family query params', async () => { + const { status, body } = await getJson( + '/api/v1/leaderboard?track=model_fixed&family=json_transform.normalize.v0', + ); + + expect(status).toBe(200); + expect(body.track).toBe('model_fixed'); + expect(body.family).toBe('json_transform.normalize.v0'); + }); +}); + +// ── 404 for unknown routes ─────────────────────────────────────────────────── + +describe('unknown routes', () => { + test('returns 404 with error', async () => { + const { status, body } = await getJson('/api/v1/nonexistent'); + + expect(status).toBe(404); + expect(body.error).toBe('Not Found'); + }); +}); + +// ── ArenaStore unit tests ─────────────────────────────────────────────────── + +describe('ArenaStore', () => { + test('submissions and evaluations are isolated', () => { + const store = new ArenaStore(); + + expect(store.getSubmission('nope')).toBeUndefined(); + expect(store.getEvaluation('nope')).toBeUndefined(); + + // Adding and retrieving works + store.addSubmission({ + submission_id: 'sub-1', + solver_id: 'reference', + track: 'non_llm', + status: 'pending', + created_at: '2026-01-01T00:00:00Z', + }); + expect(store.getSubmission('sub-1')?.solver_id).toBe('reference'); + + store.addEvaluation({ + evaluation_id: 'eval-1', + submission_id: 'sub-1', + solver_id: 'reference', + track: 'non_llm', + family: 'json_transform.normalize.v0', + task_count: 10, + status: 'completed', + passed: 8, + failed: 2, + errors: 0, + total: 10, + records: [], + created_at: '2026-01-01T00:00:00Z', + completed_at: '2026-01-01T00:01:00Z', + }); + expect(store.getEvaluation('eval-1')?.passed).toBe(8); + }); + + test('updateEvaluation merges partial updates', () => { + const store = new ArenaStore(); + + store.addEvaluation({ + evaluation_id: 'eval-2', + submission_id: 'sub-2', + solver_id: 'weak', + track: 'non_llm', + family: 'json_transform.normalize.v0', + task_count: 10, + status: 'pending', + passed: 0, + failed: 0, + errors: 0, + total: 0, + records: [], + created_at: '2026-01-01T00:00:00Z', + }); + + store.updateEvaluation('eval-2', { + status: 'completed', + passed: 3, + failed: 7, + total: 10, + }); + + const eval_ = store.getEvaluation('eval-2'); + expect(eval_?.status).toBe('completed'); + expect(eval_?.passed).toBe(3); + expect(eval_?.total).toBe(10); + // Unchanged fields preserved + expect(eval_?.solver_id).toBe('weak'); + }); + + test('getLeaderboard aggregates completed evaluations', () => { + const store = new ArenaStore(); + + // Two evaluations for the same solver + store.addEvaluation({ + evaluation_id: 'e1', + submission_id: 's1', + solver_id: 'reference', + track: 'non_llm', + family: 'json_transform.normalize.v0', + task_count: 10, + status: 'completed', + passed: 10, + failed: 0, + errors: 0, + total: 10, + records: [], + created_at: '2026-01-01T00:00:00Z', + completed_at: '2026-01-01T00:01:00Z', + }); + store.addEvaluation({ + evaluation_id: 'e2', + submission_id: 's2', + solver_id: 'reference', + track: 'non_llm', + family: 'json_transform.normalize.v0', + task_count: 5, + status: 'completed', + passed: 4, + failed: 1, + errors: 0, + total: 5, + records: [], + created_at: '2026-01-01T00:00:00Z', + completed_at: '2026-01-01T00:02:00Z', + }); + // A weak solver + store.addEvaluation({ + evaluation_id: 'e3', + submission_id: 's3', + solver_id: 'weak', + track: 'non_llm', + family: 'json_transform.normalize.v0', + task_count: 10, + status: 'completed', + passed: 0, + failed: 10, + errors: 0, + total: 10, + records: [], + created_at: '2026-01-01T00:00:00Z', + completed_at: '2026-01-01T00:01:00Z', + }); + // Pending evaluation should be excluded + store.addEvaluation({ + evaluation_id: 'e4', + submission_id: 's4', + solver_id: 'buggy-A', + track: 'non_llm', + family: 'json_transform.normalize.v0', + task_count: 10, + status: 'pending', + passed: 0, + failed: 0, + errors: 0, + total: 0, + records: [], + created_at: '2026-01-01T00:00:00Z', + }); + + const board = store.getLeaderboard('non_llm', 'json_transform.normalize.v0'); + + expect(board.length).toBe(2); + expect(board[0].solver_id).toBe('reference'); + expect(board[0].rank).toBe(1); + expect(board[0].total_tasks).toBe(15); + expect(board[0].passed).toBe(14); + expect(board[0].pass_rate).toBeCloseTo(14 / 15); + + expect(board[1].solver_id).toBe('weak'); + expect(board[1].rank).toBe(2); + expect(board[1].pass_rate).toBe(0); + }); + + test('getLeaderboard filters by track and family', () => { + const store = new ArenaStore(); + + store.addEvaluation({ + evaluation_id: 'e1', + submission_id: 's1', + solver_id: 'reference', + track: 'non_llm', + family: 'json_transform.normalize.v0', + task_count: 10, + status: 'completed', + passed: 10, + failed: 0, + errors: 0, + total: 10, + records: [], + created_at: '2026-01-01T00:00:00Z', + completed_at: '2026-01-01T00:01:00Z', + }); + store.addEvaluation({ + evaluation_id: 'e2', + submission_id: 's2', + solver_id: 'reference', + track: 'model_fixed', + family: 'json_transform.normalize.v0', + task_count: 10, + status: 'completed', + passed: 8, + failed: 2, + errors: 0, + total: 10, + records: [], + created_at: '2026-01-01T00:00:00Z', + completed_at: '2026-01-01T00:02:00Z', + }); + + const nonLlm = store.getLeaderboard('non_llm', 'json_transform.normalize.v0'); + expect(nonLlm.length).toBe(1); + expect(nonLlm[0].passed).toBe(10); + + const modelFixed = store.getLeaderboard('model_fixed', 'json_transform.normalize.v0'); + expect(modelFixed.length).toBe(1); + expect(modelFixed[0].passed).toBe(8); + }); +}); diff --git a/arena/src/index.ts b/arena/src/index.ts new file mode 100644 index 0000000..18c5fac --- /dev/null +++ b/arena/src/index.ts @@ -0,0 +1,55 @@ +/** + * @module @fresharena/arena + * + * FreshArena hosted arena service — accepts solver submissions via API, + * runs evaluations against generated tasks, and publishes a ranked leaderboard. + * + * ## Quick start + * + * ```ts + * import { createArenaServer } from '@fresharena/arena'; + * + * const arena = createArenaServer({ port: 3210 }); + * console.log(`Arena listening at ${arena.url}`); + * ``` + * + * ## API endpoints + * + * | Method | Path | Description | + * |--------|------------------------------|--------------------------| + * | GET | /api/v1/health | Health check | + * | GET | /api/v1/solvers | List known solvers | + * | POST | /api/v1/submissions | Submit a solver | + * | GET | /api/v1/submissions/:id | Get submission status | + * | POST | /api/v1/submissions/:id/evaluate | Trigger evaluation | + * | GET | /api/v1/evaluations/:id | Get evaluation results | + * | GET | /api/v1/leaderboard | Ranked solver scores | + * + * ## Evaluation pipeline + * + * 1. Generate fresh tasks via `@fresharena/core/generator` + * 2. Execute the submitted solver on each task + * 3. Verify outputs via the deterministic verifier oracle + * 4. Build per-task FAEP audit records + * 5. Aggregate into leaderboard scores + */ + +export { createArenaServer } from './server.js'; +export type { ArenaServer, ArenaServerOptions } from './server.js'; +export { ArenaStore } from './store.js'; +export { runEvaluation } from './evaluator.js'; +export type { + EvaluateRequest, + EvaluationResponse, + EvaluationStatus, + HealthResponse, + LeaderboardEntry, + LeaderboardResponse, + SolversListResponse, + SubmissionRequest, + SubmissionResponse, +} from './types.js'; +export { + EvaluateRequestSchema, + SubmissionRequestSchema, +} from './types.js'; diff --git a/arena/src/server.ts b/arena/src/server.ts new file mode 100644 index 0000000..72c876b --- /dev/null +++ b/arena/src/server.ts @@ -0,0 +1,282 @@ +import type { + EvaluateRequest, + EvaluationResponse, + HealthResponse, + LeaderboardResponse, + SolversListResponse, + SubmissionRequest, + SubmissionResponse, +} from './types.js'; +import { + EvaluateRequestSchema, + SubmissionRequestSchema, +} from './types.js'; +import { ArenaStore } from './store.js'; +import { runEvaluation } from './evaluator.js'; + +const API_PREFIX = '/api/v1'; +const ARENA_VERSION = '0.1.0'; + +export interface ArenaServerOptions { + port?: number; + hostname?: string; + store?: ArenaStore; +} + +export interface ArenaServer { + readonly url: string; + readonly port: number; + readonly store: ArenaStore; + stop(): void; +} + +/** + * Create and start the FreshArena hosted arena HTTP service. + * + * Uses Bun's built-in HTTP server (zero external dependencies). + * All routes are under `/api/v1/`. + */ +export function createArenaServer( + options: ArenaServerOptions = {}, +): ArenaServer { + const port = options.port ?? 3210; + const hostname = options.hostname ?? '0.0.0.0'; + const store = options.store ?? new ArenaStore(); + const startTime = Date.now(); + + async function handleRequest(req: Request): Promise { + const url = new URL(req.url); + const path = url.pathname; + const method = req.method; + + // GET /api/v1/health + if (method === 'GET' && path === `${API_PREFIX}/health`) { + return json({ + status: 'ok', + version: ARENA_VERSION, + uptime_seconds: Math.floor((Date.now() - startTime) / 1000), + }); + } + + // GET /api/v1/solvers + if (method === 'GET' && path === `${API_PREFIX}/solvers`) { + return listKnownSolvers(); + } + + // POST /api/v1/submissions — accept a new solver submission + if (method === 'POST' && path === `${API_PREFIX}/submissions`) { + return handleCreateSubmission(req); + } + + // GET /api/v1/submissions/:id — retrieve submission status + if ( + method === 'GET' && + path.startsWith(`${API_PREFIX}/submissions/`) + ) { + const id = path.slice(`${API_PREFIX}/submissions/`.length); + if (!id.includes('/')) { + return handleGetSubmission(id); + } + } + + // POST /api/v1/submissions/:id/evaluate — trigger evaluation + if ( + method === 'POST' && + path.startsWith(`${API_PREFIX}/submissions/`) + ) { + const suffix = path.slice(`${API_PREFIX}/submissions/`.length); + const match = suffix.match(/^([^/]+)\/evaluate$/); + if (match !== null) { + return handleStartEvaluation(match[1], req); + } + } + + // GET /api/v1/evaluations/:id — retrieve evaluation result + if ( + method === 'GET' && + path.startsWith(`${API_PREFIX}/evaluations/`) + ) { + const id = path.slice(`${API_PREFIX}/evaluations/`.length); + return handleGetEvaluation(id); + } + + // GET /api/v1/leaderboard — ranked solver scores + if (method === 'GET' && path === `${API_PREFIX}/leaderboard`) { + const track = + url.searchParams.get('track') ?? 'non_llm'; + const family = + url.searchParams.get('family') ?? + 'json_transform.normalize.v0'; + return handleLeaderboard(track, family); + } + + return json({ error: 'Not Found', path }, 404); + } + + // ── Route handlers ────────────────────────────────────────────────────── + + async function handleCreateSubmission( + req: Request, + ): Promise { + let body: unknown; + try { + body = await req.json(); + } catch { + return json({ error: 'Invalid JSON body' }, 400); + } + + const parsed = SubmissionRequestSchema.safeParse(body); + if (!parsed.success) { + return json( + { error: 'Validation failed', details: parsed.error.issues }, + 400, + ); + } + + const data = parsed.data; + const submission: SubmissionResponse = { + submission_id: crypto.randomUUID(), + solver_id: data.solver_id, + track: data.track, + status: 'pending', + created_at: new Date().toISOString(), + }; + + store.addSubmission(submission); + return json(submission, 201); + } + + function handleGetSubmission(id: string): Response { + const submission = store.getSubmission(id); + if (submission === undefined) { + return json({ error: 'Submission not found' }, 404); + } + return json(submission); + } + + async function handleStartEvaluation( + submissionId: string, + req: Request, + ): Promise { + const submission = store.getSubmission(submissionId); + if (submission === undefined) { + return json({ error: 'Submission not found' }, 404); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + body = {}; + } + + const parsed = EvaluateRequestSchema.safeParse(body); + if (!parsed.success) { + return json( + { error: 'Validation failed', details: parsed.error.issues }, + 400, + ); + } + + const data = parsed.data; + const evaluationId = crypto.randomUUID(); + + const evaluation: EvaluationResponse = { + evaluation_id: evaluationId, + submission_id: submissionId, + solver_id: submission.solver_id, + track: submission.track, + family: data.family, + task_count: data.task_count, + status: 'pending', + passed: 0, + failed: 0, + errors: 0, + total: 0, + records: [], + created_at: new Date().toISOString(), + }; + + store.addEvaluation(evaluation); + + // Fire-and-forget: evaluation runs asynchronously + void runEvaluation({ + evaluationId, + submissionId, + solverId: submission.solver_id, + track: submission.track, + family: data.family, + taskCount: data.task_count, + rootSeed: data.root_seed, + store, + }); + + return json(evaluation, 202); + } + + function handleGetEvaluation(id: string): Response { + const evaluation = store.getEvaluation(id); + if (evaluation === undefined) { + return json({ error: 'Evaluation not found' }, 404); + } + return json(evaluation); + } + + function handleLeaderboard( + track: string, + family: string, + ): Response { + const entries = store.getLeaderboard(track, family); + return json({ + track, + family, + entries, + generated_at: new Date().toISOString(), + }); + } + + async function listKnownSolvers(): Promise { + try { + const { listSolvers } = await import( + '@fresharena/core/solvers' + ); + const solvers = listSolvers(); + return json({ + solvers: solvers.map((s: { id: string; track: string; description: string }) => ({ + id: s.id, + track: s.track, + description: s.description, + })), + }); + } catch { + return json({ solvers: [] }); + } + } + + // ── Start server ──────────────────────────────────────────────────────── + + const server = Bun.serve({ + port, + hostname, + fetch: handleRequest, + }); + + return { + url: server.url.toString(), + port: server.port as number, + store, + stop: () => server.stop(), + }; +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +function json(data: T, status = 200): Response { + return new Response(JSON.stringify(data), { + status, + headers: { + 'Content-Type': 'application/json', + 'X-Powered-By': 'FreshArena', + }, + }); +} diff --git a/arena/src/store.ts b/arena/src/store.ts new file mode 100644 index 0000000..44d5572 --- /dev/null +++ b/arena/src/store.ts @@ -0,0 +1,99 @@ +import type { + EvaluationResponse, + LeaderboardEntry, + SubmissionResponse, +} from './types.js'; + +/** + * In-memory store for arena state: submissions, evaluations, and leaderboard. + * + * Phase 4 MVP uses an in-process Map. A production arena would swap this for + * a persistent store (SQLite, Postgres) behind the same interface. + */ +export class ArenaStore { + private readonly submissions = new Map(); + private readonly evaluations = new Map(); + + addSubmission(submission: SubmissionResponse): void { + this.submissions.set(submission.submission_id, submission); + } + + getSubmission(id: string): SubmissionResponse | undefined { + return this.submissions.get(id); + } + + addEvaluation(evaluation: EvaluationResponse): void { + this.evaluations.set(evaluation.evaluation_id, evaluation); + } + + getEvaluation(id: string): EvaluationResponse | undefined { + return this.evaluations.get(id); + } + + updateEvaluation( + id: string, + update: Partial, + ): void { + const existing = this.evaluations.get(id); + if (existing !== undefined) { + this.evaluations.set(id, { ...existing, ...update } as EvaluationResponse); + } + } + + /** + * Aggregate completed evaluations into a ranked leaderboard. + * Sorted by pass rate descending; ties broken by total tasks descending. + */ + getLeaderboard(track: string, family: string): LeaderboardEntry[] { + const solverStats = new Map< + string, + { total: number; passed: number; lastAt: string } + >(); + + for (const evaluation of this.evaluations.values()) { + if (evaluation.status !== 'completed') continue; + if (evaluation.track !== track) continue; + if (evaluation.family !== family) continue; + + const existing = solverStats.get(evaluation.solver_id) ?? { + total: 0, + passed: 0, + lastAt: '', + }; + existing.total += evaluation.total; + existing.passed += evaluation.passed; + if ( + evaluation.completed_at !== undefined && + evaluation.completed_at > existing.lastAt + ) { + existing.lastAt = evaluation.completed_at; + } + solverStats.set(evaluation.solver_id, existing); + } + + const entries: LeaderboardEntry[] = []; + for (const [solverId, stats] of solverStats) { + entries.push({ + solver_id: solverId, + track, + family, + total_tasks: stats.total, + passed: stats.passed, + pass_rate: stats.total > 0 ? stats.passed / stats.total : 0, + rank: 0, + last_evaluated_at: stats.lastAt, + }); + } + + entries.sort( + (a, b) => + b.pass_rate - a.pass_rate || b.total_tasks - a.total_tasks, + ); + + for (let i = 0; i < entries.length; i++) { + entries[i] = { ...entries[i], rank: i + 1 }; + } + + return entries; + } +} diff --git a/arena/src/types.ts b/arena/src/types.ts new file mode 100644 index 0000000..0223b32 --- /dev/null +++ b/arena/src/types.ts @@ -0,0 +1,91 @@ +import { z } from 'zod'; +import { EvalTrackSchema } from '@fresharena/faep-schema'; + +// ─── Request schemas (Zod-validated at API boundary) ───────────────────────── + +/** POST /api/v1/submissions request body. */ +export const SubmissionRequestSchema = z.object({ + solver_id: z.string().min(1).max(100), + track: EvalTrackSchema.optional().default('non_llm'), +}); +export type SubmissionRequest = z.infer; + +/** POST /api/v1/submissions/:id/evaluate request body. */ +export const EvaluateRequestSchema = z.object({ + task_count: z.number().int().min(1).max(100).optional().default(20), + root_seed: z.string().min(1).optional().default('arena-default-seed'), + family: z + .literal('json_transform.normalize.v0') + .optional() + .default('json_transform.normalize.v0'), +}); +export type EvaluateRequest = z.infer; + +// ─── Response types ────────────────────────────────────────────────────────── + +/** POST /api/v1/submissions response. */ +export interface SubmissionResponse { + submission_id: string; + solver_id: string; + track: string; + status: 'pending'; + created_at: string; +} + +/** Evaluation lifecycle status. */ +export type EvaluationStatus = 'pending' | 'running' | 'completed' | 'error'; + +/** GET /api/v1/evaluations/:id response. */ +export interface EvaluationResponse { + evaluation_id: string; + submission_id: string; + solver_id: string; + track: string; + family: string; + task_count: number; + status: EvaluationStatus; + passed: number; + failed: number; + errors: number; + total: number; + records: ReadonlyArray>; + created_at: string; + completed_at?: string; + error?: string; +} + +/** Single entry in the leaderboard. */ +export interface LeaderboardEntry { + solver_id: string; + track: string; + family: string; + total_tasks: number; + passed: number; + pass_rate: number; + rank: number; + last_evaluated_at: string; +} + +/** GET /api/v1/leaderboard response. */ +export interface LeaderboardResponse { + track: string; + family: string; + entries: ReadonlyArray; + generated_at: string; +} + +/** GET /api/v1/health response. */ +export interface HealthResponse { + status: 'ok'; + version: string; + uptime_seconds: number; +} + +/** GET /api/v1/solvers response. */ +export interface SolversListResponse { + solvers: ReadonlyArray<{ + id: string; + track: string; + description: string; + }>; +} diff --git a/arena/tsconfig.json b/arena/tsconfig.json new file mode 100644 index 0000000..c17b043 --- /dev/null +++ b/arena/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src"] +} diff --git a/bun.lock b/bun.lock index 4fa759b..b417c37 100644 --- a/bun.lock +++ b/bun.lock @@ -12,6 +12,16 @@ "typescript": "^5.5.0", }, }, + "arena": { + "name": "@fresharena/arena", + "version": "0.1.0", + "dependencies": { + "@fresharena/core": "workspace:*", + "@fresharena/faep-schema": "workspace:*", + "@fresharena/verifier-runtime": "workspace:*", + "zod": "^3.23.0", + }, + }, "packages/cli": { "name": "@fresharena/cli", "version": "0.1.0", @@ -135,6 +145,8 @@ "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@1.9.4", "", { "os": "win32", "cpu": "x64" }, "sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA=="], + "@fresharena/arena": ["@fresharena/arena@workspace:arena"], + "@fresharena/cli": ["@fresharena/cli@workspace:packages/cli"], "@fresharena/core": ["@fresharena/core@workspace:packages/core"], diff --git a/package.json b/package.json index aa3d25a..b1a717d 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "packageManager": "bun@1.3.14", "workspaces": [ "packages/*", + "arena", "solvers/non-llm/*", "solvers/llm/*" ], diff --git a/packages/core/package.json b/packages/core/package.json index c796278..9043655 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -30,6 +30,14 @@ "./replay": { "import": "./dist/replay/index.js", "types": "./dist/replay/index.d.ts" + }, + "./solvers": { + "import": "./dist/solvers/index.js", + "types": "./dist/solvers/index.d.ts" + }, + "./rng": { + "import": "./dist/rng.js", + "types": "./dist/rng.d.ts" } }, "files": ["dist"], From 059c18eedd266befcd8c5e838318ec038c0a1662 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sun, 19 Jul 2026 19:15:05 +0800 Subject: [PATCH 2/2] Fix #117: address review blockers in arena service - store.ts: add clear() method for proper test isolation - server.ts: add defensive .catch() on fire-and-forget evaluation - index.test.ts: fix no-op afterEach to actually clear store; add test for unknown solver_id returning error status Co-Authored-By: Claude --- arena/src/index.test.ts | 33 +++++++++++++++++++++++++++------ arena/src/server.ts | 8 +++++++- arena/src/store.ts | 5 +++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/arena/src/index.test.ts b/arena/src/index.test.ts index 6d373e2..b2ffc1e 100644 --- a/arena/src/index.test.ts +++ b/arena/src/index.test.ts @@ -12,12 +12,7 @@ beforeAll(() => { }); afterEach(() => { - // Clear store between tests for isolation - for (const key of [ - ...arena.store.getEvaluation('x') ? ['x'] : [], - ]) { - // Store only exposes get methods; tests create fresh submissions - } + arena.store.clear(); }); // ── Helpers ────────────────────────────────────────────────────────────────── @@ -217,6 +212,32 @@ describe('POST /api/v1/submissions/:id/evaluate', () => { expect(body.task_count).toBe(20); expect(body.family).toBe('json_transform.normalize.v0'); }); + + test('evaluation with unknown solver_id completes with error status', async () => { + const created = await postJson('/api/v1/submissions', { + solver_id: 'nonexistent-solver', + }); + const submissionId = created.body.submission_id as string; + + const { status, body } = await postJson( + `/api/v1/submissions/${submissionId}/evaluate`, + { task_count: 1, root_seed: 'unknown-solver-test' }, + ); + + expect(status).toBe(202); + const evaluationId = body.evaluation_id as string; + + // Wait for async evaluation to finish + await Bun.sleep(500); + + const { status: evalStatus, body: evalBody } = await getJson( + `/api/v1/evaluations/${evaluationId}`, + ); + + expect(evalStatus).toBe(200); + expect(evalBody.status).toBe('error'); + expect(evalBody.error).toBeDefined(); + }); }); // ── Get evaluation endpoint ──────────────────────────────────────────────── diff --git a/arena/src/server.ts b/arena/src/server.ts index 72c876b..b3541e2 100644 --- a/arena/src/server.ts +++ b/arena/src/server.ts @@ -199,7 +199,9 @@ export function createArenaServer( store.addEvaluation(evaluation); - // Fire-and-forget: evaluation runs asynchronously + // Evaluation runs asynchronously; errors are caught inside runEvaluation + // and persisted to the store as status='error'. The .catch() is a + // defensive guard against any unexpected unhandled rejection. void runEvaluation({ evaluationId, submissionId, @@ -209,6 +211,10 @@ export function createArenaServer( taskCount: data.task_count, rootSeed: data.root_seed, store, + }).catch(() => { + // runEvaluation already updates the store with status='error'; + // this catch only prevents an unhandled rejection if the inner + // catch itself fails (e.g. store.updateEvaluation throws). }); return json(evaluation, 202); diff --git a/arena/src/store.ts b/arena/src/store.ts index 44d5572..443252f 100644 --- a/arena/src/store.ts +++ b/arena/src/store.ts @@ -14,6 +14,11 @@ export class ArenaStore { private readonly submissions = new Map(); private readonly evaluations = new Map(); + clear(): void { + this.submissions.clear(); + this.evaluations.clear(); + } + addSubmission(submission: SubmissionResponse): void { this.submissions.set(submission.submission_id, submission); }