From a9ff3a38c4973e1664df8ceec3e179eb14845b4c Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 19 Jun 2026 04:37:56 +0200 Subject: [PATCH] fix(dashboard): enforce configured run sources --- apps/cli/src/commands/results/serve.ts | 49 ++++--- apps/cli/test/commands/results/serve.test.ts | 132 ++++++++++++++++-- .../src/content/docs/docs/tools/dashboard.mdx | 8 +- 3 files changed, 157 insertions(+), 32 deletions(-) diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 5355f6b84..2511d6cf1 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -29,7 +29,7 @@ * mismatches without prompting, self-updating, or blocking startup. * * Exported functions (for testing): - * - resolveSourceFile(source, cwd) — resolves a run manifest path + * - resolveSourceFile(source, cwd) — resolves the canonical project run source * - loadResults(content) — parses JSONL into EvaluationResult[] * - createApp(results, cwd) — Hono app factory */ @@ -55,7 +55,6 @@ import { Hono } from 'hono'; import { enforceRequiredVersion } from '../../version-check.js'; import { parseJsonlResults } from '../eval/artifact-writer.js'; -import { resolveRunManifestPath } from '../eval/result-layout.js'; import { loadRunCache, resolveRunCacheFile } from '../eval/run-cache.js'; import { findRepoRoot } from '../eval/shared.js'; import { listResultFiles } from '../inspect/utils.js'; @@ -67,12 +66,7 @@ import { } from './combine-run.js'; import { deleteLocalRun } from './delete-run.js'; import { getActiveRunStatus, getActiveRunTarget, registerEvalRoutes } from './eval-runner.js'; -import { - loadLightweightResults, - loadManifestResults, - parseResultManifest, - resolveResultSourcePath, -} from './manifest.js'; +import { loadLightweightResults, loadManifestResults, parseResultManifest } from './manifest.js'; import { type SourcedResultFileMeta, clearRemoteRunTags, @@ -90,17 +84,34 @@ import { type StudioConfig, loadStudioConfig, saveStudioConfig } from './studio- // ── Source resolution ──────────────────────────────────────────────────── /** - * Resolve a run manifest path from an explicit source, run cache, - * or directory scan. Throws if no run workspace can be found. + * Dashboard has one run source per project: the project's + * `.agentv/results/runs/` tree plus any `results:` repo configured for that + * project. Direct run workspaces and index manifests are supported by + * `agentv results report`, not the live Dashboard server. + */ +const DIRECT_DASHBOARD_SOURCE_GUIDANCE = [ + 'Dashboard reads configured project run sources only.', + 'Run it from a project root, or pass --dir so Dashboard uses /.agentv/results/runs/:', + ' agentv dashboard --dir ', + 'To browse external results, configure results.repo_url or results.repo_path in config YAML.', + 'For a one-off run bundle, use: agentv results report ', +].join('\n'); + +function unsupportedDashboardSourceError(source: string, cwd: string): Error { + const resolved = path.isAbsolute(source) ? source : path.resolve(cwd, source); + return new Error( + `Unsupported Dashboard source: ${resolved}\n${DIRECT_DASHBOARD_SOURCE_GUIDANCE}`, + ); +} + +/** + * Resolve a canonical project run manifest path from run cache or directory + * scan. Throws if an unsupported direct source is provided or no run workspace + * can be found. */ export async function resolveSourceFile(source: string | undefined, cwd: string): Promise { if (source) { - let resolved = resolveResultSourcePath(source, cwd); - if (!existsSync(resolved)) { - throw new Error(`Source file not found: ${resolved}`); - } - resolved = resolveRunManifestPath(resolved); - return resolved; + throw unsupportedDashboardSourceError(source, cwd); } // Prefer cache pointer, fall back to directory scan @@ -2036,7 +2047,7 @@ export const resultsServeCommand = command({ type: optional(string), displayName: 'source', description: - 'Run workspace directory or index.jsonl manifest to serve (defaults to most recent in .agentv/results/runs/)', + 'Legacy direct run source (unsupported); use --dir or results: config', }), port: option({ type: optional(number), @@ -2126,8 +2137,8 @@ export const resultsServeCommand = command({ let results: EvaluationResult[] = []; let sourceFile: string | undefined; - // When a source is explicitly provided, it must exist. - // Otherwise, try to auto-discover results; start empty if none found. + // Reject unsupported direct sources. Otherwise, auto-discover the + // project's configured run workspace and fall back to the empty state. if (source) { sourceFile = await resolveSourceFile(source, cwd); results = loadManifestResults(sourceFile); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 05d9aeec9..a3a5b2ae3 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -1,9 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, spyOn } from 'bun:test'; -import { execSync } from 'node:child_process'; +import { execFileSync, execSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import os from 'node:os'; import { tmpdir } from 'node:os'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { addProject, saveProjectRegistry } from '@agentv/core'; @@ -13,6 +14,14 @@ import { resolveDashboardMode, resolveSourceFile, } from '../../../src/commands/results/serve.js'; +import { assertCoreBuild } from '../../setup-core-build.js'; + +assertCoreBuild(); + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const projectRoot = path.resolve(__dirname, '../../../../..'); +const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); // ── Sample JSONL content (snake_case, matching on-disk format) ────────── @@ -250,24 +259,129 @@ function writeLocalRunArtifact( return `${experiment}::${timestamp}`; } +function writeWtgDogfoodNoncanonicalArtifact(baseDir: string): { + runDir: string; + indexPath: string; +} { + const runDir = path.join(baseDir, 'wtg-dogfood-noncanonical-run'); + mkdirSync(runDir, { recursive: true }); + const indexPath = path.join(runDir, 'index.jsonl'); + writeFileSync(indexPath, toJsonl({ ...RESULT_A, test_id: 'wtg-dogfood-noncanonical' })); + return { runDir, indexPath }; +} + +function runDashboardExpectFailure(args: string[], env: Record) { + try { + execFileSync(process.execPath, [CLI_ENTRY, 'dashboard', ...args], { + cwd: projectRoot, + encoding: 'utf8', + env: { + ...cleanGitEnv(), + ...env, + }, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 5000, + }); + throw new Error('Expected dashboard command to fail'); + } catch (error) { + const result = error as { + status?: number; + signal?: NodeJS.Signals | null; + stdout?: string | Buffer; + stderr?: string | Buffer; + }; + return { + status: result.status, + signal: result.signal, + stdout: String(result.stdout ?? ''), + stderr: String(result.stderr ?? ''), + }; + } +} + // ── resolveSourceFile ──────────────────────────────────────────────────── describe('resolveSourceFile', () => { - it('throws for nonexistent file', async () => { - await expect(resolveSourceFile('/tmp/does-not-exist.jsonl', '/tmp')).rejects.toThrow( - 'Source file not found', + it('rejects direct WTG dogfood run directories with setup guidance', async () => { + const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-serve-source-')); + const { runDir } = writeWtgDogfoodNoncanonicalArtifact(tempDir); + + await expect(resolveSourceFile(runDir, tempDir)).rejects.toThrow( + 'Dashboard reads configured project run sources only', + ); + + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('rejects direct WTG dogfood index.jsonl manifests with setup guidance', async () => { + const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-serve-source-')); + const { indexPath } = writeWtgDogfoodNoncanonicalArtifact(tempDir); + + await expect(resolveSourceFile(indexPath, tempDir)).rejects.toThrow( + 'For a one-off run bundle, use: agentv results report', ); + + rmSync(tempDir, { recursive: true, force: true }); }); - it('rejects legacy flat result files', async () => { + it('still auto-discovers canonical project run workspaces when no source is provided', async () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-serve-source-')); - const flatFile = path.join(tempDir, 'results.jsonl'); - writeFileSync(flatFile, toJsonl(RESULT_A)); + const runDir = path.join( + tempDir, + '.agentv', + 'results', + 'runs', + 'default', + '2026-06-17T00-00-00-000Z', + ); + mkdirSync(runDir, { recursive: true }); + const indexPath = path.join(runDir, 'index.jsonl'); + writeFileSync(indexPath, toJsonl(RESULT_A)); + + await expect(resolveSourceFile(undefined, tempDir)).resolves.toBe(indexPath); + + rmSync(tempDir, { recursive: true, force: true }); + }); +}); - await expect(resolveSourceFile(flatFile, tempDir)).rejects.toThrow( - 'Expected a run workspace directory or index.jsonl manifest', +describe('dashboard CLI source contract', () => { + it('fails before serving a WTG dogfood noncanonical run directory', () => { + const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-dashboard-source-cli-')); + const projectDir = path.join(tempDir, 'project'); + mkdirSync(projectDir, { recursive: true }); + const { runDir } = writeWtgDogfoodNoncanonicalArtifact(tempDir); + + const result = runDashboardExpectFailure( + [runDir, '--dir', projectDir, '--single', '--port', '43117'], + { AGENTV_HOME: path.join(tempDir, 'home') }, ); + expect(result.status).toBe(1); + expect(result.signal).toBeNull(); + expect(result.stdout).not.toContain('Serving 1 result(s)'); + expect(result.stderr).toContain('Unsupported Dashboard source'); + expect(result.stderr).toContain('agentv dashboard --dir '); + + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('fails before serving a direct WTG dogfood index.jsonl manifest', () => { + const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-dashboard-source-cli-')); + const projectDir = path.join(tempDir, 'project'); + mkdirSync(projectDir, { recursive: true }); + const { indexPath } = writeWtgDogfoodNoncanonicalArtifact(tempDir); + + const result = runDashboardExpectFailure( + [indexPath, '--dir', projectDir, '--single', '--port', '43118'], + { AGENTV_HOME: path.join(tempDir, 'home') }, + ); + + expect(result.status).toBe(1); + expect(result.signal).toBeNull(); + expect(result.stdout).not.toContain('Serving 1 result(s)'); + expect(result.stderr).toContain('Unsupported Dashboard source'); + expect(result.stderr).toContain('agentv results report '); + rmSync(tempDir, { recursive: true, force: true }); }); }); diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 9a36163af..94f1f2278 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -33,14 +33,14 @@ agentv dashboard Dashboard auto-discovers run workspaces from `.agentv/results/runs/` in the current directory and opens at `http://localhost:3117`. -You can also point it at a specific run workspace or `index.jsonl` manifest: +To open a different project, pass the project root with `--dir`: ```bash -agentv dashboard .agentv/results/runs/2026-03-30T11-45-56-989Z/index.jsonl -# or -agentv dashboard .agentv/results/runs/2026-03-30T11-45-56-989Z +agentv dashboard --dir /path/to/project ``` +Dashboard does not accept a run workspace directory or `index.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/runs/` tree, plus an external results repository or run directory configured under `results:` in YAML. For one-off inspection of a copied run bundle, use `agentv results report `. + ## Options | Option | Description |