From 82c44faf4d798d85a6e224dd94cba4cfe2ca29a7 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 23 Jun 2026 01:20:45 +0200 Subject: [PATCH 1/5] feat(results): move run artifacts under results Bead: av-vqe --- README.md | 2 +- apps/cli/src/commands/eval/artifact-writer.ts | 1 + apps/cli/src/commands/eval/result-layout.ts | 46 ++++- apps/cli/src/commands/eval/run-cache.ts | 2 +- apps/cli/src/commands/eval/run-eval.ts | 31 +-- apps/cli/src/commands/eval/wip-checkpoint.ts | 4 +- apps/cli/src/commands/inspect/filter.ts | 54 +++-- apps/cli/src/commands/inspect/list.ts | 6 +- apps/cli/src/commands/inspect/search.ts | 28 ++- apps/cli/src/commands/inspect/utils.ts | 36 +++- apps/cli/src/commands/pipeline/input.ts | 2 +- apps/cli/src/commands/pipeline/run.ts | 6 +- apps/cli/src/commands/results/combine-run.ts | 41 +++- apps/cli/src/commands/results/combine.ts | 2 +- apps/cli/src/commands/results/delete-run.ts | 22 +-- apps/cli/src/commands/results/eval-runner.ts | 2 +- apps/cli/src/commands/results/export.ts | 2 +- apps/cli/src/commands/results/manifest.ts | 1 + .../src/commands/results/projection-bundle.ts | 7 +- .../src/commands/results/remote-metadata.ts | 5 +- apps/cli/src/commands/results/remote.ts | 22 +-- apps/cli/src/commands/results/run-tags.ts | 8 + apps/cli/src/commands/results/serve.ts | 30 +-- apps/cli/src/commands/results/shared.ts | 4 +- apps/cli/src/commands/results/validate.ts | 17 +- apps/cli/src/commands/trend/index.ts | 4 +- .../commands/eval/artifact-writer.test.ts | 92 ++++----- .../test/commands/eval/result-layout.test.ts | 42 ++++ apps/cli/test/commands/eval/run-cache.test.ts | 15 +- .../cli/test/commands/results/combine.test.ts | 60 +++--- apps/cli/test/commands/results/delete.test.ts | 6 +- apps/cli/test/commands/results/export.test.ts | 8 +- .../results/remote-auto-export.test.ts | 32 +-- .../test/commands/results/run-tags.test.ts | 2 +- apps/cli/test/commands/results/serve.test.ts | 186 ++++++++---------- apps/cli/test/commands/results/shared.test.ts | 16 +- .../test/commands/results/validate.test.ts | 5 +- apps/cli/test/commands/trace/trace.test.ts | 42 +++- apps/cli/test/commands/trend/trend.test.ts | 2 +- apps/cli/test/unit/run-cache.test.ts | 4 +- apps/dashboard/src/components/EvalDetail.tsx | 2 +- .../src/components/TranscriptTimeline.tsx | 4 +- .../__fixtures__/structured-transcript.ts | 2 +- .../src/components/resume-run-helpers.test.ts | 4 +- .../components/transcript-timeline.test.tsx | 4 +- .../__fixtures__/trace-session-read-model.ts | 4 +- .../src/lib/trace-read-model.test.ts | 8 +- .../docs/docs/evaluation/running-evals.mdx | 23 +-- .../docs/docs/getting-started/quickstart.mdx | 2 +- .../content/docs/docs/guides/autoresearch.mdx | 2 +- .../docs/docs/integrations/phoenix.mdx | 2 +- .../src/content/docs/docs/tools/compare.mdx | 18 +- .../src/content/docs/docs/tools/dashboard.mdx | 10 +- .../src/content/docs/docs/tools/import.mdx | 4 +- .../src/content/docs/docs/tools/inspect.mdx | 4 +- .../src/content/docs/docs/tools/prepare.mdx | 4 +- .../src/content/docs/docs/tools/results.mdx | 51 ++--- .../web/src/content/docs/docs/tools/trend.mdx | 14 +- .../docs/docs/tools/wip-checkpoints.mdx | 18 +- examples/features/benchmark-tooling/README.md | 34 ++-- examples/features/compare/README.md | 20 +- examples/features/compare/evals/README.md | 24 +-- .../features/compare/evals/dataset.eval.yaml | 2 +- .../features/document-extraction/README.md | 6 +- .../evals/confusion-metrics.eval.yaml | 2 +- .../scripts/aggregate_metrics.ts | 8 +- examples/features/experiments/README.md | 6 +- .../features/sdk-programmatic-api/README.md | 2 +- .../features/sdk-programmatic-api/evaluate.ts | 4 +- examples/features/trace-analysis/README.md | 10 +- examples/features/trend/README.md | 6 +- examples/red-team/README.md | 2 +- examples/showcase/bug-fix-benchmark/README.md | 4 +- examples/showcase/export-screening/README.md | 4 +- .../showcase/multi-model-benchmark/README.md | 10 +- packages/core/src/evaluation/metrics.ts | 6 +- .../src/evaluation/providers/claude-cli.ts | 7 +- .../src/evaluation/providers/claude-sdk.ts | 7 +- .../core/src/evaluation/providers/claude.ts | 7 +- .../src/evaluation/providers/codex-cli.ts | 7 +- .../core/src/evaluation/providers/codex.ts | 7 +- .../src/evaluation/providers/copilot-cli.ts | 7 +- .../src/evaluation/providers/copilot-sdk.ts | 7 +- .../src/evaluation/providers/log-directory.ts | 31 +++ .../core/src/evaluation/providers/pi-cli.ts | 7 +- .../evaluation/providers/pi-coding-agent.ts | 7 +- .../evaluation/result-artifact-contract.ts | 8 +- .../core/src/evaluation/result-row-schema.ts | 1 + packages/core/src/evaluation/results-repo.ts | 4 +- packages/core/src/evaluation/run-artifacts.ts | 51 ++--- .../core/src/evaluation/trace-envelope.ts | 2 +- packages/core/src/evaluation/trace.ts | 2 +- .../execution-trace/nested-subagent.json | 6 +- .../loaders/agent-skills-parser.test.ts | 4 +- .../core/test/evaluation/orchestrator.test.ts | 15 +- .../providers/log-directory.test.ts | 45 +++++ .../core/test/evaluation/results-repo.test.ts | 36 ++-- .../test/evaluation/trace-envelope.test.ts | 6 +- .../evaluation/trace-normalization.test.ts | 6 +- .../skills/image-compress-and-docs/SKILL.md | 2 +- scripts/deploy-example-results-repos.ts | 2 +- scripts/setup-dashboard-deployment.sh | 2 +- skills-data/agentv-bench/SKILL.md | 4 +- skills-data/agentv-bench/agents/executor.md | 2 +- skills-data/agentv-bench/agents/grader.md | 2 +- .../agentv-bench/references/autoresearch.md | 8 +- .../agentv-bench/references/schemas.md | 2 +- .../references/subagent-pipeline.md | 8 +- skills-data/agentv-eval-writer/SKILL.md | 10 +- 109 files changed, 907 insertions(+), 642 deletions(-) create mode 100644 apps/cli/test/commands/eval/result-layout.test.ts create mode 100644 packages/core/src/evaluation/providers/log-directory.ts create mode 100644 packages/core/test/evaluation/providers/log-directory.test.ts diff --git a/README.md b/README.md index 87e3b0ceb..ff3929e5d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ agentv eval evals/my-eval.yaml **5. Compare results across targets:** ```bash -agentv compare .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl ``` ## Output formats diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index 42aa59798..1d6ce52e6 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -93,6 +93,7 @@ export function buildIndexArtifactEntry( timingPath: string; outputPath?: string; answerPath?: string; + tracePath?: string; transcriptPath?: string; metricsPath?: string; rawProviderLogPath?: string; diff --git a/apps/cli/src/commands/eval/result-layout.ts b/apps/cli/src/commands/eval/result-layout.ts index 1373e7089..b518ecdee 100644 --- a/apps/cli/src/commands/eval/result-layout.ts +++ b/apps/cli/src/commands/eval/result-layout.ts @@ -2,8 +2,9 @@ import { existsSync, statSync } from 'node:fs'; import path from 'node:path'; export const RESULT_INDEX_FILENAME = 'index.jsonl'; -export const RESULT_RUNS_DIRNAME = 'runs'; +export const RESULTS_DIRNAME = 'results'; export const DEFAULT_EXPERIMENT_NAME = 'default'; +const RESERVED_EXPERIMENT_NAMES = new Set(['runs']); export function normalizeExperimentName(experiment?: string): string { const trimmed = experiment?.trim(); @@ -15,6 +16,9 @@ export function normalizeExperimentName(experiment?: string): string { `Invalid experiment name "${trimmed}". Use only letters, numbers, ".", "_" and "-".`, ); } + if (RESERVED_EXPERIMENT_NAMES.has(trimmed)) { + throw new Error(`Invalid experiment name "${trimmed}". This results namespace is reserved.`); + } return trimmed; } @@ -22,19 +26,29 @@ export function createRunDirName(timestamp = new Date()): string { return timestamp.toISOString().replace(/[:.]/g, '-'); } +function defaultRunPathSegments(experiment: string | undefined, runDirName: string): string[] { + const normalizedExperiment = normalizeExperimentName(experiment); + return [normalizedExperiment, runDirName]; +} + +export function buildResultsRootDir(cwd: string): string { + return path.join(cwd, '.agentv', RESULTS_DIRNAME); +} + +export function buildDefaultRunDirFromName( + cwd: string, + experiment: string | undefined, + runDirName: string, +): string { + return path.join(buildResultsRootDir(cwd), ...defaultRunPathSegments(experiment, runDirName)); +} + export function buildDefaultRunDir( cwd: string, experiment?: string, timestamp = new Date(), ): string { - return path.join( - cwd, - '.agentv', - 'results', - RESULT_RUNS_DIRNAME, - normalizeExperimentName(experiment), - createRunDirName(timestamp), - ); + return buildDefaultRunDirFromName(cwd, experiment, createRunDirName(timestamp)); } export function buildDefaultIndexPath(cwd: string, experiment?: string): string { @@ -92,3 +106,17 @@ export function resolveRunManifestPath(filePath: string): string { return filePath; } + +export function relativeRunPathFromCwd(cwd: string, runDir: string): string | undefined { + const relative = path.relative(path.resolve(buildResultsRootDir(cwd)), path.resolve(runDir)); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) { + return undefined; + } + + const parts = relative.split(path.sep).filter(Boolean); + if (parts.length < 2 || RESERVED_EXPERIMENT_NAMES.has(parts[0] ?? '')) { + return undefined; + } + + return parts.join(path.posix.sep); +} diff --git a/apps/cli/src/commands/eval/run-cache.ts b/apps/cli/src/commands/eval/run-cache.ts index 36f302dac..d30c75536 100644 --- a/apps/cli/src/commands/eval/run-cache.ts +++ b/apps/cli/src/commands/eval/run-cache.ts @@ -15,7 +15,7 @@ const CACHE_FILENAME = 'cache.json'; * Keys use camelCase (not snake_case) for backward compat with pre-existing cache files. */ export interface RunCache { - /** Directory path for new per-run directory format (e.g. .agentv/results/runs//) */ + /** Directory path for the run workspace (e.g. .agentv/results/default//) */ readonly lastRunDir?: string; /** @deprecated Legacy flat-file pointer from old cache files. Ignored on read. */ readonly lastResultFile?: string; diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index fdf4b1869..47a6d4940 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -55,7 +55,11 @@ import { loadEnvFromHierarchy } from './env.js'; import { resolveOtelBackend } from './otel-backends.js'; import { type OutputWriter, createOutputWriter } from './output-writer.js'; import { ProgressDisplay, type Verdict, type WorkerProgress } from './progress-display.js'; -import { buildDefaultRunDir, normalizeExperimentName } from './result-layout.js'; +import { + buildDefaultRunDirFromName, + createRunDirName, + normalizeExperimentName, +} from './result-layout.js'; import { buildExclusionFilter, loadErrorTestIds, @@ -540,8 +544,12 @@ async function ensureFileExists(filePath: string, description: string): Promise< } } -function buildDefaultOutputPathForExperiment(cwd: string, experiment?: string): string { - const runDir = buildDefaultRunDir(cwd, experiment); +function buildDefaultOutputPathForExperiment( + cwd: string, + experiment: string | undefined, + runDirName: string, +): string { + const runDir = buildDefaultRunDirFromName(cwd, experiment, runDirName); mkdirSync(runDir, { recursive: true }); return path.join(runDir, 'index.jsonl'); } @@ -1132,14 +1140,6 @@ export async function runEvalCommand( ): Promise { const cwd = process.cwd(); - // Set AGENTV_RUN_TIMESTAMP so CLI targets can group artifacts under the same run folder. - if (!process.env.AGENTV_RUN_TIMESTAMP) { - process.env.AGENTV_RUN_TIMESTAMP = new Date() - .toISOString() - .replace(/:/g, '-') - .replace(/\./g, '-'); - } - // Load agentv.config.ts (if present) for default values let config: Awaited> = null; try { @@ -1296,16 +1296,21 @@ export async function runEvalCommand( const explicitDir = options.outputDir; let runDir: string; let outputPath: string; + const runDirName = process.env.AGENTV_RUN_TIMESTAMP?.trim() || createRunDirName(); if (explicitDir) { runDir = path.resolve(explicitDir); mkdirSync(runDir, { recursive: true }); outputPath = path.join(runDir, 'index.jsonl'); } else { - // Default: .agentv/results/runs/// - outputPath = buildDefaultOutputPathForExperiment(cwd, options.experiment); + // Default: .agentv/results///, using "default" when unspecified. + outputPath = buildDefaultOutputPathForExperiment(cwd, options.experiment, runDirName); runDir = path.dirname(outputPath); } + if (!process.env.AGENTV_RUN_TIMESTAMP) { + process.env.AGENTV_RUN_TIMESTAMP = path.basename(runDir); + } + process.env.AGENTV_RUN_DIR = runDir; // Initialize OTel exporter if --export-otel or --otel-file is set let otelExporter: OtelTraceExporterType | null = null; diff --git a/apps/cli/src/commands/eval/wip-checkpoint.ts b/apps/cli/src/commands/eval/wip-checkpoint.ts index 18160fef9..3345d7061 100644 --- a/apps/cli/src/commands/eval/wip-checkpoint.ts +++ b/apps/cli/src/commands/eval/wip-checkpoint.ts @@ -15,8 +15,8 @@ * Manual recovery from a WIP branch: * git clone /tmp/recovery * cd /tmp/recovery && git checkout agentv/wip// - * cp -r runs/ /.agentv/results/runs/ - * agentv eval --output /.agentv/results/runs/ --resume + * cp -r / /.agentv/results// + * agentv eval --output /.agentv/results// --resume * * All checkpoint operations are best-effort: failures are logged as warnings * and never propagate to the eval run. diff --git a/apps/cli/src/commands/inspect/filter.ts b/apps/cli/src/commands/inspect/filter.ts index 011723266..3b31139a5 100644 --- a/apps/cli/src/commands/inspect/filter.ts +++ b/apps/cli/src/commands/inspect/filter.ts @@ -1,7 +1,7 @@ /** * `agentv inspect filter` — filter evaluation results by metadata criteria. * - * Scans JSONL index files in `.agentv/results/runs/` and applies filters + * Scans JSONL index files in `.agentv/results/` and applies filters * such as target name, experiment name, score thresholds, execution status, * and tool usage. Outputs matching test IDs with summary info. * @@ -53,6 +53,26 @@ function collectIndexFiles(dir: string): string[] { return files; } +function collectCurrentResultIndexFiles(cwd: string): string[] { + const resultsDir = path.join(cwd, '.agentv', 'results'); + const files: string[] = []; + try { + const entries = readdirSync(resultsDir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name === 'runs') continue; + const fullPath = path.join(resultsDir, entry.name); + if (entry.isDirectory()) { + files.push(...collectIndexFiles(fullPath)); + } else if (entry.name === 'index.jsonl') { + files.push(fullPath); + } + } + } catch { + // Directory may not exist + } + return files; +} + /** * Extract tool names from a result record. * Looks in trace.tool_calls (Record) and output messages (tool_calls[].tool). @@ -92,6 +112,22 @@ function extractToolNames(record: Record): string[] { return [...tools]; } +function inferExperimentFromPath(filePath: string): string | undefined { + const parts = filePath.split(path.sep); + const resultsIdx = parts.lastIndexOf('results'); + if (resultsIdx === -1) return undefined; + + const runPath = parts.slice(resultsIdx + 1, -1); + if (runPath[0] === 'runs') return undefined; + if (runPath.length < 2) return undefined; + + const candidate = runPath[0]; + if (candidate && !/^\d{4}-\d{2}-\d{2}T/.test(candidate)) { + return candidate; + } + return undefined; +} + /** * Parse a single JSONL index file into filterable records. */ @@ -119,17 +155,7 @@ export function parseFilterableRecords(filePath: string): FilterableRecord[] { // Determine experiment from record or from directory path let experiment = typeof raw.experiment === 'string' ? raw.experiment : undefined; if (!experiment) { - // Infer from path: .agentv/results/runs///index.jsonl - const parts = filePath.split(path.sep); - const runsIdx = parts.indexOf('runs'); - // If there are 2+ segments between "runs" and the file, the first is the experiment - if (runsIdx !== -1 && parts.length - runsIdx >= 3) { - const candidate = parts[runsIdx + 1]; - // "default" experiment or named experiments; skip if it looks like a timestamp - if (candidate && !/^\d{4}-\d{2}-\d{2}T/.test(candidate)) { - experiment = candidate; - } - } + experiment = inferExperimentFromPath(filePath); } records.push({ @@ -208,7 +234,7 @@ function discoverFilterSources(searchPath: string | undefined, cwd: string): str return [resolved]; } - return collectIndexFiles(path.join(cwd, '.agentv', 'results', 'runs')); + return collectCurrentResultIndexFiles(cwd); } function formatFilterTable(records: FilterableRecord[]): string { @@ -268,7 +294,7 @@ export const inspectFilterCommand = command({ path: positional({ type: optional(string), displayName: 'path', - description: 'Directory or file to filter (default: .agentv/results/runs/)', + description: 'Directory or file to filter (default: .agentv/results/)', }), target: option({ type: optional(string), diff --git a/apps/cli/src/commands/inspect/list.ts b/apps/cli/src/commands/inspect/list.ts index 42bea2b72..719c08c8c 100644 --- a/apps/cli/src/commands/inspect/list.ts +++ b/apps/cli/src/commands/inspect/list.ts @@ -14,13 +14,13 @@ function formatListTable(metas: ResultFileMeta[]): string { const lines: string[] = []; if (metas.length === 0) { - lines.push(`${c.yellow}No run workspaces found in .agentv/results/runs/${c.reset}`); + lines.push(`${c.yellow}No run workspaces found in .agentv/results/${c.reset}`); lines.push(`${c.dim}Run an evaluation first: agentv run ${c.reset}`); return lines.join('\n'); } lines.push(''); - lines.push(`${c.bold}Evaluation Runs${c.reset} ${c.dim}(.agentv/results/runs/)${c.reset}`); + lines.push(`${c.bold}Evaluation Runs${c.reset} ${c.dim}(.agentv/results/)${c.reset}`); lines.push(''); // Column widths @@ -52,7 +52,7 @@ function formatListTable(metas: ResultFileMeta[]): string { export const traceListCommand = command({ name: 'list', - description: 'List recent evaluation run workspaces from .agentv/results/runs/', + description: 'List recent evaluation run workspaces from .agentv/results/', args: { limit: option({ type: optional(number), diff --git a/apps/cli/src/commands/inspect/search.ts b/apps/cli/src/commands/inspect/search.ts index 480bc6025..82407c638 100644 --- a/apps/cli/src/commands/inspect/search.ts +++ b/apps/cli/src/commands/inspect/search.ts @@ -1,7 +1,7 @@ /** * `agentv inspect search` — regex search across evaluation results and transcripts. * - * Scans JSONL files in `.agentv/results/runs/` and `.agentv/transcripts/` for + * Scans JSONL files in `.agentv/results/` and `.agentv/transcripts/` for * lines matching a regex pattern. Outputs file path, test_id, and matching * content with surrounding context. * @@ -56,6 +56,26 @@ function collectJsonlFiles(dir: string): string[] { return files; } +function collectCurrentResultJsonlFiles(cwd: string): string[] { + const resultsDir = path.join(cwd, '.agentv', 'results'); + const files: string[] = []; + try { + const entries = readdirSync(resultsDir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name === 'runs') continue; + const fullPath = path.join(resultsDir, entry.name); + if (entry.isDirectory()) { + files.push(...collectJsonlFiles(fullPath)); + } else if (entry.name.endsWith('.jsonl')) { + files.push(fullPath); + } + } + } catch { + // Directory may not exist + } + return files; +} + /** * Extract a human-readable snippet around a regex match within a string. * Returns up to `contextChars` characters on each side of the match. @@ -142,7 +162,7 @@ export function searchJsonlFile( * Discover all searchable JSONL sources under a base path. * If the path is a file, search that single file. * If it's a directory, recursively find all .jsonl files. - * If not specified, scan both .agentv/results/runs/ and .agentv/transcripts/. + * If not specified, scan both .agentv/results/ and .agentv/transcripts/. */ function discoverSources(basePath: string | undefined, cwd: string): string[] { if (basePath) { @@ -163,7 +183,7 @@ function discoverSources(basePath: string | undefined, cwd: string): string[] { // Default: scan both results and transcripts const sources: string[] = []; - sources.push(...collectJsonlFiles(path.join(cwd, '.agentv', 'results', 'runs'))); + sources.push(...collectCurrentResultJsonlFiles(cwd)); sources.push(...collectJsonlFiles(path.join(cwd, '.agentv', 'transcripts'))); return sources; } @@ -226,7 +246,7 @@ export const inspectSearchCommand = command({ type: optional(string), displayName: 'path', description: - 'Directory or file to search (default: .agentv/results/runs/ and .agentv/transcripts/)', + 'Directory or file to search (default: .agentv/results/ and .agentv/transcripts/)', }), target: option({ type: optional(string), diff --git a/apps/cli/src/commands/inspect/utils.ts b/apps/cli/src/commands/inspect/utils.ts index 574e9de81..6bcb54544 100644 --- a/apps/cli/src/commands/inspect/utils.ts +++ b/apps/cli/src/commands/inspect/utils.ts @@ -4,7 +4,7 @@ import type { EvaluationResult, TraceSummary } from '@agentv/core'; import { DEFAULT_THRESHOLD, toCamelCaseDeep, toSnakeCaseDeep } from '@agentv/core'; import { RESULT_INDEX_FILENAME, - RESULT_RUNS_DIRNAME, + buildResultsRootDir, resolveExistingRunPrimaryPath, resolveWorkspaceOrFilePath, } from '../eval/result-layout.js'; @@ -612,24 +612,31 @@ function collectRunManifestPaths( } } -export function listResultFilesFromRunsDir(runsDir: string, limit?: number): ResultFileMeta[] { +function listResultFilesFromRoot( + runsDir: string, + options?: { limit?: number; skipTopLevelDirs?: ReadonlySet }, +): ResultFileMeta[] { const files: { filePath: string; displayName: string; runId: string; sortName: string }[] = []; try { const entries = readdirSync(runsDir, { withFileTypes: true }); for (const entry of entries) { if (entry.isDirectory()) { + if (options?.skipTopLevelDirs?.has(entry.name)) { + continue; + } collectRunManifestPaths(runsDir, path.join(runsDir, entry.name), files); } } } catch { - // runs/ doesn't exist yet + // Results root doesn't exist yet } // Sort by run directory name descending (most recent first for timestamped runs). files.sort((a, b) => b.sortName.localeCompare(a.sortName)); - const limited = limit !== undefined && limit > 0 ? files.slice(0, limit) : files; + const limited = + options?.limit !== undefined && options.limit > 0 ? files.slice(0, options.limit) : files; const metas: ResultFileMeta[] = []; @@ -665,13 +672,26 @@ export function listResultFilesFromRunsDir(runsDir: string, limit?: number): Res } /** - * Enumerate canonical run manifests in `.agentv/results/runs/`. + * Enumerate canonical run manifests in `.agentv/results/`. + * + * Reserved local namespaces such as `.agentv/results/runs/` and + * `.agentv/results/metadata/` are intentionally skipped by default discovery. */ export function listResultFiles(cwd: string, limit?: number): ResultFileMeta[] { - return listResultFilesFromRunsDir( - path.join(cwd, '.agentv', 'results', RESULT_RUNS_DIRNAME), + const metas = listResultFilesFromRoot(buildResultsRootDir(cwd), { + skipTopLevelDirs: new Set(['metadata', 'runs']), + }).sort((a, b) => { + const byTimestamp = b.timestamp.localeCompare(a.timestamp); + return byTimestamp !== 0 ? byTimestamp : b.displayName.localeCompare(a.displayName); + }); + return limit !== undefined && limit > 0 ? metas.slice(0, limit) : metas; +} + +export function listResultFilesFromRunsDir(runsDir: string, limit?: number): ResultFileMeta[] { + return listResultFilesFromRoot(runsDir, { limit, - ); + skipTopLevelDirs: new Set(['metadata', 'runs']), + }); } /** diff --git a/apps/cli/src/commands/pipeline/input.ts b/apps/cli/src/commands/pipeline/input.ts index 50a15b0d6..61f46b854 100644 --- a/apps/cli/src/commands/pipeline/input.ts +++ b/apps/cli/src/commands/pipeline/input.ts @@ -58,7 +58,7 @@ export const evalInputCommand = command({ type: optional(string), long: 'out', description: - 'Output directory for extracted inputs (default: .agentv/results/runs//)', + 'Output directory for extracted inputs (default: .agentv/results//)', }), experiment: option({ type: optional(string), diff --git a/apps/cli/src/commands/pipeline/run.ts b/apps/cli/src/commands/pipeline/run.ts index fecc3b733..3ac31529c 100644 --- a/apps/cli/src/commands/pipeline/run.ts +++ b/apps/cli/src/commands/pipeline/run.ts @@ -75,7 +75,7 @@ export const evalRunCommand = command({ type: optional(string), long: 'out', description: - 'Output directory for results (default: .agentv/results/runs//)', + 'Output directory for results (default: .agentv/results//)', }), workers: option({ type: optional(number), @@ -227,14 +227,14 @@ export const evalRunCommand = command({ // ── Step 2: Invoke CLI targets in parallel ─────────────────────── if (targetInfo) { const envVars = loadEnvFile(evalDir); - // Set AGENTV_RUN_TIMESTAMP so CLI targets group artifacts under one run folder. + // Set AGENTV_RUN_* so CLI targets group artifacts under this run folder. if (!process.env.AGENTV_RUN_TIMESTAMP) { process.env.AGENTV_RUN_TIMESTAMP = new Date() .toISOString() .replace(/:/g, '-') .replace(/\./g, '-'); } - const mergedEnv = { ...process.env, ...envVars }; + const mergedEnv = { ...process.env, ...envVars, AGENTV_RUN_DIR: outDir }; const maxWorkers = workers ?? targetWorkers ?? 5; let invCompleted = 0; const invTotal = testIds.length; diff --git a/apps/cli/src/commands/results/combine-run.ts b/apps/cli/src/commands/results/combine-run.ts index f62f4b50c..c114ef4a2 100644 --- a/apps/cli/src/commands/results/combine-run.ts +++ b/apps/cli/src/commands/results/combine-run.ts @@ -33,7 +33,12 @@ import { buildTestTargetKey, buildTimingArtifact, } from '../eval/artifact-writer.js'; -import { createRunDirName, resolveRunManifestPath } from '../eval/result-layout.js'; +import { + buildDefaultRunDirFromName, + createRunDirName, + relativeRunPathFromCwd, + resolveRunManifestPath, +} from '../eval/result-layout.js'; import { type ResultManifestRecord, loadManifestResults, @@ -287,7 +292,7 @@ function defaultCombinedRunDir(cwd: string, startedAt: string | undefined): stri parsed && !Number.isNaN(parsed.getTime()) ? createRunDirName(parsed) : sanitizePathSegment(startedAt ?? 'unknown-time'); - return path.join(cwd, '.agentv', 'results', 'runs', 'combined', timestamp); + return buildDefaultRunDirFromName(cwd, 'combined', timestamp); } function uniqueRunDir(baseDir: string): string { @@ -300,13 +305,12 @@ function uniqueRunDir(baseDir: string): string { } function toRunId(cwd: string, runDir: string): string { - const runsRoot = path.join(cwd, '.agentv', 'results', 'runs'); - const relative = path.relative(runsRoot, runDir); - if (relative === '' || relative.startsWith('..') || path.isAbsolute(relative)) { + const relative = relativeRunPathFromCwd(cwd, runDir); + if (!relative) { return path.basename(runDir); } - const parts = relative.split(path.sep); - return parts.length > 1 ? `${parts[0]}::${parts.slice(1).join(path.sep)}` : relative; + const parts = relative.split(path.posix.sep); + return parts.length > 1 ? `${parts[0]}::${parts.slice(1).join(path.posix.sep)}` : relative; } const MANIFEST_PATH_FIELDS = [ @@ -316,6 +320,7 @@ const MANIFEST_PATH_FIELDS = [ 'input_path', 'output_path', 'response_path', + 'trace_path', 'transcript_path', 'metrics_path', 'raw_provider_log_path', @@ -443,6 +448,13 @@ function rewriteAndCopyRecord(row: SelectedRow, outputDir: string): ResultManife row.source.index, ); rewritten.artifact_pointers = artifactPointers; + if ( + row.record.trace_path && + rewritten.trace_path === row.record.trace_path && + artifactPointers?.trace?.path + ) { + rewritten.trace_path = artifactPointers.trace.path; + } if ( row.record.transcript_path && rewritten.transcript_path === row.record.transcript_path && @@ -476,9 +488,15 @@ export function buildCombineRunSources( ): CombineRunSource[] { return sourcePaths.map((sourcePath, index) => { const manifestPath = resolveResultSourcePath(sourcePath, cwd); + const runDir = path.dirname(resolveRunManifestPath(manifestPath)); + if (!relativeRunPathFromCwd(cwd, runDir)) { + throw new Error( + `Run workspace is outside the canonical results layout: ${runDir}. Expected .agentv/results//`, + ); + } return { - id: options?.ids?.[index] ?? path.basename(path.dirname(manifestPath)), - displayName: options?.displayNames?.[index] ?? path.basename(path.dirname(manifestPath)), + id: options?.ids?.[index] ?? toRunId(cwd, runDir), + displayName: options?.displayNames?.[index] ?? path.basename(runDir), manifestPath, tags: options?.tags?.[index], }; @@ -498,6 +516,11 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult ? path.resolve(options.cwd, options.outputDir) : defaultCombinedRunDir(options.cwd, startedAt), ); + if (!relativeRunPathFromCwd(options.cwd, runDir)) { + throw new Error( + `Output run workspace must use .agentv/results//: ${runDir}`, + ); + } const { rows, conflicts } = selectRows( loadedSources, options.duplicatePolicy, diff --git a/apps/cli/src/commands/results/combine.ts b/apps/cli/src/commands/results/combine.ts index f3c5be497..f10575dfa 100644 --- a/apps/cli/src/commands/results/combine.ts +++ b/apps/cli/src/commands/results/combine.ts @@ -86,7 +86,7 @@ export const resultsCombineCommand = command({ long: 'output', short: 'o', description: - 'Output run workspace directory (defaults to .agentv/results/runs/combined/)', + 'Output run workspace directory (defaults to .agentv/results/combined/)', }), displayName: option({ type: optional(string), diff --git a/apps/cli/src/commands/results/delete-run.ts b/apps/cli/src/commands/results/delete-run.ts index 0ee04fa87..f8f3bc90f 100644 --- a/apps/cli/src/commands/results/delete-run.ts +++ b/apps/cli/src/commands/results/delete-run.ts @@ -2,16 +2,20 @@ * Shared local run deletion primitive for `agentv results delete` and the * Dashboard API. * - * Deletes exactly one local run workspace directory under - * `.agentv/results/runs/`. Callers may pass a run ID, run workspace directory, - * or `index.jsonl` path. Remote runs and paths outside the local results tree - * are rejected before anything is removed. + * Deletes exactly one local run workspace directory under `.agentv/results/`. + * Callers may pass a run ID, run workspace directory, or `index.jsonl` path. + * Remote runs and paths outside the local results tree are rejected before + * anything is removed. */ import { existsSync, rmSync } from 'node:fs'; import path from 'node:path'; -import { RESULT_INDEX_FILENAME, resolveRunManifestPath } from '../eval/result-layout.js'; +import { + RESULT_INDEX_FILENAME, + relativeRunPathFromCwd, + resolveRunManifestPath, +} from '../eval/result-layout.js'; import { listResultFiles } from '../inspect/utils.js'; import { resolveResultSourcePath } from './manifest.js'; import { isRemoteRunId } from './remote.js'; @@ -26,10 +30,6 @@ export interface DeleteRunResult extends DeleteRunTarget { readonly deleted: true; } -function localRunsRoot(cwd: string): string { - return path.resolve(cwd, '.agentv', 'results', 'runs'); -} - function assertLocalRunManifest(cwd: string, manifestPath: string, runId: string): DeleteRunTarget { const resolvedManifestPath = path.resolve(manifestPath); if (path.basename(resolvedManifestPath) !== RESULT_INDEX_FILENAME) { @@ -37,9 +37,7 @@ function assertLocalRunManifest(cwd: string, manifestPath: string, runId: string } const runDir = path.dirname(resolvedManifestPath); - const runsRoot = localRunsRoot(cwd); - const relativeRunDir = path.relative(runsRoot, runDir); - if (relativeRunDir === '' || relativeRunDir.startsWith('..') || path.isAbsolute(relativeRunDir)) { + if (!relativeRunPathFromCwd(cwd, runDir)) { throw new Error('Run workspace is outside the local results directory'); } if (!existsSync(resolvedManifestPath)) { diff --git a/apps/cli/src/commands/results/eval-runner.ts b/apps/cli/src/commands/results/eval-runner.ts index 66fb5f6db..ac8f23fc7 100644 --- a/apps/cli/src/commands/results/eval-runner.ts +++ b/apps/cli/src/commands/results/eval-runner.ts @@ -40,7 +40,7 @@ interface DashboardRun { command: string; /** Target name passed via --target (if any). Stored so the run list can show it before the first result is written. */ target?: string; - /** Absolute path to the run directory (e.g. .agentv/results/runs/default/). Used to correlate this in-memory run with the filesystem run when the JSONL has 0 records yet. */ + /** Absolute path to the run directory (e.g. .agentv/results/default/). Used to correlate this in-memory run with the filesystem run when the JSONL has 0 records yet. */ outputDir?: string; startedAt: string; finishedAt?: string; diff --git a/apps/cli/src/commands/results/export.ts b/apps/cli/src/commands/results/export.ts index 1e916781e..769489c94 100644 --- a/apps/cli/src/commands/results/export.ts +++ b/apps/cli/src/commands/results/export.ts @@ -140,7 +140,7 @@ export const resultsExportCommand = command({ type: optional(string), displayName: 'source', description: - 'Run workspace directory or index.jsonl manifest to export (defaults to most recent in .agentv/results/runs/)', + 'Run workspace directory or index.jsonl manifest to export (defaults to most recent in .agentv/results/)', }), out: option({ type: optional(string), diff --git a/apps/cli/src/commands/results/manifest.ts b/apps/cli/src/commands/results/manifest.ts index bdb90d377..c9a272e5f 100644 --- a/apps/cli/src/commands/results/manifest.ts +++ b/apps/cli/src/commands/results/manifest.ts @@ -44,6 +44,7 @@ export interface ResultManifestRecord { readonly input_path?: string; readonly output_path?: string; readonly answer_path?: string; + readonly trace_path?: string; readonly transcript_path?: string; readonly metrics_path?: string; readonly raw_provider_log_path?: string; diff --git a/apps/cli/src/commands/results/projection-bundle.ts b/apps/cli/src/commands/results/projection-bundle.ts index e57d2b049..a81b96c15 100644 --- a/apps/cli/src/commands/results/projection-bundle.ts +++ b/apps/cli/src/commands/results/projection-bundle.ts @@ -147,9 +147,10 @@ function shortHash(parts: readonly string[], length = 20): string { } function tracePathFor(indexEntry: IndexArtifactEntry): string | undefined { - return indexEntry.artifact_dir - ? path.posix.join(indexEntry.artifact_dir, 'outputs', 'trace.json') - : undefined; + return ( + indexEntry.trace_path ?? + (indexEntry.artifact_dir ? path.posix.join(indexEntry.artifact_dir, 'trace.json') : undefined) + ); } function artifactRefs( diff --git a/apps/cli/src/commands/results/remote-metadata.ts b/apps/cli/src/commands/results/remote-metadata.ts index fa46d41d8..7c26c699d 100644 --- a/apps/cli/src/commands/results/remote-metadata.ts +++ b/apps/cli/src/commands/results/remote-metadata.ts @@ -4,8 +4,9 @@ * Remote run artifacts under `runs/**` on the results branch are treated as * immutable fetched payloads. Editable fields, starting with tags, live in a * small sidecar tree under `metadata/runs/**` inside the configured results repo - * checkout. That keeps local edits pushable by normal Git sync without rewriting - * the fetched run directory. + * checkout/branch. This is a remote-results implementation detail, not part of + * the local `.agentv/results///` layout. It keeps local + * edits pushable by normal Git sync without rewriting the fetched run bundle. * * To add another mutable field: create a sibling helper that maps the remote * run manifest to the same metadata run directory, keep the on-disk keys diff --git a/apps/cli/src/commands/results/remote.ts b/apps/cli/src/commands/results/remote.ts index 5670a6aaa..dea570482 100644 --- a/apps/cli/src/commands/results/remote.ts +++ b/apps/cli/src/commands/results/remote.ts @@ -22,6 +22,7 @@ import { syncResultsRepoForProject, } from '@agentv/core'; +import { relativeRunPathFromCwd } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; import { type ResultFileMeta, @@ -124,17 +125,8 @@ export interface RemoteResultsStatus extends ResultsRepoStatus { } function relativeLocalRunPath(cwd: string, manifestPath: string): string | undefined { - const runsRoot = path.resolve(cwd, '.agentv', 'results', 'runs'); const manifestDir = path.resolve(path.dirname(manifestPath)); - const relativeRunPath = path.relative(runsRoot, manifestDir); - if ( - relativeRunPath.length === 0 || - relativeRunPath.startsWith('..') || - path.isAbsolute(relativeRunPath) - ) { - return undefined; - } - return relativeRunPath.split(path.sep).join(path.posix.sep); + return relativeRunPathFromCwd(cwd, manifestDir); } function remoteMetadataManifestPath( @@ -180,14 +172,14 @@ function statusForResult(result: EvaluationResult): 'PASS' | 'FAIL' | 'ERROR' { } export function getRelativeRunPath(cwd: string, runDir: string): string { - const relative = path.relative(path.join(cwd, '.agentv', 'results', 'runs'), runDir); - if (!relative.startsWith('..') && !path.isAbsolute(relative)) { + const relative = relativeRunPathFromCwd(cwd, runDir); + if (relative) { return relative; } - const experiment = path.basename(path.dirname(runDir)); - const runName = path.basename(runDir); - return experiment && experiment !== runName ? path.join(experiment, runName) : runName; + throw new Error( + `Run workspace must use .agentv/results//: ${path.resolve(runDir)}`, + ); } function buildCommitTitle(payload: RemoteExportPayload): string { diff --git a/apps/cli/src/commands/results/run-tags.ts b/apps/cli/src/commands/results/run-tags.ts index 9464aa068..8f7e50459 100644 --- a/apps/cli/src/commands/results/run-tags.ts +++ b/apps/cli/src/commands/results/run-tags.ts @@ -66,6 +66,14 @@ export function runTagsPath(manifestPath: string): string { function inferRunRelativePath(manifestPath: string): string { const runDir = path.dirname(manifestPath); const segments = runDir.split(path.sep); + const resultsIndex = segments.lastIndexOf('results'); + if (resultsIndex >= 0 && resultsIndex < segments.length - 2) { + const runSegments = segments.slice(resultsIndex + 1); + const namespace = runSegments[0]; + if (namespace !== 'metadata' && namespace !== 'runs') { + return runSegments.join('/'); + } + } const runsIndex = segments.lastIndexOf('runs'); if (runsIndex >= 0 && runsIndex < segments.length - 1) { return segments.slice(runsIndex + 1).join('/'); diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 06114a4f4..06b43d6d9 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -75,6 +75,7 @@ import { Hono } from 'hono'; import { enforceRequiredVersion } from '../../version-check.js'; import { parseJsonlResults } from '../eval/artifact-writer.js'; +import { relativeRunPathFromCwd } from '../eval/result-layout.js'; import { loadRunCache, resolveRunCacheFile } from '../eval/run-cache.js'; import { findRepoRoot } from '../eval/shared.js'; import { listResultFiles } from '../inspect/utils.js'; @@ -117,13 +118,13 @@ import { type StudioConfig, loadStudioConfig, saveStudioConfig } from './studio- /** * Dashboard has one run source per project: the project's - * `.agentv/results/runs/` tree plus any `results:` repo configured for that + * `.agentv/results/` tree plus any `results:` repo configured for that * project. Direct run workspaces and index manifests are supported by * `agentv results report`, not the live Dashboard server. */ const DIRECT_DASHBOARD_SOURCE_GUIDANCE = [ 'Dashboard reads configured project run sources only.', - 'Run it from a project root, or pass --dir so Dashboard uses /.agentv/results/runs/:', + 'Run it from a project root, or pass --dir so Dashboard uses /.agentv/results/:', ' agentv dashboard --dir ', 'To browse external results, configure results.repo.remote or results.repo.path in config YAML.', 'For a one-off run bundle, use: agentv results report ', @@ -149,14 +150,18 @@ export async function resolveSourceFile(source: string | undefined, cwd: string) // Prefer cache pointer, fall back to directory scan const cache = await loadRunCache(cwd); const cachedFile = cache ? resolveRunCacheFile(cache) : ''; - if (cachedFile && existsSync(cachedFile)) { + if ( + cachedFile && + existsSync(cachedFile) && + relativeRunPathFromCwd(cwd, path.dirname(cachedFile)) + ) { return cachedFile; } const metas = listResultFiles(cwd, 10); if (metas.length === 0) { throw new Error( - 'No run workspaces found in .agentv/results/runs/\nRun an evaluation first: agentv eval ', + 'No run workspaces found in .agentv/results/\nRun an evaluation first: agentv eval ', ); } if (metas.length > 1) { @@ -875,7 +880,7 @@ function artifactFileContentResponse(c: C, filePath: string, fileContent: string function missingTranscriptMessage(): string { return [ - 'This result does not include canonical outputs/transcript.jsonl metadata.', + 'This result does not include canonical transcript.jsonl metadata.', 'Dashboard does not parse response.md or markdown transcripts for this view.', ].join(' '); } @@ -922,7 +927,7 @@ function traceSessionArtifactResponse( function missingTraceMessage(): string { return [ - 'This result does not include canonical outputs/trace.json metadata.', + 'This result does not include canonical trace.json metadata.', 'Dashboard trace sessions require an agentv.trace.v1 sidecar artifact.', ].join(' '); } @@ -2470,10 +2475,6 @@ async function handleRunDelete(c: C, { searchDir, projectId }: DataContext) { } } -function getLocalRunsRoot(searchDir: string): string { - return path.join(searchDir, '.agentv', 'results', 'runs'); -} - function validateLocalCompletedRun( searchDir: string, meta: SourcedResultFileMeta, @@ -2492,8 +2493,7 @@ function validateLocalCompletedRun( } const runDir = path.dirname(manifestPath); - const runsRoot = path.resolve(getLocalRunsRoot(searchDir)); - if (runDir !== runsRoot && runDir.startsWith(`${runsRoot}${path.sep}`) && existsSync(runDir)) { + if (relativeRunPathFromCwd(searchDir, runDir) && existsSync(runDir)) { return { ok: true }; } return { error: 'Run workspace is outside the local results directory', status: 400 }; @@ -3253,7 +3253,11 @@ export const resultsServeCommand = command({ // Auto-discover: run cache -> directory scan -> empty state const cache = await loadRunCache(cwd); const cachedFile = cache ? resolveRunCacheFile(cache) : ''; - if (cachedFile && existsSync(cachedFile)) { + if ( + cachedFile && + existsSync(cachedFile) && + relativeRunPathFromCwd(cwd, path.dirname(cachedFile)) + ) { sourceFile = cachedFile; results = loadManifestResults(cachedFile, { hydrateTranscriptTrace: false }); } else { diff --git a/apps/cli/src/commands/results/shared.ts b/apps/cli/src/commands/results/shared.ts index 0ba2b38d6..c73513246 100644 --- a/apps/cli/src/commands/results/shared.ts +++ b/apps/cli/src/commands/results/shared.ts @@ -23,7 +23,7 @@ export const sourceArg = positional({ type: optional(string), displayName: 'source', description: - 'Run workspace directory or index.jsonl manifest (defaults to most recent in .agentv/results/runs/)', + 'Run workspace directory or index.jsonl manifest (defaults to most recent in .agentv/results/)', }); /** @@ -50,7 +50,7 @@ export async function resolveSourceFile( } else { const metas = listResultFiles(cwd, 1); if (metas.length === 0) { - console.error('Error: No run workspaces found in .agentv/results/runs/'); + console.error('Error: No run workspaces found in .agentv/results/'); console.error('Run an evaluation first: agentv eval '); process.exit(1); } diff --git a/apps/cli/src/commands/results/validate.ts b/apps/cli/src/commands/results/validate.ts index f993eb328..c0ffb8dc0 100644 --- a/apps/cli/src/commands/results/validate.ts +++ b/apps/cli/src/commands/results/validate.ts @@ -3,7 +3,7 @@ * artifacts compatible with the AgentV dashboard and results commands. * * Checks: - * 1. Directory follows the `runs//` naming convention + * 1. Directory follows the `.agentv/results//` naming convention * 2. index.jsonl exists and each line has required fields * 3. Per-test grading.json exists for every entry in the index * 4. Per-test timing.json exists (warning if missing) @@ -44,20 +44,25 @@ interface IndexEntry { function checkDirectoryNaming(runDir: string): Diagnostic[] { const dirName = path.basename(runDir); const pathSegments = path.normalize(runDir).split(path.sep).filter(Boolean); - const runsIndex = pathSegments.lastIndexOf('runs'); + const resultsIndex = pathSegments.lastIndexOf('results'); const diagnostics: Diagnostic[] = []; - if (runsIndex < 0 || runsIndex >= pathSegments.length - 1) { + if (resultsIndex >= 0 && pathSegments[resultsIndex + 1] === 'runs') { + diagnostics.push({ + severity: 'error', + message: + 'The .agentv/results/runs namespace is no longer supported. Expected: .agentv/results//', + }); + } else if (resultsIndex < 0 || resultsIndex > pathSegments.length - 3) { diagnostics.push({ severity: 'warning', message: - "Directory is not under a 'runs/' tree. Expected: .agentv/results/runs//", + 'Directory is not under the canonical results tree. Expected: .agentv/results//', }); } const isNewFormat = /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/.test(dirName); - const isLegacyFormat = /^eval_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/.test(dirName); - if (!isNewFormat && !isLegacyFormat) { + if (!isNewFormat) { diagnostics.push({ severity: 'warning', message: `Directory name '${dirName}' does not match the expected pattern ''. Example: 2026-03-27T12-42-24-429Z`, diff --git a/apps/cli/src/commands/trend/index.ts b/apps/cli/src/commands/trend/index.ts index 69078ff44..c9dadf7a2 100644 --- a/apps/cli/src/commands/trend/index.ts +++ b/apps/cli/src/commands/trend/index.ts @@ -153,7 +153,7 @@ export function resolveTrendSources( if (metas.length < 2) { throw new Error( - 'Trend analysis requires at least 2 canonical run workspaces in .agentv/results/runs/', + 'Trend analysis requires at least 2 canonical run workspaces in .agentv/results/', ); } @@ -414,7 +414,7 @@ export const trendCommand = command({ last: option({ type: optional(number), long: 'last', - description: 'Use the most recent N runs from .agentv/results/runs/', + description: 'Use the most recent N runs from .agentv/results/', }), suite: option({ type: optional(string), diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index cac402eef..1d05c3e6f 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -612,9 +612,9 @@ describe('parseJsonlResults', () => { artifactPointers: { transcript: { ref: 'agentv/artifacts/v1', - key: 'transcripts/pointer-row/outputs/transcript.jsonl', + key: 'transcripts/pointer-row/transcript.jsonl', object_version: 'sha256:test', - path: 'pointer-row/outputs/transcript.jsonl', + path: 'pointer-row/transcript.jsonl', sha256: 'test', size: 1, schema_version: 'agentv.transcript.v1', @@ -633,7 +633,7 @@ describe('parseJsonlResults', () => { target: 'codex', score: 1, output: 'done', - raw_provider_log_path: 'raw-log-case/outputs/raw/provider.log', + raw_provider_log_path: 'raw-log-case/provider.log', })}\n`; const results = parseJsonlResults(content); @@ -778,7 +778,14 @@ describe('writeArtifactsFromResults', () => { ]); const alphaEntries = await readdir(path.join(paths.testArtifactDir, 'alpha')); - expect(alphaEntries.sort()).toEqual(['grading.json', 'outputs', 'timing.json']); + expect(alphaEntries.sort()).toEqual([ + 'grading.json', + 'metrics.json', + 'outputs', + 'timing.json', + 'trace.json', + 'transcript.jsonl', + ]); const alphaGrading: GradingArtifact = JSON.parse( await readFile(path.join(paths.testArtifactDir, 'alpha', 'grading.json'), 'utf8'), @@ -807,6 +814,9 @@ describe('writeArtifactsFromResults', () => { expect(indexLines).toHaveLength(2); expect(indexLines[0]?.grading_path).toBe('alpha/grading.json'); expect(indexLines[0]?.timing_path).toBe('alpha/timing.json'); + expect(indexLines[0]?.trace_path).toBe('alpha/trace.json'); + expect(indexLines[0]?.transcript_path).toBe('alpha/transcript.jsonl'); + expect(indexLines[0]?.metrics_path).toBe('alpha/metrics.json'); }); it('handles empty results array', async () => { @@ -898,16 +908,14 @@ describe('writeArtifactsFromResults', () => { await writeArtifactsFromResults(results, testDir); - const transcriptPath = path.join(testDir, 'transcript-case', 'outputs', 'transcript.jsonl'); + const transcriptPath = path.join(testDir, 'transcript-case', 'transcript.jsonl'); const transcriptLines = (await readFile(transcriptPath, 'utf8')) .trim() .split('\n') .map((line) => JSON.parse(line)); const envelope = TraceEnvelopeWireSchema.parse( - JSON.parse( - await readFile(path.join(testDir, 'transcript-case', 'outputs', 'trace.json'), 'utf8'), - ), + JSON.parse(await readFile(path.join(testDir, 'transcript-case', 'trace.json'), 'utf8')), ); const projectedEnvelope = fromTraceEnvelopeWire(envelope); const projectedTranscript = traceEnvelopeToTranscriptJsonLines(projectedEnvelope, { @@ -991,30 +999,28 @@ describe('writeArtifactsFromResults', () => { 'execute_tool', ]); await expect( - readFile(path.join(testDir, 'transcript-case', 'outputs', 'transcript.json'), 'utf8'), + readFile(path.join(testDir, 'transcript-case', 'transcript.json'), 'utf8'), ).rejects.toThrow(); const indexLine = JSON.parse( (await readFile(path.join(testDir, 'index.jsonl'), 'utf8')).trim(), ); - expect(indexLine.transcript_path).toBe('transcript-case/outputs/transcript.jsonl'); + expect(indexLine.trace_path).toBe('transcript-case/trace.json'); + expect(indexLine.transcript_path).toBe('transcript-case/transcript.jsonl'); expect(indexLine.transcript_path.endsWith(CANONICAL_TRANSCRIPT_ARTIFACT_PATH)).toBe(true); - expect(indexLine.metrics_path).toBe('transcript-case/outputs/metrics.json'); + expect(indexLine.metrics_path).toBe('transcript-case/metrics.json'); expect(indexLine.metrics_path.endsWith(CANONICAL_METRICS_ARTIFACT_PATH)).toBe(true); - expect(indexLine).not.toHaveProperty('trace_path'); - const traceContent = await readFile( - path.join(testDir, 'transcript-case', 'outputs', 'trace.json'), - ); + const traceContent = await readFile(path.join(testDir, 'transcript-case', 'trace.json')); const transcriptContent = await readFile(transcriptPath); const traceSha = sha256Hex(traceContent); const transcriptSha = sha256Hex(transcriptContent); expect(indexLine.artifact_pointers.trace).toMatchObject({ ref: AGENTV_RESULTS_ARTIFACTS_REF, - key: 'traces/transcript-case/outputs/trace.json', + key: 'traces/transcript-case/trace.json', object_version: `sha256:${traceSha}`, - path: 'transcript-case/outputs/trace.json', + path: 'transcript-case/trace.json', sha256: traceSha, size: traceContent.byteLength, schema_version: EXECUTION_TRACE_SCHEMA_VERSION, @@ -1023,9 +1029,9 @@ describe('writeArtifactsFromResults', () => { }); expect(indexLine.artifact_pointers.transcript).toMatchObject({ ref: AGENTV_RESULTS_ARTIFACTS_REF, - key: 'transcripts/transcript-case/outputs/transcript.jsonl', + key: 'transcripts/transcript-case/transcript.jsonl', object_version: `sha256:${transcriptSha}`, - path: 'transcript-case/outputs/transcript.jsonl', + path: 'transcript-case/transcript.jsonl', sha256: transcriptSha, size: transcriptContent.byteLength, schema_version: TRANSCRIPT_SCHEMA_VERSION, @@ -1129,12 +1135,10 @@ describe('writeArtifactsFromResults', () => { const indexLine = JSON.parse( (await readFile(path.join(testDir, 'index.jsonl'), 'utf8')).trim(), ); - expect(indexLine.metrics_path).toBe('summary-case/outputs/metrics.json'); + expect(indexLine.metrics_path).toBe('summary-case/metrics.json'); const summary = MetricsArtifactWireSchema.parse( - JSON.parse( - await readFile(path.join(testDir, 'summary-case', 'outputs', 'metrics.json'), 'utf8'), - ), + JSON.parse(await readFile(path.join(testDir, 'summary-case', 'metrics.json'), 'utf8')), ); expect(summary.schema_version).toBe(METRICS_SCHEMA_VERSION); @@ -1278,14 +1282,10 @@ describe('writeArtifactsFromResults', () => { const runTiming = JSON.parse(await readFile(path.join(testDir, 'timing.json'), 'utf8')); MetricsArtifactWireSchema.parse( - JSON.parse( - await readFile(path.join(testDir, 'aggregate-usage', 'outputs', 'metrics.json'), 'utf8'), - ), + JSON.parse(await readFile(path.join(testDir, 'aggregate-usage', 'metrics.json'), 'utf8')), ); MetricsArtifactWireSchema.parse( - JSON.parse( - await readFile(path.join(testDir, 'estimated-usage', 'outputs', 'metrics.json'), 'utf8'), - ), + JSON.parse(await readFile(path.join(testDir, 'estimated-usage', 'metrics.json'), 'utf8')), ); expect(aggregateTiming).toMatchObject({ @@ -1343,30 +1343,28 @@ describe('writeArtifactsFromResults', () => { await writeArtifactsFromResults(results, testDir); - const copiedRawLogPath = path.join(testDir, 'raw-log-case', 'outputs', 'raw', 'provider.log'); + const copiedRawLogPath = path.join(testDir, 'raw-log-case', 'provider.log'); expect(await readFile(copiedRawLogPath, 'utf8')).toBe(rawLog); - const transcriptPath = path.join(testDir, 'raw-log-case', 'outputs', 'transcript.jsonl'); + const transcriptPath = path.join(testDir, 'raw-log-case', 'transcript.jsonl'); await expect(readFile(transcriptPath, 'utf8')).resolves.toContain( '"schema_version":"agentv.transcript.v1"', ); await expect( - readFile(path.join(testDir, 'raw-log-case', 'outputs', 'transcript.json'), 'utf8'), + readFile(path.join(testDir, 'raw-log-case', 'transcript.json'), 'utf8'), ).rejects.toThrow(); const envelope = TraceEnvelopeWireSchema.parse( - JSON.parse( - await readFile(path.join(testDir, 'raw-log-case', 'outputs', 'trace.json'), 'utf8'), - ), + JSON.parse(await readFile(path.join(testDir, 'raw-log-case', 'trace.json'), 'utf8')), ); - expect(envelope.artifacts.raw_provider_log_path).toBe('outputs/raw/provider.log'); - expect(envelope.artifacts.transcript_path).toBe('outputs/transcript.jsonl'); + expect(envelope.artifacts.raw_provider_log_path).toBe('provider.log'); + expect(envelope.artifacts.transcript_path).toBe('transcript.jsonl'); const indexLine = JSON.parse( (await readFile(path.join(testDir, 'index.jsonl'), 'utf8')).trim(), ); - expect(indexLine.raw_provider_log_path).toBe('raw-log-case/outputs/raw/provider.log'); - expect(indexLine.transcript_path).toBe('raw-log-case/outputs/transcript.jsonl'); + expect(indexLine.raw_provider_log_path).toBe('raw-log-case/provider.log'); + expect(indexLine.transcript_path).toBe('raw-log-case/transcript.jsonl'); expect(indexLine).not.toHaveProperty('transcript_json_path'); }); @@ -1413,9 +1411,7 @@ describe('writeArtifactsFromResults', () => { expect(JSON.stringify(indexLine)).not.toContain('api_key'); const envelope = TraceEnvelopeWireSchema.parse( - JSON.parse( - await readFile(path.join(testDir, 'external-trace-case', 'outputs', 'trace.json'), 'utf8'), - ), + JSON.parse(await readFile(path.join(testDir, 'external-trace-case', 'trace.json'), 'utf8')), ); expect(envelope.external_trace).toEqual(indexLine.external_trace); expect(envelope.source.metadata ?? {}).not.toHaveProperty('external_trace'); @@ -1434,18 +1430,18 @@ describe('writeArtifactsFromResults', () => { await writeArtifactsFromResults(results, testDir); - const transcriptPath = path.join(testDir, 'no-transcript-case', 'outputs', 'transcript.jsonl'); + const transcriptPath = path.join(testDir, 'no-transcript-case', 'transcript.jsonl'); await expect(readFile(transcriptPath, 'utf8')).rejects.toThrow(); const indexLine = JSON.parse( (await readFile(path.join(testDir, 'index.jsonl'), 'utf8')).trim(), ); expect(indexLine).not.toHaveProperty('transcript_path'); - expect(indexLine.metrics_path).toBe('no-transcript-case/outputs/metrics.json'); + expect(indexLine.metrics_path).toBe('no-transcript-case/metrics.json'); expect(indexLine.artifact_pointers.trace).toMatchObject({ ref: AGENTV_RESULTS_ARTIFACTS_REF, - key: 'traces/no-transcript-case/outputs/trace.json', - path: 'no-transcript-case/outputs/trace.json', + key: 'traces/no-transcript-case/trace.json', + path: 'no-transcript-case/trace.json', schema_version: EXECUTION_TRACE_SCHEMA_VERSION, media_type: TRACE_JSON_MEDIA_TYPE, family: 'traces', @@ -1454,9 +1450,7 @@ describe('writeArtifactsFromResults', () => { expect(indexLine.artifact_pointers).not.toHaveProperty('metrics'); const envelope = TraceEnvelopeWireSchema.parse( - JSON.parse( - await readFile(path.join(testDir, 'no-transcript-case', 'outputs', 'trace.json'), 'utf8'), - ), + JSON.parse(await readFile(path.join(testDir, 'no-transcript-case', 'trace.json'), 'utf8')), ); expect(envelope.artifacts).not.toHaveProperty('transcript_path'); expect(envelope.artifacts.metrics_path).toBe(CANONICAL_METRICS_ARTIFACT_PATH); diff --git a/apps/cli/test/commands/eval/result-layout.test.ts b/apps/cli/test/commands/eval/result-layout.test.ts new file mode 100644 index 000000000..d45e3c740 --- /dev/null +++ b/apps/cli/test/commands/eval/result-layout.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'bun:test'; +import path from 'node:path'; + +import { + buildDefaultRunDir, + buildDefaultRunDirFromName, + normalizeExperimentName, + relativeRunPathFromCwd, +} from '../../../src/commands/eval/result-layout.js'; + +describe('result layout', () => { + it('groups default run directories under the default experiment', () => { + const cwd = '/repo'; + const timestamp = new Date('2026-06-22T12:34:56.789Z'); + + expect(buildDefaultRunDir(cwd, undefined, timestamp)).toBe( + path.join('/repo', '.agentv', 'results', 'default', '2026-06-22T12-34-56-789Z'), + ); + }); + + it('groups named experiment run directories under the experiment', () => { + expect(buildDefaultRunDirFromName('/repo', 'with-skills', '2026-run')).toBe( + path.join('/repo', '.agentv', 'results', 'with-skills', '2026-run'), + ); + }); + + it('reserves the deprecated runs namespace', () => { + expect(() => normalizeExperimentName('runs')).toThrow('reserved'); + expect( + relativeRunPathFromCwd( + '/repo', + path.join('/repo', '.agentv', 'results', 'runs', 'default', '2026-run'), + ), + ).toBeUndefined(); + expect( + relativeRunPathFromCwd( + '/repo', + path.join('/repo', '.agentv', 'results', 'default', '2026-run'), + ), + ).toBe('default/2026-run'); + }); +}); diff --git a/apps/cli/test/commands/eval/run-cache.test.ts b/apps/cli/test/commands/eval/run-cache.test.ts index c2ee4f7f6..e8a327cbb 100644 --- a/apps/cli/test/commands/eval/run-cache.test.ts +++ b/apps/cli/test/commands/eval/run-cache.test.ts @@ -5,15 +5,18 @@ import { type RunCache, resolveRunCacheFile } from '../../../src/commands/eval/r describe('resolveRunCacheFile', () => { it('should resolve new directory-based cache to index.jsonl inside dir', () => { - const cache: RunCache = { lastRunDir: '/results/runs/2026-03-24T00-00-00-000Z', timestamp: '' }; + const cache: RunCache = { + lastRunDir: '/results/default/2026-03-24T00-00-00-000Z', + timestamp: '', + }; expect(resolveRunCacheFile(cache)).toBe( - path.join('/results/runs/2026-03-24T00-00-00-000Z', 'index.jsonl'), + path.join('/results/default/2026-03-24T00-00-00-000Z', 'index.jsonl'), ); }); it('ignores legacy file-based cache entries', () => { const cache: RunCache = { - lastResultFile: '/results/runs/eval_2026-03-24.jsonl', + lastResultFile: '/results/default/eval_2026-03-24.jsonl', timestamp: '', }; expect(resolveRunCacheFile(cache)).toBe(''); @@ -21,12 +24,12 @@ describe('resolveRunCacheFile', () => { it('should prefer lastRunDir over lastResultFile when both present', () => { const cache: RunCache = { - lastRunDir: '/results/runs/2026-03-24T00-00-00-000Z', - lastResultFile: '/results/runs/eval_old.jsonl', + lastRunDir: '/results/default/2026-03-24T00-00-00-000Z', + lastResultFile: '/results/default/eval_old.jsonl', timestamp: '', }; expect(resolveRunCacheFile(cache)).toBe( - path.join('/results/runs/2026-03-24T00-00-00-000Z', 'index.jsonl'), + path.join('/results/default/2026-03-24T00-00-00-000Z', 'index.jsonl'), ); }); diff --git a/apps/cli/test/commands/results/combine.test.ts b/apps/cli/test/commands/results/combine.test.ts index b403debcc..be55be12f 100644 --- a/apps/cli/test/commands/results/combine.test.ts +++ b/apps/cli/test/commands/results/combine.test.ts @@ -45,7 +45,7 @@ describe('results combine', () => { }); function seedRun(name: string, records: object[]): string { - const runDir = path.join(tempDir, '.agentv', 'results', 'runs', name); + const runDir = path.join(tempDir, '.agentv', 'results', 'default', name); mkdirSync(path.join(runDir, 'demo', 'test-a'), { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(...records), 'utf8'); writeFileSync(path.join(runDir, 'demo', 'test-a', 'grading.json'), '{"assertions":[]}\n'); @@ -98,15 +98,16 @@ describe('results combine', () => { const first = seedRun('run-a', [ result({ artifact_dir: 'demo/test-a', - transcript_path: 'demo/test-a/outputs/transcript.jsonl', - metrics_path: 'demo/test-a/outputs/metrics.json', - raw_provider_log_path: 'demo/test-a/outputs/raw/provider.log', + trace_path: 'demo/test-a/trace.json', + transcript_path: 'demo/test-a/transcript.jsonl', + metrics_path: 'demo/test-a/metrics.json', + raw_provider_log_path: 'demo/test-a/provider.log', artifact_pointers: { trace: { ref: 'agentv/artifacts/v1', - key: 'traces/demo/test-a/outputs/trace.json', + key: 'traces/demo/test-a/trace.json', object_version: 'sha256:trace', - path: 'demo/test-a/outputs/trace.json', + path: 'demo/test-a/trace.json', sha256: 'trace', size: 18, schema_version: 'agentv.trace.v1', @@ -115,9 +116,9 @@ describe('results combine', () => { }, transcript: { ref: 'agentv/artifacts/v1', - key: 'transcripts/demo/test-a/outputs/transcript.jsonl', + key: 'transcripts/demo/test-a/transcript.jsonl', object_version: 'sha256:transcript', - path: 'demo/test-a/outputs/transcript.jsonl', + path: 'demo/test-a/transcript.jsonl', sha256: 'transcript', size: 180, schema_version: 'agentv.transcript.v1', @@ -127,10 +128,10 @@ describe('results combine', () => { }, }), ]); - mkdirSync(path.join(first, 'demo', 'test-a', 'outputs', 'raw'), { recursive: true }); - writeFileSync(path.join(first, 'demo', 'test-a', 'outputs', 'trace.json'), '{"trace":[]}\n'); + mkdirSync(path.join(first, 'demo', 'test-a', 'outputs'), { recursive: true }); + writeFileSync(path.join(first, 'demo', 'test-a', 'trace.json'), '{"trace":[]}\n'); writeFileSync( - path.join(first, 'demo', 'test-a', 'outputs', 'transcript.jsonl'), + path.join(first, 'demo', 'test-a', 'transcript.jsonl'), `${JSON.stringify({ schema_version: 'agentv.transcript.v1', test_id: 'test-a', @@ -142,11 +143,11 @@ describe('results combine', () => { })}\n`, ); writeFileSync( - path.join(first, 'demo', 'test-a', 'outputs', 'metrics.json'), + path.join(first, 'demo', 'test-a', 'metrics.json'), '{"schema_version":"agentv.metrics.v1"}\n', ); writeFileSync( - path.join(first, 'demo', 'test-a', 'outputs', 'raw', 'provider.log'), + path.join(first, 'demo', 'test-a', 'provider.log'), '{"event":"provider-native"}\n', ); const second = seedRun('run-b', [ @@ -172,37 +173,32 @@ describe('results combine', () => { const [record] = readIndex(combined.manifestPath); expect(record.artifact_dir).toBe('sources/source-1/demo/test-a'); - expect(record.transcript_path).toBe('sources/source-1/demo/test-a/outputs/transcript.jsonl'); - expect(record.metrics_path).toBe('sources/source-1/demo/test-a/outputs/metrics.json'); - expect(record.raw_provider_log_path).toBe( - 'sources/source-1/demo/test-a/outputs/raw/provider.log', - ); + expect(record.trace_path).toBe('sources/source-1/demo/test-a/trace.json'); + expect(record.transcript_path).toBe('sources/source-1/demo/test-a/transcript.jsonl'); + expect(record.metrics_path).toBe('sources/source-1/demo/test-a/metrics.json'); + expect(record.raw_provider_log_path).toBe('sources/source-1/demo/test-a/provider.log'); expect(record.artifact_pointers).toMatchObject({ trace: { - key: 'traces/sources/source-1/demo/test-a/outputs/trace.json', - path: 'sources/source-1/demo/test-a/outputs/trace.json', + key: 'traces/sources/source-1/demo/test-a/trace.json', + path: 'sources/source-1/demo/test-a/trace.json', }, transcript: { - key: 'transcripts/sources/source-1/demo/test-a/outputs/transcript.jsonl', - path: 'sources/source-1/demo/test-a/outputs/transcript.jsonl', + key: 'transcripts/sources/source-1/demo/test-a/transcript.jsonl', + path: 'sources/source-1/demo/test-a/transcript.jsonl', }, }); expect(record.artifact_pointers).not.toHaveProperty('metrics'); + expect(existsSync(path.join(combined.runDir, 'sources/source-1/demo/test-a/trace.json'))).toBe( + true, + ); expect( - existsSync(path.join(combined.runDir, 'sources/source-1/demo/test-a/outputs/trace.json')), - ).toBe(true); - expect( - existsSync( - path.join(combined.runDir, 'sources/source-1/demo/test-a/outputs/transcript.jsonl'), - ), + existsSync(path.join(combined.runDir, 'sources/source-1/demo/test-a/transcript.jsonl')), ).toBe(true); expect( - existsSync(path.join(combined.runDir, 'sources/source-1/demo/test-a/outputs/metrics.json')), + existsSync(path.join(combined.runDir, 'sources/source-1/demo/test-a/metrics.json')), ).toBe(true); expect( - existsSync( - path.join(combined.runDir, 'sources/source-1/demo/test-a/outputs/raw/provider.log'), - ), + existsSync(path.join(combined.runDir, 'sources/source-1/demo/test-a/provider.log')), ).toBe(true); }); diff --git a/apps/cli/test/commands/results/delete.test.ts b/apps/cli/test/commands/results/delete.test.ts index 49357524a..a89cf0dc0 100644 --- a/apps/cli/test/commands/results/delete.test.ts +++ b/apps/cli/test/commands/results/delete.test.ts @@ -24,7 +24,7 @@ describe('results delete', () => { }); function seedRun(runId: string): string { - const runDir = path.join(tempDir, '.agentv', 'results', 'runs', ...runId.split('::')); + const runDir = path.join(tempDir, '.agentv', 'results', ...runId.split('::')); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), @@ -50,7 +50,7 @@ describe('results delete', () => { }); it('resolves and deletes by workspace path', () => { - const runDir = seedRun('2026-06-01T10-00-00-000Z'); + const runDir = seedRun('default::2026-06-01T10-00-00-000Z'); const target = resolveDeleteRunTarget(tempDir, runDir); expect(target.runDir).toBe(runDir); @@ -60,7 +60,7 @@ describe('results delete', () => { }); it('rejects remote IDs and paths outside the local runs directory', () => { - seedRun('2026-06-01T10-00-00-000Z'); + seedRun('default::2026-06-01T10-00-00-000Z'); const outsideDir = path.join(tempDir, 'outside-run'); mkdirSync(outsideDir, { recursive: true }); writeFileSync(path.join(outsideDir, 'index.jsonl'), toJsonl({ score: 1 }), 'utf8'); diff --git a/apps/cli/test/commands/results/export.test.ts b/apps/cli/test/commands/results/export.test.ts index 434c45136..97e5085ae 100644 --- a/apps/cli/test/commands/results/export.test.ts +++ b/apps/cli/test/commands/results/export.test.ts @@ -241,7 +241,7 @@ describe('results export', () => { it('deriveExportRunId keeps retry identity tied to the source run', () => { expect( deriveExportRunId( - path.join(tempDir, '.agentv', 'results', 'runs', 'demo', '2026-run', 'index.jsonl'), + path.join(tempDir, '.agentv', 'results', 'demo', '2026-run', 'index.jsonl'), ), ).toBe('2026-run'); expect(deriveExportRunId(path.join(tempDir, 'legacy-results.jsonl'))).toBe('legacy-results'); @@ -354,9 +354,9 @@ describe('results export', () => { output_path: 'privacy/test-private/outputs/answer.md', answer_path: 'privacy/test-private/outputs/answer.md', response_path: 'privacy/test-private/outputs/response.md', - trace_path: 'privacy/test-private/outputs/trace.json', + trace_path: 'privacy/test-private/trace.json', }); - expect(bundle.entries[0].trace.envelope_ref).toBe('privacy/test-private/outputs/trace.json'); + expect(bundle.entries[0].trace.envelope_ref).toBe('privacy/test-private/trace.json'); expect(bundle.entries[0].trace_envelope.artifacts).toBeDefined(); expect(bundle.entries[0].feedback.grading_path).toBe('privacy/test-private/grading.json'); expect(bundle.entries[0].raw_content).toBeDefined(); @@ -419,7 +419,7 @@ describe('results export', () => { output_path: 'demo/test-greeting/outputs/answer.md', answer_path: 'demo/test-greeting/outputs/answer.md', response_path: 'demo/test-greeting/outputs/response.md', - transcript_path: 'demo/test-greeting/outputs/transcript.jsonl', + transcript_path: 'demo/test-greeting/transcript.jsonl', input_path: 'demo/test-greeting/input.md', }); expect(entries[0].projection_identity).toMatchObject({ diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index 8ec7568a2..f1fead56e 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -60,7 +60,7 @@ ${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''} path } function writeRunArtifacts(projectDir: string): string { - const runDir = path.join(projectDir, '.agentv', 'results', 'runs', 'default', 'run-001'); + const runDir = path.join(projectDir, '.agentv', 'results', 'default', 'run-001'); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), @@ -78,15 +78,15 @@ function sha256Hex(content: Buffer | string): string { } function writeRunArtifactsWithPointers(projectDir: string): string { - const runDir = path.join(projectDir, '.agentv', 'results', 'runs', 'default', 'run-002'); - const outputsDir = path.join(runDir, 'alpha', 'outputs'); - mkdirSync(outputsDir, { recursive: true }); + const runDir = path.join(projectDir, '.agentv', 'results', 'default', 'run-002'); + const artifactDir = path.join(runDir, 'alpha'); + mkdirSync(artifactDir, { recursive: true }); const traceContent = Buffer.from('{"schema_version":"agentv.trace.v1","spans":[]}\n'); const transcriptContent = Buffer.from( '{"schema_version":"agentv.transcript.v1","role":"assistant","content":"ok"}\n', ); - writeFileSync(path.join(outputsDir, 'trace.json'), traceContent); - writeFileSync(path.join(outputsDir, 'transcript.jsonl'), transcriptContent); + writeFileSync(path.join(artifactDir, 'trace.json'), traceContent); + writeFileSync(path.join(artifactDir, 'transcript.jsonl'), transcriptContent); const traceSha = sha256Hex(traceContent); const transcriptSha = sha256Hex(transcriptContent); writeFileSync( @@ -97,9 +97,9 @@ function writeRunArtifactsWithPointers(projectDir: string): string { artifact_pointers: { trace: { ref: AGENTV_RESULTS_ARTIFACTS_REF, - key: 'traces/alpha/outputs/trace.json', + key: 'traces/alpha/trace.json', object_version: `sha256:${traceSha}`, - path: 'alpha/outputs/trace.json', + path: 'alpha/trace.json', sha256: traceSha, size: traceContent.byteLength, schema_version: 'agentv.trace.v1', @@ -108,9 +108,9 @@ function writeRunArtifactsWithPointers(projectDir: string): string { }, transcript: { ref: AGENTV_RESULTS_ARTIFACTS_REF, - key: 'transcripts/alpha/outputs/transcript.jsonl', + key: 'transcripts/alpha/transcript.jsonl', object_version: `sha256:${transcriptSha}`, - path: 'alpha/outputs/transcript.jsonl', + path: 'alpha/transcript.jsonl', sha256: transcriptSha, size: transcriptContent.byteLength, schema_version: 'agentv.transcript.v1', @@ -228,24 +228,24 @@ describe('maybeAutoExportRunArtifacts', () => { ); expect(resultTree).toContain('runs/default/run-002/index.jsonl'); expect(resultTree).toContain('runs/default/run-002/benchmark.json'); - expect(resultTree).not.toContain('runs/default/run-002/alpha/outputs/trace.json'); - expect(resultTree).not.toContain('runs/default/run-002/alpha/outputs/transcript.jsonl'); + expect(resultTree).not.toContain('runs/default/run-002/alpha/trace.json'); + expect(resultTree).not.toContain('runs/default/run-002/alpha/transcript.jsonl'); const index = JSON.parse( git( `git --git-dir "${remoteDir}" show ${resultsBranch}:runs/default/run-002/index.jsonl`, rootDir, ), ); - expect(index.artifact_pointers.trace.key).toBe('runs/default/run-002/alpha/outputs/trace.json'); + expect(index.artifact_pointers.trace.key).toBe('runs/default/run-002/alpha/trace.json'); expect(index.artifact_pointers.transcript.key).toBe( - 'runs/default/run-002/alpha/outputs/transcript.jsonl', + 'runs/default/run-002/alpha/transcript.jsonl', ); const artifactTree = git( `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_ARTIFACTS_REF}`, rootDir, ); - expect(artifactTree).toContain('runs/default/run-002/alpha/outputs/trace.json'); - expect(artifactTree).toContain('runs/default/run-002/alpha/outputs/transcript.jsonl'); + expect(artifactTree).toContain('runs/default/run-002/alpha/trace.json'); + expect(artifactTree).toContain('runs/default/run-002/alpha/transcript.jsonl'); }, 20_000); it('returns already_published when the final results branch is already up to date', async () => { diff --git a/apps/cli/test/commands/results/run-tags.test.ts b/apps/cli/test/commands/results/run-tags.test.ts index 23dabb79c..2c555c259 100644 --- a/apps/cli/test/commands/results/run-tags.test.ts +++ b/apps/cli/test/commands/results/run-tags.test.ts @@ -17,7 +17,7 @@ describe('run tags sidecar', () => { beforeEach(() => { tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-run-tags-')); - const runDir = path.join(tempDir, '.agentv', 'results', 'runs', 'default', '2026-clear-tags'); + const runDir = path.join(tempDir, '.agentv', 'results', 'default', '2026-clear-tags'); mkdirSync(runDir, { recursive: true }); manifestPath = path.join(runDir, 'index.jsonl'); writeFileSync(manifestPath, '{"test_id":"alpha","score":1}\n', 'utf8'); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index b9a69b1fb..7d3287de2 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -93,6 +93,19 @@ function toJsonl(...records: object[]): string { return `${records.map((r) => JSON.stringify(r)).join('\n')}\n`; } +function localResultsExperimentDir(baseDir: string, experiment = 'default'): string { + return path.join(baseDir, '.agentv', 'results', experiment); +} + +function localRunDir(baseDir: string, experiment: string, timestamp: string): string { + return path.join(localResultsExperimentDir(baseDir, experiment), timestamp); +} + +function localRunDirFromRunId(baseDir: string, runId: string): string { + const [experiment, timestamp] = runId.includes('::') ? runId.split('::') : ['default', runId]; + return localRunDir(baseDir, experiment ?? 'default', timestamp ?? runId); +} + function traceSessionEnvelope(input?: { runId?: string; testId?: string; @@ -151,7 +164,7 @@ function traceSessionEnvelope(input?: { version: '1', }, artifacts: { - trace_path: 'outputs/trace.json', + trace_path: 'trace.json', }, }; } @@ -337,7 +350,7 @@ function writeLocalRunArtifact( /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, '$1T$2:$3:$4.$5Z', ); - const runDir = path.join(projectDir, '.agentv', 'results', 'runs', experiment, timestamp); + const runDir = localRunDir(projectDir, experiment, timestamp); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl({ ...resultRecord, experiment })); writeFileSync( @@ -430,14 +443,7 @@ describe('resolveSourceFile', () => { it('still auto-discovers canonical project run workspaces when no source is provided', async () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-serve-source-')); - const runDir = path.join( - tempDir, - '.agentv', - 'results', - 'runs', - 'default', - '2026-06-17T00-00-00-000Z', - ); + const runDir = localRunDir(tempDir, 'default', '2026-06-17T00-00-00-000Z'); mkdirSync(runDir, { recursive: true }); const indexPath = path.join(runDir, 'index.jsonl'); writeFileSync(indexPath, toJsonl(RESULT_A)); @@ -935,7 +941,7 @@ describe('serve app', () => { describe('GET /api/runs', () => { function createLocalRun(baseDir: string, filename: string, ...records: object[]) { - const runDir = path.join(baseDir, '.agentv', 'results', 'runs', filename); + const runDir = localRunDir(baseDir, 'default', filename); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(...records)); } @@ -1035,7 +1041,7 @@ describe('serve app', () => { }); it('tags local runs with source metadata', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-00-00-000Z'; const runDir = path.join(runsDir, filename); @@ -1133,7 +1139,7 @@ describe('serve app', () => { }); it('exposes materialized final state and oplog watermark for local run tags', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-00-00-000Z'; const runDir = path.join(runsDir, filename); @@ -1202,7 +1208,7 @@ describe('serve app', () => { }); it('preserves a local tag clear watermark after DELETE /tags', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-30-00-000Z'; const runDir = path.join(runsDir, filename); @@ -1279,7 +1285,7 @@ describe('serve app', () => { }); it('computes pass_rate using the configured dashboard threshold', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-00-00-000Z'; const runDir = path.join(runsDir, filename); @@ -1300,7 +1306,7 @@ describe('serve app', () => { }); it('reports execution errors separately from quality failures in run summaries', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-00-00-000Z'; const runDir = path.join(runsDir, filename); @@ -1378,7 +1384,7 @@ describe('serve app', () => { }); it('infers the experiment name from the run id when live results have not written it yet', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'issue-1198-live-name'); + const runsDir = localResultsExperimentDir(tempDir, 'issue-1198-live-name'); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T12-00-00-000Z'; const runDir = path.join(runsDir, filename); @@ -1412,7 +1418,6 @@ describe('serve app', () => { 'EntityProcess-agentv-evals', '.agentv', 'results', - 'runs', 'default', '2026-03-26T10-00-00-000Z', ); @@ -1870,7 +1875,7 @@ describe('serve app', () => { it('rejects remote tag edits when the configured results path is not writable', async () => { const plainResultsDir = path.join(tempDir, 'plain-results'); const timestamp = '2026-03-26T13-00-00-000Z'; - const runDir = path.join(plainResultsDir, '.agentv', 'results', 'runs', 'default', timestamp); + const runDir = localRunDir(plainResultsDir, 'default', timestamp); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); @@ -2010,11 +2015,8 @@ describe('serve app', () => { try { const benchmarkDir = path.join(tempDir, 'bench-one'); - const runDir = path.join( + const runDir = localRunDir( benchmarkDir, - '.agentv', - 'results', - 'runs', 'issue-1198-benchmark', '2026-03-25T12-00-00-000Z', ); @@ -2431,7 +2433,7 @@ describe('serve app', () => { }); it('loads results from an existing run file', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-00-00-000Z'; const runDir = path.join(runsDir, filename); @@ -2477,10 +2479,7 @@ describe('serve app', () => { records: object[] = [RESULT_A], opts?: { experiment?: string; tags?: string[]; baseDir?: string }, ): { runId: string; runDir: string; manifestPath: string } { - const runsDir = path.join(opts?.baseDir ?? tempDir, '.agentv', 'results', 'runs'); - const runDir = opts?.experiment - ? path.join(runsDir, opts.experiment, name) - : path.join(runsDir, name); + const runDir = localRunDir(opts?.baseDir ?? tempDir, opts?.experiment ?? 'default', name); mkdirSync(runDir, { recursive: true }); const manifestPath = path.join(runDir, 'index.jsonl'); writeFileSync(manifestPath, toJsonl(...records)); @@ -2529,9 +2528,7 @@ describe('serve app', () => { const detail = (await detailRes.json()) as { results: Array<{ testId: string }> }; expect(detail.results.map((r) => r.testId).sort()).toEqual(['test-greeting', 'test-math']); - const combinedDir = path.dirname( - path.join(tempDir, '.agentv', 'results', 'runs', data.run_id.replace('::', path.sep), 'x'), - ); + const combinedDir = localRunDirFromRunId(tempDir, data.run_id); const tags = JSON.parse(readFileSync(path.join(combinedDir, 'tags.json'), 'utf8')) as { tags: string[]; }; @@ -2667,7 +2664,6 @@ describe('serve app', () => { 'EntityProcess-agentv-evals', '.agentv', 'results', - 'runs', 'default', '2026-06-01T11-00-00-000Z', ); @@ -2704,22 +2700,10 @@ describe('serve app', () => { addProject(otherProjectDir); const first = seedRun('2026-06-01T10-00-00-000Z', [RESULT_A], { baseDir: projectDir }); - const secondRunDir = path.join( - projectDir, - '.agentv', - 'results', - 'runs', - '2026-06-01T11-00-00-000Z', - ); + const secondRunDir = localRunDir(projectDir, 'default', '2026-06-01T11-00-00-000Z'); mkdirSync(secondRunDir, { recursive: true }); writeFileSync(path.join(secondRunDir, 'index.jsonl'), toJsonl(RESULT_B)); - const otherRunDir = path.join( - otherProjectDir, - '.agentv', - 'results', - 'runs', - '2026-06-01T10-00-00-000Z', - ); + const otherRunDir = localRunDir(otherProjectDir, 'default', '2026-06-01T10-00-00-000Z'); mkdirSync(otherRunDir, { recursive: true }); writeFileSync(path.join(otherRunDir, 'index.jsonl'), toJsonl(RESULT_A)); @@ -2733,9 +2717,7 @@ describe('serve app', () => { }); expect(combine.status).toBe(201); expect(existsSync(otherRunDir)).toBe(true); - expect(existsSync(path.join(projectDir, '.agentv', 'results', 'runs', first.runId))).toBe( - true, - ); + expect(existsSync(localRunDirFromRunId(projectDir, first.runId))).toBe(true); } finally { homedirSpy.mockRestore(); } @@ -2748,10 +2730,7 @@ describe('serve app', () => { records: object[] = [RESULT_A], opts?: { experiment?: string; baseDir?: string }, ): { runId: string; runDir: string } { - const runsDir = path.join(opts?.baseDir ?? tempDir, '.agentv', 'results', 'runs'); - const runDir = opts?.experiment - ? path.join(runsDir, opts.experiment, name) - : path.join(runsDir, name); + const runDir = localRunDir(opts?.baseDir ?? tempDir, opts?.experiment ?? 'default', name); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(...records)); writeFileSync(path.join(runDir, 'tags.json'), '{"tags":["stale"]}\n'); @@ -2788,7 +2767,6 @@ describe('serve app', () => { 'EntityProcess-agentv-evals', '.agentv', 'results', - 'runs', 'default', '2026-06-01T11-00-00-000Z', ); @@ -2852,7 +2830,7 @@ describe('serve app', () => { recordOverrides?: Record, ): string { const runId = `${experiment}::${timestamp}`; - const runDir = path.join(projectDir, '.agentv', 'results', 'runs', experiment, timestamp); + const runDir = localRunDir(projectDir, experiment, timestamp); const tracePath = path.join(runDir, traceArtifactPath); mkdirSync(path.dirname(tracePath), { recursive: true }); writeFileSync(tracePath, traceContent); @@ -2869,7 +2847,7 @@ describe('serve app', () => { } it('projects a local AgentV trace sidecar through the Dashboard read model', async () => { - const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const traceArtifactPath = 'demo/test-greeting/trace.json'; const runId = 'with-trace::2026-03-25T09-00-00-000Z'; writeLocalTraceRun( tempDir, @@ -2918,7 +2896,7 @@ describe('serve app', () => { const projectDir = path.join(tempDir, 'project-one'); mkdirSync(path.join(projectDir, '.agentv'), { recursive: true }); const project = addProject(projectDir); - const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const traceArtifactPath = 'demo/test-greeting/trace.json'; const runId = writeLocalTraceRun( projectDir, 'project-trace', @@ -2963,7 +2941,7 @@ describe('serve app', () => { }; expect(traceData.schema_version).toBe('agentv.dashboard.trace_artifact.v1'); expect(traceData.status).toBe('missing'); - expect(traceData.message).toContain('outputs/trace.json'); + expect(traceData.message).toContain('trace.json'); const detailRes = await app.request(`/api/runs/${encodeURIComponent(runId)}`); expect(detailRes.status).toBe(200); @@ -2972,10 +2950,10 @@ describe('serve app', () => { }); it('returns a typed dangling state when the trace pointer cannot be read', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'dangling-trace'); + const runsDir = localResultsExperimentDir(tempDir, 'dangling-trace'); const runId = 'dangling-trace::2026-03-25T10-45-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-45-00-000Z'); - const artifactPath = 'demo/test-greeting/outputs/trace.json'; + const artifactPath = 'demo/test-greeting/trace.json'; mkdirSync(timestampDir, { recursive: true }); writeFileSync( @@ -3004,7 +2982,7 @@ describe('serve app', () => { }); it('normalizes supported OTLP trace artifacts instead of returning unsupported', async () => { - const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const traceArtifactPath = 'demo/test-greeting/trace.json'; const runId = writeLocalTraceRun( tempDir, 'otlp-trace', @@ -3085,7 +3063,7 @@ describe('serve app', () => { it('rejects trace artifact paths that escape the run workspace', async () => { const secret = 'outside trace secret'; writeFileSync(path.join(tempDir, 'outside-trace.json'), secret); - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'escaped-trace'); + const runsDir = localResultsExperimentDir(tempDir, 'escaped-trace'); const runId = 'escaped-trace::2026-03-25T11-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T11-00-00-000Z'); const artifactPath = '../../../../../outside-trace.json'; @@ -3116,10 +3094,10 @@ describe('serve app', () => { describe('GET /api/runs/:filename/evals/:evalId/transcript', () => { it('loads canonical transcript JSONL lazily from the manifest pointer', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'with-transcript'); + const runsDir = localResultsExperimentDir(tempDir, 'with-transcript'); const runId = 'with-transcript::2026-03-25T10-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-00-00-000Z'); - const transcriptArtifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; + const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl'; const answerArtifactPath = 'demo/test-greeting/outputs/answer.md'; const transcriptPath = path.join(timestampDir, transcriptArtifactPath); const answerPath = path.join(timestampDir, answerArtifactPath); @@ -3133,6 +3111,7 @@ describe('serve app', () => { mkdirSync(path.dirname(transcriptPath), { recursive: true }); writeFileSync(transcriptPath, transcriptJsonl); + mkdirSync(path.dirname(answerPath), { recursive: true }); writeFileSync(answerPath, 'Hello, Alice!'); writeFileSync( path.join(timestampDir, 'index.jsonl'), @@ -3177,10 +3156,10 @@ describe('serve app', () => { }); it('loads pointer-shaped transcript metadata when it resolves to a local artifact path', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'pointer-transcript'); + const runsDir = localResultsExperimentDir(tempDir, 'pointer-transcript'); const runId = 'pointer-transcript::2026-03-25T11-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T11-00-00-000Z'); - const artifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; + const artifactPath = 'demo/test-greeting/transcript.jsonl'; const transcriptPath = path.join(timestampDir, artifactPath); const transcriptJsonl = `${JSON.stringify({ test_id: 'test-greeting', @@ -3230,8 +3209,8 @@ describe('serve app', () => { const experiment = 'sidecar-only'; const timestamp = '2026-03-25T15-00-00-000Z'; const runId = `remote::${experiment}::${timestamp}`; - const transcriptArtifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; - const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl'; + const traceArtifactPath = 'demo/test-greeting/trace.json'; const transcriptKey = `runs/${experiment}/${timestamp}/${transcriptArtifactPath}`; const traceKey = `runs/${experiment}/${timestamp}/${traceArtifactPath}`; const transcriptJsonl = `${JSON.stringify({ @@ -3263,10 +3242,12 @@ describe('serve app', () => { artifact_pointers: { trace: { ref: AGENTV_RESULTS_ARTIFACTS_REF, + path: traceArtifactPath, key: traceKey, }, transcript: { ref: AGENTV_RESULTS_ARTIFACTS_REF, + path: transcriptArtifactPath, key: transcriptKey, }, }, @@ -3385,7 +3366,7 @@ describe('serve app', () => { expect(traceRes.headers.get('content-type')).toContain('application/json'); expect(await traceRes.text()).toBe(traceJson); expect(existsSync(path.join(cloneDir, ...traceKey.split('/')))).toBe(false); - }); + }, 15000); it('returns a clear missing state when no transcript pointer is recorded', async () => { const runId = writeLocalRunArtifact( @@ -3403,14 +3384,14 @@ describe('serve app', () => { expect(res.status).toBe(200); const data = (await res.json()) as { status: string; message: string }; expect(data.status).toBe('missing'); - expect(data.message).toContain('outputs/transcript.jsonl'); + expect(data.message).toContain('transcript.jsonl'); }); it('returns a clear dangling state when the transcript pointer cannot be read', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'dangling-transcript'); + const runsDir = localResultsExperimentDir(tempDir, 'dangling-transcript'); const runId = 'dangling-transcript::2026-03-25T13-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T13-00-00-000Z'); - const artifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; + const artifactPath = 'demo/test-greeting/transcript.jsonl'; mkdirSync(timestampDir, { recursive: true }); writeFileSync( @@ -3443,10 +3424,10 @@ describe('serve app', () => { const outsidePath = path.join(tempDir, 'outside-transcript.jsonl'); writeFileSync(outsidePath, secret); - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'escaped-transcript'); + const runsDir = localResultsExperimentDir(tempDir, 'escaped-transcript'); const runId = 'escaped-transcript::2026-03-25T13-30-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T13-30-00-000Z'); - const artifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; + const artifactPath = 'demo/test-greeting/transcript.jsonl'; const symlinkPath = path.join(timestampDir, artifactPath); mkdirSync(path.dirname(symlinkPath), { recursive: true }); @@ -3478,10 +3459,10 @@ describe('serve app', () => { const outsidePath = path.join(tempDir, 'outside-answer.md'); writeFileSync(outsidePath, secret); - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'escaped-answer'); + const runsDir = localResultsExperimentDir(tempDir, 'escaped-answer'); const runId = 'escaped-answer::2026-03-25T13-45-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T13-45-00-000Z'); - const transcriptArtifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; + const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl'; const answerArtifactPath = 'demo/test-greeting/outputs/answer.md'; const transcriptPath = path.join(timestampDir, transcriptArtifactPath); const answerPath = path.join(timestampDir, answerArtifactPath); @@ -3495,6 +3476,7 @@ describe('serve app', () => { mkdirSync(path.dirname(transcriptPath), { recursive: true }); writeFileSync(transcriptPath, transcriptJsonl); + mkdirSync(path.dirname(answerPath), { recursive: true }); symlinkSync(outsidePath, answerPath); writeFileSync( path.join(timestampDir, 'index.jsonl'), @@ -3528,8 +3510,8 @@ describe('serve app', () => { it('does not read transcript or trace bodies for list, detail, or aggregate routes', async () => { const timestamp = '2026-03-25T14-00-00-000Z'; - const transcriptArtifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; - const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl'; + const traceArtifactPath = 'demo/test-greeting/trace.json'; const runId = writeLocalRunArtifact(tempDir, 'lazy-guard', timestamp, { ...RESULT_A, transcript_path: transcriptArtifactPath, @@ -3586,7 +3568,7 @@ describe('serve app', () => { describe('GET /api/runs/:filename/evals/:evalId/files/*', () => { it('loads file content for experiment-scoped run ids', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'with-skills'); + const runsDir = localResultsExperimentDir(tempDir, 'with-skills'); const runId = 'with-skills::2026-03-25T10-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-00-00-000Z'); const responsePath = path.join( @@ -3619,10 +3601,10 @@ describe('serve app', () => { }); it('serves transcript JSONL artifacts as browser-visible raw text and downloads', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'with-transcript'); + const runsDir = localResultsExperimentDir(tempDir, 'with-transcript'); const runId = 'with-transcript::2026-03-25T10-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-00-00-000Z'); - const artifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; + const artifactPath = 'demo/test-greeting/transcript.jsonl'; const transcriptPath = path.join(timestampDir, artifactPath); const transcriptJsonl = `${JSON.stringify({ test_id: 'test-greeting', @@ -3664,7 +3646,7 @@ describe('serve app', () => { const outsidePath = path.join(tempDir, 'outside-response.md'); writeFileSync(outsidePath, secret); - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'escaped-file'); + const runsDir = localResultsExperimentDir(tempDir, 'escaped-file'); const runId = 'escaped-file::2026-03-25T10-30-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-30-00-000Z'); const artifactPath = 'demo/test-greeting/outputs/response.md'; @@ -3698,7 +3680,7 @@ describe('serve app', () => { // Four runs, each in its own run workspace, with the tags documented // below. This setup exercises the OR filter semantics used by // `/api/compare?tags=`. - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const runs: Array<{ @@ -3918,7 +3900,7 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', target: 'gpt-4o', - output: '.agentv/results/runs/2026-05-06T00-00-00-000Z', + output: '.agentv/results/default/2026-05-06T00-00-00-000Z', resume: true, }), }); @@ -3928,7 +3910,7 @@ describe('serve app', () => { expect(res.status).toBe(202); const data = (await res.json()) as { command: string }; expect(data.command).toContain('--resume'); - expect(data.command).toContain('--output .agentv/results/runs/2026-05-06T00-00-00-000Z'); + expect(data.command).toContain('--output .agentv/results/default/2026-05-06T00-00-00-000Z'); }); it('builds --rerun-failed + --output flags from the request', async () => { @@ -3939,14 +3921,14 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', target: 'gpt-4o', - output: 'runs/r1', + output: '.agentv/results/default/r1', rerun_failed: true, }), }); expect(res.status).toBe(202); const data = (await res.json()) as { command: string }; expect(data.command).toContain('--rerun-failed'); - expect(data.command).toContain('--output runs/r1'); + expect(data.command).toContain('--output .agentv/results/default/r1'); }); it('builds --retry-errors from the request', async () => { @@ -3956,12 +3938,12 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - retry_errors: 'runs/r0/index.jsonl', + retry_errors: '.agentv/results/default/r0/index.jsonl', }), }); expect(res.status).toBe(202); const data = (await res.json()) as { command: string }; - expect(data.command).toContain('--retry-errors runs/r0/index.jsonl'); + expect(data.command).toContain('--retry-errors .agentv/results/default/r0/index.jsonl'); }); it('rejects resume + rerun_failed combo with 400', async () => { @@ -3971,7 +3953,7 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - output: 'runs/r1', + output: '.agentv/results/default/r1', resume: true, rerun_failed: true, }), @@ -3988,9 +3970,9 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - output: 'runs/r1', + output: '.agentv/results/default/r1', resume: true, - retry_errors: 'runs/r0/index.jsonl', + retry_errors: '.agentv/results/default/r0/index.jsonl', }), }); expect(res.status).toBe(400); @@ -4004,7 +3986,7 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', resume: true, - output: 'runs/r1', + output: '.agentv/results/default/r1', }), }); expect(res.status).toBe(403); @@ -4018,7 +4000,7 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', resume: true, - output: 'runs/r1', + output: '.agentv/results/default/r1', }), }); expect(res.status).toBe(403); @@ -4086,14 +4068,14 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', target: 'gpt-4o', - output: 'runs/r1', + output: '.agentv/results/default/r1', resume: true, }), }); expect(res.status).toBe(200); const data = (await res.json()) as { command: string }; expect(data.command).toContain('--resume'); - expect(data.command).toContain('--output runs/r1'); + expect(data.command).toContain('--output .agentv/results/default/r1'); expect(data.command).not.toContain('--rerun-failed'); }); @@ -4104,7 +4086,7 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - output: 'runs/r1', + output: '.agentv/results/default/r1', rerun_failed: true, }), }); @@ -4121,12 +4103,12 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - retry_errors: 'runs/r0/index.jsonl', + retry_errors: '.agentv/results/default/r0/index.jsonl', }), }); expect(res.status).toBe(200); const data = (await res.json()) as { command: string }; - expect(data.command).toContain('--retry-errors runs/r0/index.jsonl'); + expect(data.command).toContain('--retry-errors .agentv/results/default/r0/index.jsonl'); }); }); @@ -4139,7 +4121,7 @@ describe('serve app', () => { describe('GET /api/runs/:filename (resume metadata)', () => { it('includes run_dir and suite_filter for local runs with benchmark.json', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-05-06T00-00-00-000Z'; const runDir = path.join(runsDir, filename); @@ -4172,12 +4154,12 @@ describe('serve app', () => { source: 'local' | 'remote'; }; expect(data.source).toBe('local'); - expect(data.run_dir).toBe(path.join('.agentv', 'results', 'runs', filename)); + expect(data.run_dir).toBe(path.join('.agentv', 'results', 'default', filename)); expect(data.suite_filter).toBe('examples/demo.eval.yaml'); }); it('omits suite_filter when benchmark.json is missing', async () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-05-06T00-00-01-000Z'; const runDir = path.join(runsDir, filename); diff --git a/apps/cli/test/commands/results/shared.test.ts b/apps/cli/test/commands/results/shared.test.ts index 5e701116e..9eaab0a6e 100644 --- a/apps/cli/test/commands/results/shared.test.ts +++ b/apps/cli/test/commands/results/shared.test.ts @@ -22,7 +22,7 @@ describe('results shared source resolution', () => { }); it('resolves an explicit run workspace directory to index.jsonl', async () => { - const runDir = path.join(tempDir, '.agentv', 'results', 'runs', '2026-03-25T10-00-00-000Z'); + const runDir = path.join(tempDir, '.agentv', 'results', 'default', '2026-03-25T10-00-00-000Z'); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id":"t1","score":1}\n'); @@ -36,14 +36,14 @@ describe('results shared source resolution', () => { tempDir, '.agentv', 'results', - 'runs', + 'default', '2026-03-24T10-00-00-000Z', ); const newerRunDir = path.join( tempDir, '.agentv', 'results', - 'runs', + 'default', '2026-03-25T10-00-00-000Z', ); mkdirSync(olderRunDir, { recursive: true }); @@ -80,9 +80,9 @@ describe('results shared source resolution', () => { }); it('hydrates transcripts from artifact pointers when transcript_path is absent', () => { - const runDir = path.join(tempDir, '.agentv', 'results', 'runs', '2026-03-25T10-00-00-000Z'); - const transcriptRelativePath = 'pointer-case/outputs/transcript.jsonl'; - mkdirSync(path.join(runDir, 'pointer-case', 'outputs'), { recursive: true }); + const runDir = path.join(tempDir, '.agentv', 'results', 'default', '2026-03-25T10-00-00-000Z'); + const transcriptRelativePath = 'pointer-case/transcript.jsonl'; + mkdirSync(path.join(runDir, 'pointer-case'), { recursive: true }); writeFileSync( path.join(runDir, transcriptRelativePath), `${JSON.stringify({ @@ -108,7 +108,7 @@ describe('results shared source resolution', () => { artifact_pointers: { transcript: { ref: 'agentv/artifacts/v1', - key: 'transcripts/pointer-case/outputs/transcript.jsonl', + key: 'transcripts/pointer-case/transcript.jsonl', object_version: 'sha256:test', path: transcriptRelativePath, sha256: 'test', @@ -129,7 +129,7 @@ describe('results shared source resolution', () => { }); it('rejects eval-case-only rows with migration guidance', () => { - const runDir = path.join(tempDir, '.agentv', 'results', 'runs', '2026-03-25T10-00-00-000Z'); + const runDir = path.join(tempDir, '.agentv', 'results', 'default', '2026-03-25T10-00-00-000Z'); mkdirSync(runDir, { recursive: true }); const indexPath = path.join(runDir, 'index.jsonl'); writeFileSync(indexPath, '{"id":"case-a","prompt":"What is 2 + 2?"}\n'); diff --git a/apps/cli/test/commands/results/validate.test.ts b/apps/cli/test/commands/results/validate.test.ts index c418ab8e6..5ebdd0742 100644 --- a/apps/cli/test/commands/results/validate.test.ts +++ b/apps/cli/test/commands/results/validate.test.ts @@ -6,7 +6,7 @@ import path from 'node:path'; import { validateRunDirectory } from '../../../src/commands/results/validate.js'; describe('results validate', () => { - it('accepts experiment-scoped canonical run directories without layout warnings', () => { + it('accepts experiment-scoped result directories without layout warnings', () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-validate-test-')); try { @@ -14,7 +14,6 @@ describe('results validate', () => { tempDir, '.agentv', 'results', - 'runs', 'with-skills', '2026-03-27T12-42-24-429Z', ); @@ -34,7 +33,7 @@ describe('results validate', () => { expect(diagnostics.filter((d) => d.severity === 'error')).toEqual([]); expect(diagnostics.map((d) => d.message)).not.toContain( - "Directory is not under a 'runs/' tree. Expected: .agentv/results/runs//", + 'Directory is not under the canonical results tree. Expected: .agentv/results//', ); expect( diagnostics.some((d) => d.message.includes('does not match the expected pattern')), diff --git a/apps/cli/test/commands/trace/trace.test.ts b/apps/cli/test/commands/trace/trace.test.ts index d0db2cbcb..21f36cc17 100644 --- a/apps/cli/test/commands/trace/trace.test.ts +++ b/apps/cli/test/commands/trace/trace.test.ts @@ -265,8 +265,8 @@ describe('trace utils', () => { expect(metas).toEqual([]); }); - it('should enumerate run workspaces in .agentv/results/runs/', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + it('should enumerate run workspaces in .agentv/results/default/', () => { + const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); mkdirSync(runsDir, { recursive: true }); const olderRunDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); @@ -309,8 +309,30 @@ describe('trace utils', () => { expect(metas).toEqual([]); }); - it('should respect limit', () => { + it('should skip deprecated results/runs workspaces by default', () => { const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); + mkdirSync(runDir, { recursive: true }); + writeFileSync(path.join(runDir, 'index.jsonl'), `${RESULT_WITH_TRACE}\n`); + + const metas = listResultFiles(tempDir); + + expect(metas).toEqual([]); + }); + + it('should skip local metadata workspaces by default', () => { + const metadataDir = path.join(tempDir, '.agentv', 'results', 'metadata'); + const runDir = path.join(metadataDir, 'default', '2026-02-20T21-38-05-833Z'); + mkdirSync(runDir, { recursive: true }); + writeFileSync(path.join(runDir, 'index.jsonl'), `${RESULT_WITH_TRACE}\n`); + + const metas = listResultFiles(tempDir); + + expect(metas).toEqual([]); + }); + + it('should respect limit', () => { + const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); mkdirSync(runsDir, { recursive: true }); const olderRunDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); @@ -325,8 +347,8 @@ describe('trace utils', () => { expect(metas[0].filename).toBe('2026-02-21T10-00-00-000Z'); }); - it('should ignore non-directory entries in runs/', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + it('should ignore non-directory entries in results experiments', () => { + const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); mkdirSync(runsDir, { recursive: true }); writeFileSync(path.join(runsDir, 'notes.txt'), 'not a result file'); @@ -336,8 +358,8 @@ describe('trace utils', () => { expect(metas).toHaveLength(0); }); - it('should discover index.jsonl inside run directories in runs/', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + it('should discover index.jsonl inside grouped run directories', () => { + const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); const runDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); mkdirSync(runDir, { recursive: true }); @@ -355,7 +377,7 @@ describe('trace utils', () => { }); it('should discover nested experiment run directories and emit safe run ids', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = path.join(tempDir, '.agentv', 'results'); const runDir = path.join(runsDir, 'with-skills', '2026-02-20T21-38-05-833Z'); mkdirSync(runDir, { recursive: true }); @@ -369,7 +391,7 @@ describe('trace utils', () => { }); it('should use benchmark metadata display names when listing run workspaces', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = path.join(tempDir, '.agentv', 'results'); const runDir = path.join(runsDir, 'combined', '2026-02-20T21-38-05-833Z'); mkdirSync(runDir, { recursive: true }); @@ -391,7 +413,7 @@ describe('trace utils', () => { }); it('should skip directories without index.jsonl', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); + const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); const emptyDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); mkdirSync(emptyDir, { recursive: true }); diff --git a/apps/cli/test/commands/trend/trend.test.ts b/apps/cli/test/commands/trend/trend.test.ts index 0b6c229d7..db1a6b2bc 100644 --- a/apps/cli/test/commands/trend/trend.test.ts +++ b/apps/cli/test/commands/trend/trend.test.ts @@ -35,7 +35,7 @@ async function createRunWorkspace( runName: string, records: readonly RunRecordInput[], ): Promise<{ runDir: string; indexPath: string }> { - const runDir = path.join(rootDir, '.agentv', 'results', 'runs', runName); + const runDir = path.join(rootDir, '.agentv', 'results', 'default', runName); await mkdir(runDir, { recursive: true }); const indexPath = path.join(runDir, 'index.jsonl'); await writeFile( diff --git a/apps/cli/test/unit/run-cache.test.ts b/apps/cli/test/unit/run-cache.test.ts index 11496d054..8109e5934 100644 --- a/apps/cli/test/unit/run-cache.test.ts +++ b/apps/cli/test/unit/run-cache.test.ts @@ -30,7 +30,7 @@ describe('resolveCachedRunDir', () => { it('returns the cached run dir when it exists on disk', async () => { const cwd = setupCwd(); - const runDir = path.join(cwd, '.agentv', 'results', 'runs', 'default', '2026-01-01'); + const runDir = path.join(cwd, '.agentv', 'results', 'default', '2026-01-01'); mkdirSync(runDir, { recursive: true }); writeCache(cwd, runDir); @@ -53,7 +53,7 @@ describe('resolveCachedRunDir', () => { it('returns undefined when the cached dir has been deleted', async () => { const cwd = setupCwd(); - const staleDir = path.join(cwd, '.agentv', 'results', 'runs', 'default', '2026-01-01'); + const staleDir = path.join(cwd, '.agentv', 'results', 'default', '2026-01-01'); writeCache(cwd, staleDir); expect(await resolveCachedRunDir(cwd)).toBeUndefined(); diff --git a/apps/dashboard/src/components/EvalDetail.tsx b/apps/dashboard/src/components/EvalDetail.tsx index c9c8059a4..90d3d1494 100644 --- a/apps/dashboard/src/components/EvalDetail.tsx +++ b/apps/dashboard/src/components/EvalDetail.tsx @@ -493,7 +493,7 @@ function TranscriptTab({

No structured transcript

{transcriptData?.message ?? - 'This run does not include canonical outputs/transcript.jsonl. Dashboard does not parse response.md or markdown transcripts for this view.'} + 'This run does not include canonical transcript.jsonl. Dashboard does not parse response.md or markdown transcripts for this view.'}

); diff --git a/apps/dashboard/src/components/TranscriptTimeline.tsx b/apps/dashboard/src/components/TranscriptTimeline.tsx index 409f957b2..eab1f3d5e 100644 --- a/apps/dashboard/src/components/TranscriptTimeline.tsx +++ b/apps/dashboard/src/components/TranscriptTimeline.tsx @@ -1,5 +1,5 @@ /** - * Structured transcript viewer for canonical `outputs/transcript.jsonl` files. + * Structured transcript viewer for canonical `transcript.jsonl` files. * * The component intentionally reads only transcript JSONL rows derived from * AgentV trace data. It does not parse `response.md` or markdown transcripts; @@ -175,7 +175,7 @@ function findFilePathBySuffix( } export function findTranscriptPath(nodes: readonly FileNode[]): string | undefined { - return findFilePathBySuffix(nodes, ['outputs/transcript.jsonl']); + return findFilePathBySuffix(nodes, ['transcript.jsonl']); } export function findAnswerPath(nodes: readonly FileNode[]): string | undefined { diff --git a/apps/dashboard/src/components/__fixtures__/structured-transcript.ts b/apps/dashboard/src/components/__fixtures__/structured-transcript.ts index 9009cd6e7..93f37e21b 100644 --- a/apps/dashboard/src/components/__fixtures__/structured-transcript.ts +++ b/apps/dashboard/src/components/__fixtures__/structured-transcript.ts @@ -99,7 +99,7 @@ export const structuredTranscriptFiles: FileNode[] = [ }, { name: 'transcript.jsonl', - path: 'final-json-answer__codex/outputs/transcript.jsonl', + path: 'final-json-answer__codex/transcript.jsonl', type: 'file', }, { diff --git a/apps/dashboard/src/components/resume-run-helpers.test.ts b/apps/dashboard/src/components/resume-run-helpers.test.ts index 62ed94416..70991808f 100644 --- a/apps/dashboard/src/components/resume-run-helpers.test.ts +++ b/apps/dashboard/src/components/resume-run-helpers.test.ts @@ -66,13 +66,13 @@ describe('buildResumeRequestBody', () => { expect( buildResumeRequestBody({ mode: 'resume', - runDir: '.agentv/results/runs/2026-05-06T00-00-00-000Z', + runDir: '.agentv/results/default/2026-05-06T00-00-00-000Z', suiteFilter: 'examples/demo.eval.yaml', target: 'gpt-4o', }), ).toEqual({ suite_filter: 'examples/demo.eval.yaml', - output: '.agentv/results/runs/2026-05-06T00-00-00-000Z', + output: '.agentv/results/default/2026-05-06T00-00-00-000Z', target: 'gpt-4o', resume: true, }); diff --git a/apps/dashboard/src/components/transcript-timeline.test.tsx b/apps/dashboard/src/components/transcript-timeline.test.tsx index 82405de45..98bee480b 100644 --- a/apps/dashboard/src/components/transcript-timeline.test.tsx +++ b/apps/dashboard/src/components/transcript-timeline.test.tsx @@ -43,7 +43,7 @@ describe('TranscriptTimeline', () => { it('finds canonical transcript and answer artifacts without selecting response.md', () => { expect(findTranscriptPath(structuredTranscriptFiles)).toBe( - 'final-json-answer__codex/outputs/transcript.jsonl', + 'final-json-answer__codex/transcript.jsonl', ); expect(findAnswerPath(structuredTranscriptFiles)).toBe( 'final-json-answer__codex/outputs/answer.md', @@ -57,7 +57,7 @@ describe('TranscriptTimeline', () => { entries={parsed.entries} finalAnswer={'{"answer":42,"source":"src/app.ts"}'} answerPath="final-json-answer__codex/outputs/answer.md" - transcriptPath="final-json-answer__codex/outputs/transcript.jsonl" + transcriptPath="final-json-answer__codex/transcript.jsonl" answerHref="/api/raw-answer" transcriptHref="/api/raw-transcript" transcriptDownloadHref="/api/download-transcript" diff --git a/apps/dashboard/src/lib/__fixtures__/trace-session-read-model.ts b/apps/dashboard/src/lib/__fixtures__/trace-session-read-model.ts index d62895d3f..062df00fc 100644 --- a/apps/dashboard/src/lib/__fixtures__/trace-session-read-model.ts +++ b/apps/dashboard/src/lib/__fixtures__/trace-session-read-model.ts @@ -119,9 +119,9 @@ export const traceSessionEnvelopeFixture = { }, }, artifacts: { - trace_path: 'outputs/trace.json', + trace_path: 'trace.json', answer_path: 'outputs/answer.md', - transcript_path: 'outputs/transcript.jsonl', + transcript_path: 'transcript.jsonl', secret_token_path: 'outputs/secret-token.txt', unsafe_url_path: 'https://phoenix.example/artifacts/trace.json?api_key=secret', traversal_path: '../outside/trace.json', diff --git a/apps/dashboard/src/lib/trace-read-model.test.ts b/apps/dashboard/src/lib/trace-read-model.test.ts index 2f19aa46e..67b34fa2d 100644 --- a/apps/dashboard/src/lib/trace-read-model.test.ts +++ b/apps/dashboard/src/lib/trace-read-model.test.ts @@ -35,7 +35,7 @@ function flattenTree(nodes: readonly TraceSpanNode[]): TraceSpanNode[] { describe('trace session read model', () => { it('projects snake_case trace artifacts into stable Dashboard span trees', () => { const session = traceEnvelopeToTraceSessionResponse(traceSessionEnvelopeFixture, { - artifactPath: 'nested-session__codex/outputs/trace.json', + artifactPath: 'nested-session__codex/trace.json', }); const tree = buildTraceSpanTree(session.spans); @@ -47,12 +47,12 @@ describe('trace session read model', () => { trace_id: 'trace-123', root_span_id: 'root-span', source: { - artifact_path: 'nested-session__codex/outputs/trace.json', + artifact_path: 'nested-session__codex/trace.json', }, artifact_links: [ { name: 'answer_path', path: 'outputs/answer.md' }, - { name: 'trace_path', path: 'outputs/trace.json' }, - { name: 'transcript_path', path: 'outputs/transcript.jsonl' }, + { name: 'trace_path', path: 'trace.json' }, + { name: 'transcript_path', path: 'transcript.jsonl' }, ], }); expect(session.spans.map((span) => span.id)).toEqual([ diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index d838d1a0d..22efc61e1 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -11,7 +11,7 @@ sidebar: agentv eval evals/my-eval.yaml ``` -Results are written to `.agentv/results/runs//index.jsonl`. Each line is a JSON object with one result per test case, and the run workspace also stores the manifest and related artifacts. Use this generated run folder as the portable audit surface: copy or sync the run directory, not a hand-authored parallel bundle. +Results are written to `.agentv/results///index.jsonl`. When no experiment is defined, AgentV uses `.agentv/results/default//index.jsonl`. Each line is a JSON object with one result per test case, and the run workspace also stores the manifest and related artifacts. Use this generated run folder as the portable audit surface: copy or sync the run directory, not a hand-authored parallel bundle. Each `scores[]` entry includes per-grader timing: @@ -150,7 +150,7 @@ result-oriented workflows. For full-fidelity span inspection, export OTLP JSON e ```bash # Summary-level inspection from the run manifest -agentv inspect stats .agentv/results/runs//index.jsonl +agentv inspect stats .agentv/results/default//index.jsonl # Full-fidelity OTLP JSON trace (importable by OTel backends like Jaeger, Grafana) agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json @@ -326,13 +326,13 @@ AgentV ships three flags for picking up a partial run. They differ only in **whi agentv eval evals/my-eval.yaml --resume # Or target a specific run dir explicitly -agentv eval evals/my-eval.yaml --output .agentv/results/runs/ --resume +agentv eval evals/my-eval.yaml --output .agentv/results/default/ --resume # Re-run errors AND failed cases against the last run dir agentv eval evals/my-eval.yaml --rerun-failed # Re-run only execution errors from any prior run by path -agentv eval evals/my-eval.yaml --retry-errors .agentv/results/runs//index.jsonl +agentv eval evals/my-eval.yaml --retry-errors .agentv/results/default//index.jsonl ``` After any failing run, the CLI prints the exact `--rerun-failed` command for the run dir that just completed — copy/paste it. If the process or pod disappeared before you could access the local run directory and results auto-push was enabled, recover the partial run from [WIP checkpoints](/docs/tools/wip-checkpoints/) first, then use the same `--resume` flow. @@ -434,10 +434,11 @@ See the [Import tool docs](/docs/tools/import/) for all providers and options. ## Transcript And Trace Artifacts -Each result row's `artifact_dir` can include both `outputs/trace.json` and -`outputs/transcript.jsonl`. The run root does not contain a mixed transcript -artifact; use each index row's `transcript_path` to find the per-result -transcript. +Each result row's `artifact_dir` is a case-local folder under the timestamped +run bundle. It can include `trace.json`, `transcript.jsonl`, `provider.log`, +`grading.json`, `timing.json`, `metrics.json`, and generated outputs under +`outputs/`. The run root does not contain a mixed transcript artifact; use each +index row's `transcript_path` to find the per-result transcript. Rows also include `artifact_pointers` for AgentV-owned artifact storage. Pointer entries such as `artifact_pointers.trace` and `artifact_pointers.transcript` @@ -460,11 +461,11 @@ pointer/backend contract instead of using Git LFS as the core abstraction so S3, B2, or other object stores can use the same `key`, `object_version`, `sha256`, `size`, `media_type`, and `schema_version` fields later. -`outputs/trace.json` is the full-fidelity `agentv.trace.v1` sidecar. +`trace.json` is the full-fidelity `agentv.trace.v1` sidecar. It stores the canonical span graph, source metadata, capture/redaction policy, conversion warnings, score provenance, and opaque evidence references. -`outputs/transcript.jsonl` is the canonical AgentV transcript/timeline artifact. +`transcript.jsonl` is the canonical AgentV transcript/timeline artifact. It uses provider-neutral `agentv.transcript.v1` rows with stable top-level fields for message order, role/content, tool calls and paired results, timing, token usage, cost, source metadata, capture state, and trace pointers. @@ -473,7 +474,7 @@ Provider-native payloads can appear only inside opaque nested fields such as When an agent provider captures a native stream or session log, the result row may also include `raw_provider_log_path`, pointing at -`outputs/raw/provider.log`. That file is raw evidence copied byte-for-byte from +`provider.log`. That file is raw evidence copied byte-for-byte from the provider log and is not parsed, normalized, or required for replay, import, Agent Skills conversion, or grading. AgentV does not write or maintain a parallel `outputs/transcript.json` source of truth. diff --git a/apps/web/src/content/docs/docs/getting-started/quickstart.mdx b/apps/web/src/content/docs/docs/getting-started/quickstart.mdx index 6a32e5a6b..91f2f937d 100644 --- a/apps/web/src/content/docs/docs/getting-started/quickstart.mdx +++ b/apps/web/src/content/docs/docs/getting-started/quickstart.mdx @@ -66,7 +66,7 @@ tests: agentv eval ./evals/example.yaml ``` -Results appear in `.agentv/results/runs//index.jsonl` with scores, reasoning, and execution traces. +Results appear in `.agentv/results/default//index.jsonl` with scores, reasoning, and execution traces. ## Next Steps diff --git a/apps/web/src/content/docs/docs/guides/autoresearch.mdx b/apps/web/src/content/docs/docs/guides/autoresearch.mdx index f5f23362a..963ee7d08 100644 --- a/apps/web/src/content/docs/docs/guides/autoresearch.mdx +++ b/apps/web/src/content/docs/docs/guides/autoresearch.mdx @@ -76,7 +76,7 @@ No CLI flags or YAML schema changes needed — the skill handles everything. Each autoresearch session creates a self-contained experiment directory: ``` -.agentv/results/runs/autoresearch-/ +.agentv/results/autoresearch-/ ├── _autoresearch/ │ ├── iterations.jsonl # Per-cycle data (score, decision, mutation) │ └── trajectory.html # Live-updating Chart.js visualization diff --git a/apps/web/src/content/docs/docs/integrations/phoenix.mdx b/apps/web/src/content/docs/docs/integrations/phoenix.mdx index 3fec46b37..748513fc3 100644 --- a/apps/web/src/content/docs/docs/integrations/phoenix.mdx +++ b/apps/web/src/content/docs/docs/integrations/phoenix.mdx @@ -35,7 +35,7 @@ Use Dashboard for AgentV-owned run and trace review: agentv dashboard ``` -Dashboard reads configured project run sources, local `.agentv/results/runs/` +Dashboard reads configured project run sources, local `.agentv/results/` workspaces, remote results repositories, trace sidecars, transcripts, and artifact manifests. It does not require Phoenix, the `px` CLI, Phoenix database tables, or any Phoenix runtime process. diff --git a/apps/web/src/content/docs/docs/tools/compare.mdx b/apps/web/src/content/docs/docs/tools/compare.mdx index 4923846f0..bafb85722 100644 --- a/apps/web/src/content/docs/docs/tools/compare.mdx +++ b/apps/web/src/content/docs/docs/tools/compare.mdx @@ -12,10 +12,10 @@ The `compare` command computes deltas between two evaluation runs for A/B testin Run two evaluations and compare them: ```bash -agentv eval evals/my-eval.yaml --output runs/before +agentv eval evals/my-eval.yaml --output .agentv/results/default/before # ... make changes to your agent ... -agentv eval evals/my-eval.yaml --output runs/after -agentv compare runs/before/index.jsonl runs/after/index.jsonl +agentv eval evals/my-eval.yaml --output .agentv/results/default/after +agentv compare .agentv/results/default/before/index.jsonl .agentv/results/default/after/index.jsonl ``` ## Options @@ -123,13 +123,13 @@ Compare different model versions: ```bash # Run baseline evaluation -agentv eval evals/*.yaml --target gpt-4 --output runs/baseline +agentv eval evals/*.yaml --target gpt-4 --output .agentv/results/default/baseline # Run candidate evaluation -agentv eval evals/*.yaml --target gpt-4o --output runs/candidate +agentv eval evals/*.yaml --target gpt-4o --output .agentv/results/default/candidate # Compare results -agentv compare runs/baseline/index.jsonl runs/candidate/index.jsonl +agentv compare .agentv/results/default/baseline/index.jsonl .agentv/results/default/candidate/index.jsonl ``` ### Prompt Optimization @@ -138,13 +138,13 @@ Compare before/after prompt changes: ```bash # Run with original prompt -agentv eval evals/*.yaml --output runs/before +agentv eval evals/*.yaml --output .agentv/results/default/before # Modify prompt, then run again -agentv eval evals/*.yaml --output runs/after +agentv eval evals/*.yaml --output .agentv/results/default/after # Compare with strict threshold -agentv compare runs/before/index.jsonl runs/after/index.jsonl --threshold 0.05 +agentv compare .agentv/results/default/before/index.jsonl .agentv/results/default/after/index.jsonl --threshold 0.05 ``` ### CI Quality Gate diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 9fedb9ae1..195771902 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -31,7 +31,7 @@ The `dashboard` command launches a web-based dashboard for browsing evaluation r agentv dashboard ``` -Dashboard auto-discovers run workspaces from `.agentv/results/runs/` in the current directory and opens at `http://localhost:3117`. +Dashboard auto-discovers run workspaces from `.agentv/results///` in the current directory and opens at `http://localhost:3117`. Runs without an explicit experiment use `.agentv/results/default//`. To open a different project, pass the project root with `--dir`: @@ -39,7 +39,7 @@ To open a different project, pass the project root with `--dir`: agentv dashboard --dir /path/to/project ``` -Dashboard does not accept a run workspace directory or `index.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/runs/` tree, plus an external results repository or run directory configured under `results:` in YAML. For one-off inspection of a copied run bundle, use `agentv results report `. +Dashboard does not accept a run workspace directory or `index.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/` tree, plus an external results repository or run directory configured under `results:` in YAML. The old `.agentv/results/runs/**` layout is not a Dashboard-visible layout. For one-off inspection of a copied run bundle, use `agentv results report `. ## Data boundary @@ -163,7 +163,7 @@ Select 2+ rows with the checkboxes and click the sticky **Compare N** action to Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `index.jsonl` in the run workspace, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to record an empty tag state with an operation watermark. -Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `.agentv/results/metadata/runs/.../tags.json` in the configured results repo clone. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. +Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `metadata/runs/.../tags.json` in the configured results repo clone/branch. That overlay path is a remote-results implementation detail, not part of the local `.agentv/results///` layout. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. Use tags to annotate ad-hoc variants, experiment cross-cuts, or status flags you didn't plan for up front — `baseline`, `v2-prompt`, `slow`, `after-retry-fix`, `regression`, etc. Unlike `experiment` — which groups runs and is baked into the JSONL at eval-run time — tags are mutable, multi-valued, and never touch the original run data. @@ -381,7 +381,7 @@ Each run writes to a unique timestamped directory, so concurrent pushes from mul ### What happens to existing local runs? -Existing runs already present under `.agentv/results/runs/` stay exactly where they are and continue to appear in Dashboard as **local** runs. +Existing runs already present under `.agentv/results///` stay exactly where they are and continue to appear in Dashboard as **local** runs. Runs in the removed `.agentv/results/runs/**` layout are not discovered by Dashboard. Adding a `results` block does **not** backfill those historical runs into the results branch automatically. Result publishing only affects runs created after the results repo is configured. `sync.auto_push` controls network push and best-effort WIP checkpoints for in-progress `agentv eval` runs. @@ -417,7 +417,7 @@ After sync, newly fetched remote runs appear in the list with a **remote** sourc **Sync Project** fetches the results repo and only changes the clone when Git says it is safe: - A clean clone that is behind the remote is fast-forwarded. -- Safe uncommitted changes under `.agentv/results/**`, such as remote tag metadata overlays, are committed and pushed when `sync.auto_push: true`. +- Safe uncommitted changes under the configured results repo's owned result and metadata paths, such as remote tag overlays under `metadata/runs/**`, are committed and pushed when `sync.auto_push: true`. - A local results repo that is ahead is pushed when `sync.auto_push: true` and the committed paths are all under `.agentv/results/**`. - Dirty non-results files, dirty metadata plus remote changes, diverged history, unresolved conflicts, missing upstream branches, non-results commits ahead, and rejected pushes are blocked instead of reset. diff --git a/apps/web/src/content/docs/docs/tools/import.mdx b/apps/web/src/content/docs/docs/tools/import.mdx index 86ff592aa..ad23a6303 100644 --- a/apps/web/src/content/docs/docs/tools/import.mdx +++ b/apps/web/src/content/docs/docs/tools/import.mdx @@ -174,8 +174,8 @@ row keys. Rows without `schema_version`, `capture`, or `trace` from older AgentV transcript exports remain replayable. New eval run artifacts write the v1 shape. -For eval run artifacts, `outputs/transcript.jsonl` is derived from -`outputs/trace.json`; it is a portable message/event projection, not a second +For eval run artifacts, `transcript.jsonl` is derived from +`trace.json`; it is a portable message/event projection, not a second canonical trace source or a provider-native session dump. Provider-native session or stream logs, when captured during an eval run, are separate raw evidence artifacts referenced by `raw_provider_log_path`; Agent Skills import, diff --git a/apps/web/src/content/docs/docs/tools/inspect.mdx b/apps/web/src/content/docs/docs/tools/inspect.mdx index c7c66c9cf..bbac42fbd 100644 --- a/apps/web/src/content/docs/docs/tools/inspect.mdx +++ b/apps/web/src/content/docs/docs/tools/inspect.mdx @@ -19,7 +19,7 @@ For full tool-call inspection, prefer OTLP JSON exports over eval manifests. ### `inspect list` -Enumerate canonical evaluation run workspaces from `.agentv/results/runs/`. +Enumerate canonical evaluation run workspaces from `.agentv/results/`. ```bash agentv inspect list [--limit N] [--format json|table] @@ -94,7 +94,7 @@ agentv inspect show trace.otlp.json --format json \ | jq '[.[] | select(.cost_usd > 0.10) | {test_id, score, cost: .cost_usd}]' # Compare providers -agentv inspect stats .agentv/results/runs//index.jsonl --group-by target --format json \ +agentv inspect stats .agentv/results/default//index.jsonl --group-by target --format json \ | jq '.groups[] | {label, score_mean: .metrics.score.mean}' ``` diff --git a/apps/web/src/content/docs/docs/tools/prepare.mdx b/apps/web/src/content/docs/docs/tools/prepare.mdx index 975ae20b2..47defa41e 100644 --- a/apps/web/src/content/docs/docs/tools/prepare.mdx +++ b/apps/web/src/content/docs/docs/tools/prepare.mdx @@ -32,7 +32,7 @@ After the human or external agent finishes editing files in `workspace/`, grade agentv grade evals/foo.eval.yaml \ --test-id case-1 \ --prepared /tmp/agentv-case-1 \ - --output .agentv/results/runs/manual-case-1 + --output .agentv/results/manual-case-1 ``` `grade` reads `agentv_prepare.json`, verifies it matches the eval/test, captures workspace changes from the prepared baseline when available, and runs the eval's graders against the final workspace. The target provider is not invoked. @@ -61,7 +61,7 @@ Supported `--trace` inputs: | Format | Typical source | |--------|----------------| -| `agentv.trace.v1` JSON or JSONL | `outputs/trace.json` from an AgentV run or replay/export workflow | +| `agentv.trace.v1` JSON or JSONL | `trace.json` from an AgentV run or replay/export workflow | | AgentV transcript JSONL | `agentv import claude`, `agentv import codex`, or `agentv import copilot` output | Single-record trace files are accepted directly. Multi-record files are matched by `test_id` and target. The selected trace is projected into AgentV's normal `trace` and `messages` grader context, so `tool-trajectory`, execution-metrics, and code graders receive the same shape they see during eval runs. diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index 6d49909c4..10c27873b 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -40,10 +40,10 @@ Examples: ```bash # Generate report.html next to the run manifest -agentv results report .agentv/results/runs/2026-03-14T10-32-00_claude +agentv results report .agentv/results/default/2026-03-14T10-32-00_claude # Use an explicit output path -agentv results report .agentv/results/runs/2026-03-14T10-32-00_claude/index.jsonl \ +agentv results report .agentv/results/default/2026-03-14T10-32-00_claude/index.jsonl \ --out ./reports/human-review.html ``` @@ -62,17 +62,17 @@ One minimal publication workflow is: ```bash # 1. Run an eval and sync or copy the run workspace into your public results repo. -agentv eval evals/demo.eval.yaml --output .agentv/results/runs/demo-live +agentv eval evals/demo.eval.yaml --output .agentv/results/demo-live # 2. In the public results repo, render the report into the Pages source directory. -agentv results report .agentv/results/runs/demo-live --out docs/index.html +agentv results report .agentv/results/demo-live --out docs/index.html # 3. Review the generated HTML before publishing. grep -RInE 'sk-[A-Za-z0-9]|Bearer |localhost|127\.0\.0\.1|/home/|/Users/|/tmp/' docs/index.html # 4. Commit the run artifacts and docs/index.html, then enable GitHub Pages # for the repository's docs/ directory or the branch used for Pages. -git add .agentv/results/runs/demo-live docs/index.html README.md +git add .agentv/results/demo-live docs/index.html README.md git commit -m "docs(results): publish static AgentV report" git push ``` @@ -110,15 +110,15 @@ Duplicate policy is explicit: ### Metrics sidecar -Each per-case artifact directory includes `outputs/metrics.json` +Each per-case artifact directory includes `metrics.json` (`schema_version: "agentv.metrics.v1"`). This is an AgentV-owned -derived projection over `outputs/trace.json`, the result row, and +derived projection over `trace.json`, the result row, and `grading.json`. It is the compact executor behavior summary for dashboards, comparison exports, and metric-style graders; it is not canonical trace storage and does not carry token/cost usage. -`outputs/transcript.jsonl` remains the ordered conversational/log compatibility -projection. Full trace detail stays in `outputs/trace.json` +`transcript.jsonl` remains the ordered conversational/log compatibility +projection. Full trace detail stays in `trace.json` (`agentv.trace.v1`). `benchmark.json` remains the run-level aggregate summary, and `index.jsonl` carries the lightweight `metrics_path` plus the trace/transcript artifact pointers used for detached payload publishing. @@ -126,7 +126,7 @@ Duration, token, and cost usage remains in `timing.json`, including source labels such as `provider_reported`, `token_estimated`, `aggregate`, or `unavailable`. -The `metrics` section aligns with Claude Agent Skills `outputs/metrics.json` +The `metrics` section aligns with Claude Agent Skills `metrics.json` while adding AgentV/Vercel-style detail: | Field group | Purpose | @@ -138,26 +138,27 @@ Vercel `@vercel/agent-eval` `results.o11y` maps into AgentV like this: | Vercel field | AgentV field | Artifact location | |--------------|--------------|-------------------| -| `shellCommands` | `metrics.shell_commands` | `outputs/metrics.json` | -| `filesRead` | `metrics.files_read` | `outputs/metrics.json` | -| `filesModified` | `metrics.files_modified` | `outputs/metrics.json` | -| `toolCalls` | `metrics.tool_call_events`, `metrics.tool_calls`, and `metrics.tool_call_counts` | `outputs/metrics.json`; compact counts can also appear in `grading.json.execution_metrics` and `benchmark.json.run_summary[*].tool_calls` | -| `totalToolCalls` | `metrics.total_tool_calls` | `outputs/metrics.json` | -| `webFetches` | `metrics.web_fetches` | `outputs/metrics.json` | -| `totalTurns` | `metrics.total_turns` | `outputs/metrics.json`; conversational rows remain in `outputs/transcript.jsonl` | -| `errors` | `metrics.errors` | `outputs/metrics.json`; compact count remains in `grading.json.execution_metrics.errors_encountered` | -| `thinkingBlocks` | `metrics.reasoning_blocks` and `thinking_blocks` | `outputs/metrics.json` | +| `shellCommands` | `metrics.shell_commands` | `metrics.json` | +| `filesRead` | `metrics.files_read` | `metrics.json` | +| `filesModified` | `metrics.files_modified` | `metrics.json` | +| `toolCalls` | `metrics.tool_call_events`, `metrics.tool_calls`, and `metrics.tool_call_counts` | `metrics.json`; compact counts can also appear in `grading.json.execution_metrics` and `benchmark.json.run_summary[*].tool_calls` | +| `totalToolCalls` | `metrics.total_tool_calls` | `metrics.json` | +| `webFetches` | `metrics.web_fetches` | `metrics.json` | +| `totalTurns` | `metrics.total_turns` | `metrics.json`; conversational rows remain in `transcript.jsonl` | +| `errors` | `metrics.errors` | `metrics.json`; compact count remains in `grading.json.execution_metrics.errors_encountered` | +| `thinkingBlocks` | `metrics.reasoning_blocks` and `thinking_blocks` | `metrics.json` | Agent Skills eval artifacts map into AgentV like this: | Agent Skills pattern | AgentV field | Artifact location | |----------------------|--------------|-------------------| | Authored `evals/evals.json` cases | AgentV eval cases and task bundle paths | Eval source plus optional `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` in `index.jsonl` | -| Per-case `outputs/` | Final answer and sidecars | `outputs/answer.md`, `outputs/response.md`, `outputs/trace.json`, `outputs/transcript.jsonl`, `outputs/metrics.json` | +| Per-case `outputs/` | Generated target output artifacts | `outputs/answer.md`, `outputs/response.md` | +| Per-case sidecars | Trace, transcript, metrics, and raw provider evidence | `trace.json`, `transcript.jsonl`, `metrics.json`, `provider.log` | | Per-case `timing.json` | Duration, token totals, cost, and usage source labels | `timing.json` | | Per-case `grading.json` | Assertions, graders, execution metrics, workspace changes | `grading.json`; summary fields can reference the same trace/result facts | | Iteration-level `benchmark.json` | Pass rate, time, tokens, tool calls, cost aggregates | Run-level `benchmark.json` | -| Transcript/log outlier analysis | Ordered transcript and canonical trace | `outputs/transcript.jsonl` for log compatibility; `outputs/trace.json` for full detail | +| Transcript/log outlier analysis | Ordered transcript and canonical trace | `transcript.jsonl` for log compatibility; `trace.json` for full detail | | Aggregate pass rate/time/tokens/delta | Run summaries and comparison tooling | `benchmark.json`, result comparisons, and projection bundles | ### Vendor-neutral projection bundle @@ -214,10 +215,10 @@ policy so downstream processing is auditable. For lightweight terminal workflows: ```bash -agentv results summary .agentv/results/runs/ -agentv results failures .agentv/results/runs/ -agentv results show .agentv/results/runs/ --test-id my-case -agentv results validate .agentv/results/runs/ +agentv results summary .agentv/results/default/ +agentv results failures .agentv/results/default/ +agentv results show .agentv/results/default/ --test-id my-case +agentv results validate .agentv/results/default/ ``` For a review-centric workflow built around these artifacts, see [Human Review Checkpoint](/docs/guides/human-review/). diff --git a/apps/web/src/content/docs/docs/tools/trend.mdx b/apps/web/src/content/docs/docs/tools/trend.mdx index 66898ca02..342f5ce67 100644 --- a/apps/web/src/content/docs/docs/tools/trend.mdx +++ b/apps/web/src/content/docs/docs/tools/trend.mdx @@ -29,9 +29,9 @@ Point directly at run workspaces or `index.jsonl` manifests when you need a spec ```bash agentv trend \ - .agentv/results/runs/2026-03-01T10-00-00-000Z/ \ - .agentv/results/runs/2026-03-08T10-00-00-000Z/index.jsonl \ - .agentv/results/runs/2026-03-15T10-00-00-000Z/ + .agentv/results/default/2026-03-01T10-00-00-000Z/ \ + .agentv/results/default/2026-03-08T10-00-00-000Z/index.jsonl \ + .agentv/results/default/2026-03-15T10-00-00-000Z/ ``` Concrete regression-gating example: @@ -45,8 +45,8 @@ agentv trend --last 8 --suite code-review --target claude-sonnet \ `trend` only accepts canonical run workspaces: -- `.agentv/results/runs//` -- `.agentv/results/runs//index.jsonl` +- `.agentv/results///` +- `.agentv/results///index.jsonl` Legacy flat `results.jsonl` files are rejected. The command stays on lightweight `index.jsonl` manifests and does not require per-test artifact hydration. @@ -54,7 +54,7 @@ Legacy flat `results.jsonl` files are rejected. The command stays on lightweight | Option | Description | |--------|-------------| -| `--last ` | Use the most recent `n` runs from `.agentv/results/runs/` | +| `--last ` | Use the most recent `n` runs from `.agentv/results/` | | `--suite ` | Filter records to one suite | | `--target ` | Filter records to one target inside each run | | `--slope-threshold ` | Minimum absolute slope required to classify improving or degrading (default: `0.01`) | @@ -112,7 +112,7 @@ Regression Gate: threshold=0.010 fail_on_degrading=true triggered=true "runs": [ { "label": "2026-03-01T10:00:00.000Z", - "path": "/repo/.agentv/results/runs/2026-03-01T10-00-00-000Z/index.jsonl", + "path": "/repo/.agentv/results/default/2026-03-01T10-00-00-000Z/index.jsonl", "timestamp": "2026-03-01T10:00:00.000Z", "matched_test_count": 42, "mean_score": 0.92 diff --git a/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx b/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx index 6052720ba..a6d2db05d 100644 --- a/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx +++ b/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx @@ -22,9 +22,9 @@ If no results repo is configured, or auto-push is disabled, `agentv eval` still | Location | Path or ref | What it contains | | --- | --- | --- | -| Local project | `.agentv/results/runs///benchmark.json` | A run-start stub with `metadata.planned_test_count` and the eval file path when known. This lets Dashboard recognize incomplete local runs as resumable. | -| Local project | `.agentv/results/runs///index.jsonl` | Result rows appended as test cases finish. Rows use the normal snake_case result JSONL format. | -| Results repo remote | `agentv/wip//` | A forced-updated branch containing the checkpointed run under `.agentv/results/runs//`. | +| Local project | `.agentv/results///benchmark.json` | A run-start stub with `metadata.planned_test_count` and the eval file path when known. This lets Dashboard recognize incomplete local runs as resumable. | +| Local project | `.agentv/results///index.jsonl` | Result rows appended as test cases finish. Rows use the normal snake_case result JSONL format. | +| Results repo remote | `agentv/wip//` | A forced-updated branch containing the checkpointed run under `.agentv/results//`. | | Results repo storage branch | Configured `results.repo.branch`; local checkout configs default to `agentv/results/v1` | The final published run after `agentv eval` completes and the normal auto-export succeeds. | The WIP branch name is derived from the current host and the run directory basename. Non-branch-safe characters are replaced with `-`; the host component is capped at 40 characters and the run component at 60 characters. @@ -55,19 +55,19 @@ git branch -r --list 'origin/agentv/wip/*' git switch --detach origin/agentv/wip// # 4. Inspect the checkpointed run path. -find .agentv/results/runs -name benchmark.json +find .agentv/results -name benchmark.json -# 5. Copy the run tree into the eval project, preserving paths under runs/. +# 5. Copy the run tree into the eval project, preserving experiment paths. PROJECT=/path/to/eval-project -mkdir -p "$PROJECT/.agentv/results/runs" -rsync -a .agentv/results/runs/ "$PROJECT/.agentv/results/runs/" +mkdir -p "$PROJECT/.agentv/results" +rsync -a .agentv/results/ "$PROJECT/.agentv/results/" # 6. Resume from the recovered run directory. cd "$PROJECT" -agentv eval --output .agentv/results/runs// --resume +agentv eval --output .agentv/results// --resume ``` -If the recovered `benchmark.json` contains `metadata.eval_file`, use that as ``. If the run lives directly under `.agentv/results/runs//` instead of an experiment directory, pass that path to `--output`. +If the recovered `benchmark.json` contains `metadata.eval_file`, use that as ``. After the resumed run publishes successfully, AgentV cleans up any WIP branch it creates for the resumed run. Delete the original orphaned branch manually when you no longer need it: diff --git a/examples/features/benchmark-tooling/README.md b/examples/features/benchmark-tooling/README.md index 7660e1b55..7c3497df5 100644 --- a/examples/features/benchmark-tooling/README.md +++ b/examples/features/benchmark-tooling/README.md @@ -10,7 +10,7 @@ Utilities for multi-model benchmarking workflows with AgentV. ```bash # Compare a recent canonical run -agentv compare .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl ``` Output: @@ -34,19 +34,19 @@ Pairwise Summary: ```bash # N-way matrix (all targets) -agentv compare .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl # With baseline regression check (exits 1 if any target regresses) -agentv compare .agentv/results/runs//index.jsonl --baseline gpt-4.1 +agentv compare .agentv/results/default//index.jsonl --baseline gpt-4.1 # Pairwise from combined file -agentv compare .agentv/results/runs//index.jsonl --baseline gpt-4.1 --candidate gpt-5-mini +agentv compare .agentv/results/default//index.jsonl --baseline gpt-4.1 --candidate gpt-5-mini # Filter to specific targets -agentv compare .agentv/results/runs//index.jsonl --targets gpt-4.1 --targets gpt-5-mini +agentv compare .agentv/results/default//index.jsonl --targets gpt-4.1 --targets gpt-5-mini # JSON output -agentv compare .agentv/results/runs//index.jsonl --json +agentv compare .agentv/results/default//index.jsonl --json ``` ### Pairwise Mode @@ -54,7 +54,7 @@ agentv compare .agentv/results/runs//index.jsonl --json Extract a head-to-head comparison between two specific targets: ```bash -agentv compare .agentv/results/runs//index.jsonl --baseline gpt-4.1 --candidate gpt-5-mini +agentv compare .agentv/results/default//index.jsonl --baseline gpt-4.1 --candidate gpt-5-mini ``` ``` @@ -88,7 +88,7 @@ Each line includes a `target` field to identify which model produced the result: ### Key Files - `evals/benchmark.eval.yaml` - Example eval config with 3 tests -- canonical run workspaces under `.agentv/results/runs//` +- canonical run workspaces under `.agentv/results/default//` ## split-by-target @@ -130,11 +130,11 @@ Use `agentv compare` directly on the canonical run manifest for pairwise or matr bun agentv eval my-eval.yaml # 2. Compare any two targets from the same run -bun agentv compare .agentv/results/runs//index.jsonl \ +bun agentv compare .agentv/results/default//index.jsonl \ --baseline gpt-4.1 --candidate claude-sonnet-4 # 3. JSON output for CI pipelines -bun agentv compare .agentv/results/runs//index.jsonl \ +bun agentv compare .agentv/results/default//index.jsonl \ --baseline gpt-4.1 --candidate claude-sonnet-4 --json ``` @@ -148,8 +148,8 @@ Computes aggregate win/loss/tie rates from `agentv compare --json` output, makin ```bash # Save comparison output to a file -bun agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl --json > comparison.json +bun agentv compare .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl --json > comparison.json # Print a human-readable summary table bun examples/features/benchmark-tooling/scripts/win-rate-summary.ts comparison.json @@ -167,10 +167,10 @@ Pass a directory of comparison JSON files to get per-metric win rates. Each file ```bash # Run comparisons for different metrics -bun agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl --json > comparisons/accuracy.json -bun agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl --json > comparisons/latency.json +bun agentv compare .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl --json > comparisons/accuracy.json +bun agentv compare .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl --json > comparisons/latency.json # Aggregate across all metrics bun examples/features/benchmark-tooling/scripts/win-rate-summary.ts comparisons/ @@ -285,7 +285,7 @@ bun examples/features/benchmark-tooling/scripts/benchmark-report.ts ./by-target/ bun agentv eval my-eval.yaml # 2. Compare two targets from the run manifest -bun agentv compare .agentv/results/runs//index.jsonl \ +bun agentv compare .agentv/results/default//index.jsonl \ --baseline gpt-4.1 --candidate claude-sonnet-4 --json > comparison.json # 3. Get win-rate summary diff --git a/examples/features/compare/README.md b/examples/features/compare/README.md index 05df34e8f..7162defd6 100644 --- a/examples/features/compare/README.md +++ b/examples/features/compare/README.md @@ -16,30 +16,30 @@ Demonstrates comparing canonical run manifests using the `agentv compare` comman # From repository root # N-way matrix from a canonical run manifest -agentv compare .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl # Pairwise from the same combined run manifest -agentv compare .agentv/results/runs//index.jsonl \ +agentv compare .agentv/results/default//index.jsonl \ --baseline gpt-4.1 --candidate gpt-5-mini # CI regression gate: exit 1 if any target regresses vs baseline -agentv compare .agentv/results/runs//index.jsonl \ +agentv compare .agentv/results/default//index.jsonl \ --baseline gpt-4.1 # Two-run pairwise comparison -agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl # With custom threshold for win/loss classification -agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl --threshold 0.05 +agentv compare .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl --threshold 0.05 # JSON output for CI pipelines -agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl --json +agentv compare .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl --json ``` ## Key Files -- canonical run workspaces under `.agentv/results/runs//` +- canonical run workspaces under `.agentv/results/default//` - `evals/README.md` - Detailed usage documentation diff --git a/examples/features/compare/evals/README.md b/examples/features/compare/evals/README.md index 79fa226d3..908703fc7 100644 --- a/examples/features/compare/evals/README.md +++ b/examples/features/compare/evals/README.md @@ -12,14 +12,14 @@ Compare model performance across different configurations: ## Sample Files -- canonical run workspaces under `.agentv/results/runs//` +- canonical run workspaces under `.agentv/results/default//` ## Usage ### N-Way Matrix (run manifest) ```bash -agentv compare .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl ``` Output: @@ -41,14 +41,14 @@ Pairwise Summary: ### Baseline Regression Check ```bash -agentv compare .agentv/results/runs//index.jsonl --baseline gpt-4.1 +agentv compare .agentv/results/default//index.jsonl --baseline gpt-4.1 # Exits 1 if any target regresses vs gpt-4.1 ``` ### Pairwise from a Single Run Manifest ```bash -agentv compare .agentv/results/runs//index.jsonl --baseline gpt-4.1 --candidate gpt-5-mini +agentv compare .agentv/results/default//index.jsonl --baseline gpt-4.1 --candidate gpt-5-mini ``` ``` @@ -66,13 +66,13 @@ Summary: 0 wins, 0 losses, 3 ties | Mean Δ: -0.017 | Status: regressed ### Two-Run Pairwise ```bash -agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl ``` Output: ``` -Comparing: .agentv/results/runs//index.jsonl → .agentv/results/runs//index.jsonl +Comparing: .agentv/results/default//index.jsonl → .agentv/results/default//index.jsonl Test ID Baseline Candidate Delta Result ─────────────── ──────── ───────── ──────── ──────── @@ -90,8 +90,8 @@ Summary: 1 win, 0 losses, 4 ties | Mean Δ: +0.054 | Status: improved Use a stricter threshold (0.05) for win/loss classification: ```bash -agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl --threshold 0.05 +agentv compare .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl --threshold 0.05 ``` ### JSON Output @@ -99,7 +99,7 @@ agentv compare .agentv/results/runs//index.jsonl \ For machine-readable output (CI pipelines, scripts): ```bash -agentv compare .agentv/results/runs//index.jsonl --json +agentv compare .agentv/results/default//index.jsonl --json ``` Output uses snake_case for Python ecosystem compatibility: @@ -130,8 +130,8 @@ Use exit codes for automated quality gates: ```bash # N-way: fail if any target regresses vs baseline -agentv compare .agentv/results/runs//index.jsonl --baseline gpt-4.1 || echo "Regression detected!" +agentv compare .agentv/results/default//index.jsonl --baseline gpt-4.1 || echo "Regression detected!" # Two-run: fail if candidate regresses -agentv compare .agentv/results/runs//index.jsonl .agentv/results/runs//index.jsonl || echo "Regression detected!" +agentv compare .agentv/results/default//index.jsonl .agentv/results/default//index.jsonl || echo "Regression detected!" ``` diff --git a/examples/features/compare/evals/dataset.eval.yaml b/examples/features/compare/evals/dataset.eval.yaml index b9cff5bc0..d73b019c0 100644 --- a/examples/features/compare/evals/dataset.eval.yaml +++ b/examples/features/compare/evals/dataset.eval.yaml @@ -3,7 +3,7 @@ # agentv eval evals/dataset.eval.yaml --target baseline # agentv eval evals/dataset.eval.yaml --target candidate # Then compare: -# agentv compare .agentv/results/runs//index.jsonl .agentv/results/runs//index.jsonl +# agentv compare .agentv/results/default//index.jsonl .agentv/results/default//index.jsonl name: compare-demo description: Demo eval for generating baseline and candidate results to compare diff --git a/examples/features/document-extraction/README.md b/examples/features/document-extraction/README.md index 4ee45bd5e..42a2adee4 100644 --- a/examples/features/document-extraction/README.md +++ b/examples/features/document-extraction/README.md @@ -25,7 +25,7 @@ bun agentv eval examples/features/document-extraction/evals/confusion-metrics.ev # Aggregate TP/TN/FP/FN into a table (only works with confusion-metrics.eval.yaml) bun run examples/features/document-extraction/scripts/aggregate_metrics.ts \ - .agentv/results/runs//index.jsonl + .agentv/results/default//index.jsonl ``` ## Pattern 1: Field Accuracy (`field-accuracy.eval.yaml`) @@ -71,7 +71,7 @@ graders: **Output**: Aggregate metrics table with fractional precision/recall: ``` -Processed 5 evaluation results from .agentv/results/runs//index.jsonl +Processed 5 evaluation results from .agentv/results/default//index.jsonl Field | TP | TN | FP | FN | Precision | Recall | F1 | Count ---------------+----+----+----+----+-----------+--------+-------+------ @@ -96,7 +96,7 @@ Macro-F1: 0.759 The `aggregate_metrics.ts` script only works with graders that emit `details.metrics`: ```bash -bun run scripts/aggregate_metrics.ts .agentv/results/runs//index.jsonl [options] +bun run scripts/aggregate_metrics.ts .agentv/results/default//index.jsonl [options] Options: --grader Filter to a specific grader diff --git a/examples/features/document-extraction/evals/confusion-metrics.eval.yaml b/examples/features/document-extraction/evals/confusion-metrics.eval.yaml index e723a24e5..174602285 100644 --- a/examples/features/document-extraction/evals/confusion-metrics.eval.yaml +++ b/examples/features/document-extraction/evals/confusion-metrics.eval.yaml @@ -22,7 +22,7 @@ # # Aggregate: # bun run examples/features/document-extraction/scripts/aggregate_metrics.ts \ -# .agentv/results/runs//index.jsonl +# .agentv/results/default//index.jsonl # description: Header field confusion metrics (TP/TN/FP/FN aggregation) diff --git a/examples/features/document-extraction/scripts/aggregate_metrics.ts b/examples/features/document-extraction/scripts/aggregate_metrics.ts index 59eafa70b..2eea67ed8 100644 --- a/examples/features/document-extraction/scripts/aggregate_metrics.ts +++ b/examples/features/document-extraction/scripts/aggregate_metrics.ts @@ -6,9 +6,9 @@ * per attribute across the whole dataset. * * Usage: - * bun run scripts/aggregate_metrics.ts .agentv/results/runs//index.jsonl - * bun run scripts/aggregate_metrics.ts .agentv/results/runs//index.jsonl --grader header_confusion - * bun run scripts/aggregate_metrics.ts .agentv/results/runs//index.jsonl --format csv + * bun run scripts/aggregate_metrics.ts .agentv/results/default//index.jsonl + * bun run scripts/aggregate_metrics.ts .agentv/results/default//index.jsonl --grader header_confusion + * bun run scripts/aggregate_metrics.ts .agentv/results/default//index.jsonl --format csv */ import * as fs from 'node:fs'; @@ -241,7 +241,7 @@ Options: Example: bun run scripts/aggregate_metrics.ts .agentv/results/eval-001.jsonl - bun run scripts/aggregate_metrics.ts .agentv/results/runs//index.jsonl --grader header_confusion --format csv + bun run scripts/aggregate_metrics.ts .agentv/results/default//index.jsonl --grader header_confusion --format csv `); process.exit(0); } diff --git a/examples/features/experiments/README.md b/examples/features/experiments/README.md index cda2f0eda..f1fd81066 100644 --- a/examples/features/experiments/README.md +++ b/examples/features/experiments/README.md @@ -39,7 +39,7 @@ The experiment label is written to `manifest.json` and propagated to `index.json Each run produces a separate directory. The experiment is metadata, not a path segment: ``` -.agentv/results/runs/ +.agentv/results/ ├── 2026-03-28T10-00-00-000Z/ # with_skills run │ ├── manifest.json # { "experiment": "with_skills", ... } │ └── coding-ability/ @@ -58,8 +58,8 @@ After both runs complete and are graded: ```bash # Compare the two runs -agentv compare .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl +agentv compare .agentv/results/with-skills//index.jsonl \ + .agentv/results/without-skills//index.jsonl ``` ## Key Files diff --git a/examples/features/sdk-programmatic-api/README.md b/examples/features/sdk-programmatic-api/README.md index 304f9293d..178abb3cb 100644 --- a/examples/features/sdk-programmatic-api/README.md +++ b/examples/features/sdk-programmatic-api/README.md @@ -7,7 +7,7 @@ Demonstrates using `evaluate()` from `@agentv/sdk` to run evaluations as a libra 1. Imports `evaluate()` from `@agentv/sdk` 2. Defines tests inline with `assert` 3. Runs the evaluation and prints summary statistics -4. Writes canonical AgentV run artifacts under `.agentv/results/runs/...` +4. Writes canonical AgentV run artifacts under `.agentv/results/...` ## How to Run diff --git a/examples/features/sdk-programmatic-api/evaluate.ts b/examples/features/sdk-programmatic-api/evaluate.ts index 107b77ff9..fc24f9b68 100644 --- a/examples/features/sdk-programmatic-api/evaluate.ts +++ b/examples/features/sdk-programmatic-api/evaluate.ts @@ -33,7 +33,7 @@ const { results, summary } = await evaluate({ onResult: (result) => { console.log(` ${result.testId}: score=${result.score.toFixed(2)}`); }, - outputDir: '.agentv/results/runs/sdk-programmatic-api-example', + outputDir: '.agentv/results/sdk-programmatic-api-example', experiment: 'sdk-programmatic-api', }); @@ -43,4 +43,4 @@ console.log(`Passed: ${summary.passed}`); console.log(`Failed: ${summary.failed}`); console.log(`Mean score: ${summary.meanScore.toFixed(2)}`); console.log(`Duration: ${summary.durationMs}ms`); -console.log('Artifacts: .agentv/results/runs/sdk-programmatic-api-example'); +console.log('Artifacts: .agentv/results/sdk-programmatic-api-example'); diff --git a/examples/features/trace-analysis/README.md b/examples/features/trace-analysis/README.md index c9f167d39..ef3b71cc4 100644 --- a/examples/features/trace-analysis/README.md +++ b/examples/features/trace-analysis/README.md @@ -9,7 +9,7 @@ Demonstrates `agentv trace` subcommands for headless trace inspection and analys bun agentv trace list # Show summary trace details from the run manifest -bun agentv trace show .agentv/results/runs//index.jsonl +bun agentv trace show .agentv/results/default//index.jsonl # Show hierarchical trace tree from an OTLP export bun agentv trace show traces/eval.otlp.json --tree @@ -18,13 +18,13 @@ bun agentv trace show traces/eval.otlp.json --tree bun agentv trace show traces/eval.otlp.json --test-id research-question --tree # Compute percentile statistics -bun agentv trace stats .agentv/results/runs//index.jsonl +bun agentv trace stats .agentv/results/default//index.jsonl # Group stats by target provider -bun agentv trace stats .agentv/results/runs//index.jsonl --group-by target +bun agentv trace stats .agentv/results/default//index.jsonl --group-by target # JSON output for piping to jq -bun agentv trace stats .agentv/results/runs//index.jsonl --format json | jq '.groups[].metrics' +bun agentv trace stats .agentv/results/default//index.jsonl --format json | jq '.groups[].metrics' ``` ## What's in the Example Data @@ -53,6 +53,6 @@ bun agentv trace show traces/eval.otlp.json --format json \ | jq '[.[] | select(.cost_usd > 0.10) | {test_id, score, cost: .cost_usd}]' # Compare scores by target provider -bun agentv trace stats .agentv/results/runs//index.jsonl --group-by target --format json \ +bun agentv trace stats .agentv/results/default//index.jsonl --group-by target --format json \ | jq '.groups[] | {label, score_mean: .metrics.score.mean}' ``` diff --git a/examples/features/trend/README.md b/examples/features/trend/README.md index a592e3dc4..50329245b 100644 --- a/examples/features/trend/README.md +++ b/examples/features/trend/README.md @@ -29,8 +29,8 @@ Most real users will run `trend` against their latest eval history with `--last` To reproduce that flow from this example directory, first copy the sample runs into the normal runtime layout: ```bash -mkdir -p .agentv/results/runs -cp -R sample-runs/* .agentv/results/runs/ +mkdir -p .agentv/results/default +cp -R sample-runs/* .agentv/results/default/ ``` Then run: @@ -66,7 +66,7 @@ Interpretation: ## Explicit Inputs -If you want to see the same analysis without copying files into `.agentv/results/runs/`, point `trend` at the sample runs directly: +If you want to see the same analysis without copying files into `.agentv/results/`, point `trend` at the sample runs directly: ```bash bun ../../../apps/cli/src/cli.ts trend \ diff --git a/examples/red-team/README.md b/examples/red-team/README.md index 916040f13..def28252a 100644 --- a/examples/red-team/README.md +++ b/examples/red-team/README.md @@ -48,7 +48,7 @@ agentv eval examples/red-team/suites/llm01-prompt-injection.yaml --target your-t # is present verbatim in the JSONL): agentv eval examples/red-team/ \ && jq 'select(.metadata.governance.owasp_llm_top_10_2025 // [] | index("LLM01"))' \ - .agentv/results/runs/default/*/index.jsonl + .agentv/results/default/*/index.jsonl ``` The pack is **opt-in**. It is not bundled into `agentv init` — copy the diff --git a/examples/showcase/bug-fix-benchmark/README.md b/examples/showcase/bug-fix-benchmark/README.md index 52370150a..f026ea3b5 100644 --- a/examples/showcase/bug-fix-benchmark/README.md +++ b/examples/showcase/bug-fix-benchmark/README.md @@ -48,8 +48,8 @@ agentv eval evals/bug-fixes.eval.yaml \ ```bash agentv compare \ - .agentv/results/runs//index.jsonl \ - .agentv/results/runs//index.jsonl + .agentv/results/default//index.jsonl \ + .agentv/results/default//index.jsonl ``` ## Test Case: Issue #912 diff --git a/examples/showcase/export-screening/README.md b/examples/showcase/export-screening/README.md index fa16236c4..fef75aede 100644 --- a/examples/showcase/export-screening/README.md +++ b/examples/showcase/export-screening/README.md @@ -51,7 +51,7 @@ Use the wrapper script to compute a confusion matrix and policy-weighted overall structured CI result JSON file (defaults to `results.ci_check.json`): ```bash -bun run ./evals/ci_check.ts .agentv/results/runs//index.jsonl --threshold 0.95 --check-class High +bun run ./evals/ci_check.ts .agentv/results/default//index.jsonl --threshold 0.95 --check-class High ``` ### Multi-Sample CI Gating @@ -153,7 +153,7 @@ bun run ./evals/ci_check.ts --eval ./evals/dataset.eval.yaml --threshold 0.95 bun run ./evals/ci_check.ts --eval ./evals/dataset.eval.yaml --samples 5 --threshold 0.90 # Or check an existing run manifest -bun run ./evals/ci_check.ts .agentv/results/runs//index.jsonl --threshold 0.95 +bun run ./evals/ci_check.ts .agentv/results/default//index.jsonl --threshold 0.95 ``` ### Options diff --git a/examples/showcase/multi-model-benchmark/README.md b/examples/showcase/multi-model-benchmark/README.md index 0767d89ed..f9c8f7a13 100644 --- a/examples/showcase/multi-model-benchmark/README.md +++ b/examples/showcase/multi-model-benchmark/README.md @@ -55,16 +55,16 @@ The eval produces a canonical run workspace with `target` in each `index.jsonl` ```bash # N-way matrix — see all models at once -agentv compare .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl # Designate a baseline for CI regression gating -agentv compare .agentv/results/runs//index.jsonl --baseline copilot +agentv compare .agentv/results/default//index.jsonl --baseline copilot # Pairwise: compare two specific targets -agentv compare .agentv/results/runs//index.jsonl --baseline copilot --candidate claude +agentv compare .agentv/results/default//index.jsonl --baseline copilot --candidate claude # JSON output for CI integration -agentv compare .agentv/results/runs//index.jsonl --json +agentv compare .agentv/results/default//index.jsonl --json ``` ### Expected Output @@ -154,7 +154,7 @@ benchmark.eval.yaml └────────┬────────────────┘ │ ▼ - .agentv/results/runs// + .agentv/results/default// index.jsonl │ ▼ diff --git a/packages/core/src/evaluation/metrics.ts b/packages/core/src/evaluation/metrics.ts index cec0ead87..8cbd2b2f4 100644 --- a/packages/core/src/evaluation/metrics.ts +++ b/packages/core/src/evaluation/metrics.ts @@ -2,10 +2,10 @@ * AgentV metrics v1. * * This is a derived per-case executor metrics projection over `EvaluationResult` - * and `agentv.trace.v1`. It aligns with Agent Skills `outputs/metrics.json` + * and `agentv.trace.v1`. It aligns with AgentV's case-local `metrics.json` * while carrying the compact Vercel-style observability fields. It is not the - * canonical trace store; full detail stays in `outputs/trace.json`, ordered - * transcript compatibility rows stay in `outputs/transcript.jsonl`, and + * canonical trace store; full detail stays in `trace.json`, ordered + * transcript compatibility rows stay in `transcript.jsonl`, and * duration/token/cost usage stays in `timing.json`. */ diff --git a/packages/core/src/evaluation/providers/claude-cli.ts b/packages/core/src/evaluation/providers/claude-cli.ts index 91099a44c..1977a56ac 100644 --- a/packages/core/src/evaluation/providers/claude-cli.ts +++ b/packages/core/src/evaluation/providers/claude-cli.ts @@ -8,6 +8,7 @@ import path from 'node:path'; import { trackChild } from '../../runtime/child-tracker.js'; import { extractTextContent, toContentArray } from './claude-content.js'; import { recordClaudeLogEntry } from './claude-log-tracker.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; import type { ClaudeResolvedConfig } from './targets.js'; @@ -207,7 +208,7 @@ export class ClaudeCliProvider implements Provider { return undefined; } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { const disabled = isClaudeCliLogStreamingDisabled(); if (disabled) { return undefined; @@ -215,13 +216,13 @@ export class ClaudeCliProvider implements Provider { if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'claude-cli'); + return resolveDefaultProviderLogDir('claude-cli', request); } private async createStreamLogger( request: ProviderRequest, ): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/providers/claude-sdk.ts b/packages/core/src/evaluation/providers/claude-sdk.ts index ac29bfd47..49959c8c0 100644 --- a/packages/core/src/evaluation/providers/claude-sdk.ts +++ b/packages/core/src/evaluation/providers/claude-sdk.ts @@ -6,6 +6,7 @@ import path from 'node:path'; import { extractTextContent, toContentArray } from './claude-content.js'; import { recordClaudeLogEntry } from './claude-log-tracker.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; import type { ClaudeResolvedConfig } from './targets.js'; @@ -232,7 +233,7 @@ export class ClaudeSdkProvider implements Provider { return undefined; } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { const disabled = isClaudeLogStreamingDisabled(); if (disabled) { return undefined; @@ -240,13 +241,13 @@ export class ClaudeSdkProvider implements Provider { if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'claude'); + return resolveDefaultProviderLogDir('claude', request); } private async createStreamLogger( request: ProviderRequest, ): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/providers/claude.ts b/packages/core/src/evaluation/providers/claude.ts index 2ac222e4f..d0951fbb4 100644 --- a/packages/core/src/evaluation/providers/claude.ts +++ b/packages/core/src/evaluation/providers/claude.ts @@ -6,6 +6,7 @@ import path from 'node:path'; import { extractTextContent, toContentArray } from './claude-content.js'; import { recordClaudeLogEntry } from './claude-log-tracker.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; import type { ClaudeResolvedConfig } from './targets.js'; import type { @@ -229,7 +230,7 @@ export class ClaudeProvider implements Provider { return undefined; } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { const disabled = isClaudeLogStreamingDisabled(); if (disabled) { return undefined; @@ -237,13 +238,13 @@ export class ClaudeProvider implements Provider { if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'claude'); + return resolveDefaultProviderLogDir('claude', request); } private async createStreamLogger( request: ProviderRequest, ): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/providers/codex-cli.ts b/packages/core/src/evaluation/providers/codex-cli.ts index 286fbcad6..82b842b9e 100644 --- a/packages/core/src/evaluation/providers/codex-cli.ts +++ b/packages/core/src/evaluation/providers/codex-cli.ts @@ -9,6 +9,7 @@ import { promisify } from 'node:util'; import { trackChild } from '../../runtime/child-tracker.js'; import { recordCodexLogEntry } from './codex-log-tracker.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; import type { CodexResolvedConfig } from './targets.js'; import type { Provider, ProviderRequest, ProviderResponse } from './types.js'; @@ -202,7 +203,7 @@ export class CodexCliProvider implements Provider { } } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { const disabled = isCodexLogStreamingDisabled(); if (disabled || this.config.streamLog === false) { return undefined; @@ -210,13 +211,13 @@ export class CodexCliProvider implements Provider { if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'codex'); + return resolveDefaultProviderLogDir('codex', request); } private async createStreamLogger( request: ProviderRequest, ): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/providers/codex.ts b/packages/core/src/evaluation/providers/codex.ts index 6da74a3c3..71e3d19c5 100644 --- a/packages/core/src/evaluation/providers/codex.ts +++ b/packages/core/src/evaluation/providers/codex.ts @@ -5,6 +5,7 @@ import { mkdir } from 'node:fs/promises'; import path from 'node:path'; import { recordCodexLogEntry } from './codex-log-tracker.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; import type { CodexResolvedConfig } from './targets.js'; @@ -319,7 +320,7 @@ export class CodexProvider implements Provider { return undefined; } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { const disabled = isCodexLogStreamingDisabled(); if (disabled || this.config.streamLog === false) { return undefined; @@ -327,13 +328,13 @@ export class CodexProvider implements Provider { if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'codex'); + return resolveDefaultProviderLogDir('codex', request); } private async createStreamLogger( request: ProviderRequest, ): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/providers/copilot-cli.ts b/packages/core/src/evaluation/providers/copilot-cli.ts index 08a7e5a76..ce47bd600 100644 --- a/packages/core/src/evaluation/providers/copilot-cli.ts +++ b/packages/core/src/evaluation/providers/copilot-cli.ts @@ -20,6 +20,7 @@ import { resolveCopilotTimeoutMs, resolvePlatformCliPath, } from './copilot-utils.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; import type { CopilotCliResolvedConfig, CopilotCustomProviderConfig } from './targets.js'; @@ -541,21 +542,21 @@ export class CopilotCliProvider implements Provider { return 'copilot'; } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { if (isLogStreamingDisabled('AGENTV_COPILOT_CLI_STREAM_LOGS')) { return undefined; } if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'copilot-cli'); + return resolveDefaultProviderLogDir('copilot-cli', request); } private async createStreamLogger( request: ProviderRequest, mode: 'acp' | 'prompt', ): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/providers/copilot-sdk.ts b/packages/core/src/evaluation/providers/copilot-sdk.ts index d4b1fa42e..3a09dfe60 100644 --- a/packages/core/src/evaluation/providers/copilot-sdk.ts +++ b/packages/core/src/evaluation/providers/copilot-sdk.ts @@ -12,6 +12,7 @@ import { resolveCopilotTimeoutMs, resolvePlatformCliPath, } from './copilot-utils.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; import type { CopilotSdkResolvedConfig } from './targets.js'; @@ -428,20 +429,20 @@ export class CopilotSdkProvider implements Provider { return undefined; } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { if (isLogStreamingDisabled('AGENTV_COPILOT_SDK_STREAM_LOGS')) { return undefined; } if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'copilot-sdk'); + return resolveDefaultProviderLogDir('copilot-sdk', request); } private async createStreamLogger( request: ProviderRequest, ): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/providers/log-directory.ts b/packages/core/src/evaluation/providers/log-directory.ts new file mode 100644 index 000000000..110ccdab7 --- /dev/null +++ b/packages/core/src/evaluation/providers/log-directory.ts @@ -0,0 +1,31 @@ +import path from 'node:path'; + +import type { ProviderRequest } from './types.js'; + +function safePathSegment(value: string | undefined, fallback: string): string { + const trimmed = value?.trim(); + if (!trimmed) { + return fallback; + } + return trimmed.replace(/[/\\:*?"<>|]/g, '_'); +} + +export function resolveDefaultProviderLogDir( + providerName: string, + request?: Pick, +): string | undefined { + const runDir = process.env.AGENTV_RUN_DIR?.trim(); + if (runDir) { + if (request?.evalCaseId) { + const segments = [ + request.suite ? safePathSegment(request.suite, 'default') : undefined, + safePathSegment(request.evalCaseId, 'unknown'), + 'logs', + providerName, + ].filter((segment): segment is string => segment !== undefined); + return path.join(path.resolve(runDir), ...segments); + } + return path.join(path.resolve(runDir), '_logs', providerName); + } + return undefined; +} diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts index 64b7bf345..d06135ea0 100644 --- a/packages/core/src/evaluation/providers/pi-cli.ts +++ b/packages/core/src/evaluation/providers/pi-cli.ts @@ -17,6 +17,7 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { trackChild } from '../../runtime/child-tracker.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { recordPiLogEntry } from './pi-log-tracker.js'; import { @@ -325,15 +326,15 @@ export class PiCliProvider implements Provider { } } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'pi-cli'); + return resolveDefaultProviderLogDir('pi-cli', request); } private async createStreamLogger(request: ProviderRequest): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts index 8a6a4651b..11fb978d6 100644 --- a/packages/core/src/evaluation/providers/pi-coding-agent.ts +++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts @@ -18,6 +18,7 @@ import { createInterface } from 'node:readline'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { getAgentvDataDir } from '../../paths.js'; +import { resolveDefaultProviderLogDir } from './log-directory.js'; import { recordPiLogEntry } from './pi-log-tracker.js'; import { normalizeAzureSdkBaseUrl, @@ -590,15 +591,15 @@ export class PiCodingAgentProvider implements Provider { return selected.length > 0 ? (selected as any[]) : sdk.codingTools; } - private resolveLogDirectory(): string | undefined { + private resolveLogDirectory(request: ProviderRequest): string | undefined { if (this.config.logDir) { return path.resolve(this.config.logDir); } - return path.join(process.cwd(), '.agentv', 'logs', 'pi-coding-agent'); + return resolveDefaultProviderLogDir('pi-coding-agent', request); } private async createStreamLogger(request: ProviderRequest): Promise { - const logDir = this.resolveLogDirectory(); + const logDir = this.resolveLogDirectory(request); if (!logDir) { return undefined; } diff --git a/packages/core/src/evaluation/result-artifact-contract.ts b/packages/core/src/evaluation/result-artifact-contract.ts index 1471e8614..1a7166c21 100644 --- a/packages/core/src/evaluation/result-artifact-contract.ts +++ b/packages/core/src/evaluation/result-artifact-contract.ts @@ -6,7 +6,7 @@ * per-result artifact directories; these pointers describe where those same * AgentV-owned artifacts belong when projected to a results ref, sidecar ref, * or object store. Use pointers for large detached payload bytes, not as the - * discovery path for ordinary sidecars such as `outputs/metrics.json`; normal + * discovery path for ordinary sidecars such as `metrics.json`; normal * sidecars should use explicit path fields such as `metrics_path`. * * Git remote publishing treats the configured results branch as the @@ -25,9 +25,9 @@ export const AGENTV_RESULTS_REFS = { oplog: AGENTV_RESULTS_OPLOG_REF, } as const; -export const CANONICAL_TRACE_ARTIFACT_PATH = 'outputs/trace.json' as const; -export const CANONICAL_TRANSCRIPT_ARTIFACT_PATH = 'outputs/transcript.jsonl' as const; -export const CANONICAL_METRICS_ARTIFACT_PATH = 'outputs/metrics.json' as const; +export const CANONICAL_TRACE_ARTIFACT_PATH = 'trace.json' as const; +export const CANONICAL_TRANSCRIPT_ARTIFACT_PATH = 'transcript.jsonl' as const; +export const CANONICAL_METRICS_ARTIFACT_PATH = 'metrics.json' as const; export const TRANSCRIPT_SCHEMA_VERSION = 'agentv.transcript.v1' as const; export const METRICS_SCHEMA_VERSION = 'agentv.metrics.v1' as const; diff --git a/packages/core/src/evaluation/result-row-schema.ts b/packages/core/src/evaluation/result-row-schema.ts index 555c59c90..0655ecaf7 100644 --- a/packages/core/src/evaluation/result-row-schema.ts +++ b/packages/core/src/evaluation/result-row-schema.ts @@ -43,6 +43,7 @@ const RESULT_ROW_ALIASES = { testId: 'test_id', timingPath: 'timing_path', tokenUsage: 'token_usage', + tracePath: 'trace_path', transcriptPath: 'transcript_path', workspacePath: 'workspace_path', } as const; diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index e29c8ef1e..33eb3bfcd 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -24,7 +24,7 @@ import { const execFileAsync = promisify(execFile); // Local working-tree run workspace inside the eval repo. Local commands // (`agentv eval` default --output, inspect/trend/export/combine/serve) read and -// write runs here. This is NOT the on-branch layout — see resolveResultsRepoRunsDir. +// write runs here. This is NOT the on-branch layout — see RESULTS_REPO_RUNS_DIR. const RESULTS_REPO_RESULTS_DIR = '.agentv/results'; // On-branch / results-repo-clone storage layout. The results branch (e.g. // agentv/results/v1) already namespaces results, so runs are stored flat at @@ -1726,7 +1726,7 @@ export async function stageResultsArtifacts(params: { export function resolveResultsRepoRunsDir(config: ResultsConfig): string { const normalized = normalizeResultsConfig(config); - return path.join(normalized.path, RESULTS_REPO_RESULTS_DIR, 'runs'); + return path.join(normalized.path, RESULTS_REPO_RESULTS_DIR); } export async function directorySizeBytes(targetPath: string): Promise { diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index 24fc4a5cf..20c6161b7 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -243,6 +243,7 @@ export interface IndexArtifactEntry { readonly timing_path: string; readonly output_path?: string; readonly answer_path?: string; + readonly trace_path?: string; readonly transcript_path?: string; readonly metrics_path?: string; readonly artifact_pointers?: ResultArtifactPointersWire; @@ -894,12 +895,12 @@ function rawProviderLogSourcePath(result: EvaluationResult): string | undefined return sourcePath ? sourcePath : undefined; } -function rawProviderLogArtifactPath(outputsDir: string): string { - return path.join(outputsDir, 'raw', 'provider.log'); +function rawProviderLogArtifactPath(testDir: string): string { + return path.join(testDir, 'provider.log'); } -async function copyRawProviderLogArtifact(sourcePath: string, outputsDir: string): Promise { - const destinationPath = rawProviderLogArtifactPath(outputsDir); +async function copyRawProviderLogArtifact(sourcePath: string, testDir: string): Promise { + const destinationPath = rawProviderLogArtifactPath(testDir); if (path.resolve(sourcePath) === path.resolve(destinationPath)) { return destinationPath; } @@ -912,7 +913,7 @@ async function copyRawProviderLogArtifact(sourcePath: string, outputsDir: string interface TraceEnvelopeSidecarParams { readonly result: EvaluationResult; readonly outputDir: string; - readonly outputsDir: string; + readonly testDir: string; readonly evalPath?: string; readonly experiment?: string; readonly runId?: string; @@ -933,9 +934,7 @@ function buildTraceEnvelopeSidecar(params: TraceEnvelopeSidecarParams): TraceEnv response_path: params.result.output.length > 0 ? 'outputs/response.md' : undefined, transcript_path: hasTranscript ? CANONICAL_TRANSCRIPT_ARTIFACT_PATH : undefined, metrics_path: CANONICAL_METRICS_ARTIFACT_PATH, - raw_provider_log_path: rawProviderLogSourcePath(params.result) - ? 'outputs/raw/provider.log' - : undefined, + raw_provider_log_path: rawProviderLogSourcePath(params.result) ? 'provider.log' : undefined, }, duplicatePolicy: params.duplicatePolicy, }); @@ -946,7 +945,7 @@ async function writeTraceEnvelopeSidecar( ): Promise { const envelope = buildTraceEnvelopeSidecar(params); await writeFile( - path.join(params.outputsDir, 'trace.json'), + path.join(params.testDir, CANONICAL_TRACE_ARTIFACT_PATH), `${JSON.stringify(toTraceEnvelopeWire(envelope), null, 2)}\n`, 'utf8', ); @@ -1028,6 +1027,7 @@ export function buildIndexArtifactEntry( timingPath: string; outputPath?: string; answerPath?: string; + tracePath?: string; transcriptPath?: string; metricsPath?: string; artifactPointers?: ResultArtifactPointersWire; @@ -1069,6 +1069,9 @@ export function buildIndexArtifactEntry( answer_path: options.answerPath ? toRelativeArtifactPath(options.outputDir, options.answerPath) : undefined, + trace_path: options.tracePath + ? toRelativeArtifactPath(options.outputDir, options.tracePath) + : undefined, transcript_path: options.transcriptPath ? toRelativeArtifactPath(options.outputDir, options.transcriptPath) : undefined, @@ -1135,12 +1138,13 @@ export function buildResultIndexArtifact( input_path: input ? path.posix.join(artifactSubdir, 'input.md') : undefined, output_path: hasAnswer ? path.posix.join(artifactSubdir, 'outputs', 'answer.md') : undefined, answer_path: hasAnswer ? path.posix.join(artifactSubdir, 'outputs', 'answer.md') : undefined, + trace_path: path.posix.join(artifactSubdir, CANONICAL_TRACE_ARTIFACT_PATH), transcript_path: hasTranscript - ? path.posix.join(artifactSubdir, 'outputs', 'transcript.jsonl') + ? path.posix.join(artifactSubdir, CANONICAL_TRANSCRIPT_ARTIFACT_PATH) : undefined, - metrics_path: path.posix.join(artifactSubdir, 'outputs', 'metrics.json'), + metrics_path: path.posix.join(artifactSubdir, CANONICAL_METRICS_ARTIFACT_PATH), raw_provider_log_path: hasRawProviderLog - ? path.posix.join(artifactSubdir, 'outputs', 'raw', 'provider.log') + ? path.posix.join(artifactSubdir, 'provider.log') : undefined, artifact_pointers: options?.artifactPointers, response_path: hasAnswer @@ -1548,25 +1552,25 @@ export async function writePerTestArtifacts( } const rawProviderLogSource = rawProviderLogSourcePath(result); if (rawProviderLogSource) { - await copyRawProviderLogArtifact(rawProviderLogSource, outputsDir); + await copyRawProviderLogArtifact(rawProviderLogSource, testDir); } - const tracePath = path.join(outputsDir, 'trace.json'); + const tracePath = path.join(testDir, CANONICAL_TRACE_ARTIFACT_PATH); const envelope = await writeTraceEnvelopeSidecar({ result, outputDir, - outputsDir, + testDir, evalPath: resolveEnvelopeEvalPath(result, testByTestId, options?.evalFile), experiment: options?.experiment, runId: options?.runId, duplicatePolicy, }); const transcriptPath = hasTranscriptProjection(result, envelope) - ? path.join(outputsDir, 'transcript.jsonl') + ? path.join(testDir, CANONICAL_TRANSCRIPT_ARTIFACT_PATH) : undefined; if (transcriptPath) { await writeTranscriptJsonl(transcriptPath, result, envelope); } - const metricsPath = path.join(outputsDir, 'metrics.json'); + const metricsPath = path.join(testDir, CANONICAL_METRICS_ARTIFACT_PATH); await writeMetricsArtifact({ filePath: metricsPath, result, @@ -1646,20 +1650,20 @@ export async function writeArtifactsFromResults( const envelope = buildTraceEnvelopeSidecar({ result, outputDir, - outputsDir, + testDir, evalPath: resolveEnvelopeEvalPath(result, testByTestId, options?.evalFile), experiment: options?.experiment, runId: options?.runId, duplicatePolicy, }); const transcriptPath = hasTranscriptProjection(result, envelope) - ? path.join(outputsDir, 'transcript.jsonl') + ? path.join(testDir, CANONICAL_TRANSCRIPT_ARTIFACT_PATH) : undefined; - const tracePath = path.join(outputsDir, 'trace.json'); - const metricsPath = path.join(outputsDir, 'metrics.json'); + const tracePath = path.join(testDir, CANONICAL_TRACE_ARTIFACT_PATH); + const metricsPath = path.join(testDir, CANONICAL_METRICS_ARTIFACT_PATH); const rawProviderLogSource = rawProviderLogSourcePath(result); const rawProviderLogPath = rawProviderLogSource - ? rawProviderLogArtifactPath(outputsDir) + ? rawProviderLogArtifactPath(testDir) : undefined; const projectionIdentity = envelope.projectionIdentity; if (!projectionIdentity) { @@ -1731,7 +1735,7 @@ export async function writeArtifactsFromResults( await writeFile(plan.responsePath, result.output, 'utf8'); } if (plan.rawProviderLogSource) { - await copyRawProviderLogArtifact(plan.rawProviderLogSource, plan.outputsDir); + await copyRawProviderLogArtifact(plan.rawProviderLogSource, plan.testDir); } await writeFile( plan.tracePath, @@ -1769,6 +1773,7 @@ export async function writeArtifactsFromResults( timingPath: plan.perTestTimingPath, outputPath: plan.answerPath, answerPath: plan.answerPath, + tracePath: plan.tracePath, transcriptPath: plan.transcriptPath, metricsPath: plan.metricsPath, artifactPointers, diff --git a/packages/core/src/evaluation/trace-envelope.ts b/packages/core/src/evaluation/trace-envelope.ts index 6f01fdb51..a22a3efca 100644 --- a/packages/core/src/evaluation/trace-envelope.ts +++ b/packages/core/src/evaluation/trace-envelope.ts @@ -8,7 +8,7 @@ * `gen_ai.operation.name` and `openinference.span.kind` are copied exactly and * never case-converted. * - * Derived views such as Provider `Message[]`, `outputs/transcript.jsonl`, + * Derived views such as Provider `Message[]`, `transcript.jsonl`, * `TraceSummary`, compact tool trajectories, replay provider responses, and * OTLP JSON export bodies must project from this artifact. Transcript JSONL * uses AgentV transcript events on the root span so compatibility rows can diff --git a/packages/core/src/evaluation/trace.ts b/packages/core/src/evaluation/trace.ts index 4582488b0..25053073c 100644 --- a/packages/core/src/evaluation/trace.ts +++ b/packages/core/src/evaluation/trace.ts @@ -729,7 +729,7 @@ function toTraceError(error: TraceError | string): TraceError { /** * Build the result-local trace read model for an evaluation case from provider * messages and execution metrics. This is the projection used by result JSONL, - * code-grader stdin, `outputs/answer.md`, and `outputs/transcript.jsonl`. + * code-grader stdin, `outputs/answer.md`, and `transcript.jsonl`. */ export function buildTraceFromMessages(options: BuildTraceOptions = {}): Trace { const messages = buildTraceMessages(options.input, options.output); diff --git a/packages/core/test/evaluation/fixtures/execution-trace/nested-subagent.json b/packages/core/test/evaluation/fixtures/execution-trace/nested-subagent.json index a11e5c704..448ecd835 100644 --- a/packages/core/test/evaluation/fixtures/execution-trace/nested-subagent.json +++ b/packages/core/test/evaluation/fixtures/execution-trace/nested-subagent.json @@ -166,7 +166,7 @@ }, "source": { "kind": "agentv_run", - "path": ".agentv/results/runs/run-nested-subagent/index.jsonl", + "path": ".agentv/results/default/run-nested-subagent/index.jsonl", "provider": "codex", "format": "agentv_result", "version": "1", @@ -188,7 +188,7 @@ } }, "artifacts": { - "trace_path": "outputs/trace.json", - "transcript_path": "outputs/transcript.jsonl" + "trace_path": "trace.json", + "transcript_path": "transcript.jsonl" } } diff --git a/packages/core/test/evaluation/loaders/agent-skills-parser.test.ts b/packages/core/test/evaluation/loaders/agent-skills-parser.test.ts index 40183638e..1c3d4a4a3 100644 --- a/packages/core/test/evaluation/loaders/agent-skills-parser.test.ts +++ b/packages/core/test/evaluation/loaders/agent-skills-parser.test.ts @@ -224,8 +224,8 @@ describe('parseAgentSkillsEvals', () => { { id: 1, prompt: 'test prompt', - transcript_path: 'outputs/transcript.jsonl', - raw_provider_log_path: 'outputs/raw/provider.log', + transcript_path: 'transcript.jsonl', + raw_provider_log_path: 'provider.log', }, ], }); diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index a7c71ad39..89fda3f30 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -749,21 +749,22 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, const outputDir = path.join(tempDir, 'artifacts'); await writeArtifactsFromResults([result], outputDir); - const outputsDir = path.join(outputDir, 'test-dataset', 'case-1', 'outputs'); - expect(readFileSync(path.join(outputsDir, 'raw', 'provider.log'), 'utf8')).toBe( + const artifactDir = path.join(outputDir, 'test-dataset', 'case-1'); + const outputsDir = path.join(artifactDir, 'outputs'); + expect(readFileSync(path.join(artifactDir, 'provider.log'), 'utf8')).toBe( '{"event":"provider-native"}\n', ); - expect(readdirSync(outputsDir)).toContain('transcript.jsonl'); + expect(readdirSync(artifactDir)).toContain('transcript.jsonl'); + expect(readdirSync(outputsDir)).not.toContain('transcript.jsonl'); expect(readdirSync(outputsDir)).not.toContain('transcript.json'); const indexRows = readFileSync(path.join(outputDir, 'index.jsonl'), 'utf8') .trim() .split('\n') .map((line) => JSON.parse(line) as Record); - expect(indexRows[0]?.raw_provider_log_path).toBe( - 'test-dataset/case-1/outputs/raw/provider.log', - ); - expect(indexRows[0]?.transcript_path).toBe('test-dataset/case-1/outputs/transcript.jsonl'); + expect(indexRows[0]?.raw_provider_log_path).toBe('test-dataset/case-1/provider.log'); + expect(indexRows[0]?.trace_path).toBe('test-dataset/case-1/trace.json'); + expect(indexRows[0]?.transcript_path).toBe('test-dataset/case-1/transcript.jsonl'); }); it('reports failed progress status for batch item errors', async () => { diff --git a/packages/core/test/evaluation/providers/log-directory.test.ts b/packages/core/test/evaluation/providers/log-directory.test.ts new file mode 100644 index 000000000..627140d62 --- /dev/null +++ b/packages/core/test/evaluation/providers/log-directory.test.ts @@ -0,0 +1,45 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import path from 'node:path'; + +import { resolveDefaultProviderLogDir } from '../../../src/evaluation/providers/log-directory.js'; + +describe('resolveDefaultProviderLogDir', () => { + const previousRunDir = process.env.AGENTV_RUN_DIR; + + afterEach(() => { + if (previousRunDir === undefined) { + process.env.AGENTV_RUN_DIR = undefined; + } else { + process.env.AGENTV_RUN_DIR = previousRunDir; + } + }); + + it('places default provider logs inside the case folder for the active run', () => { + process.env.AGENTV_RUN_DIR = path.join('/repo', '.agentv', 'results', 'default', 'run-001'); + + expect( + resolveDefaultProviderLogDir('copilot-cli', { + suite: 'demo-suite', + evalCaseId: 'case/one', + }), + ).toBe( + path.join( + '/repo', + '.agentv', + 'results', + 'default', + 'run-001', + 'demo-suite', + 'case_one', + 'logs', + 'copilot-cli', + ), + ); + }); + + it('does not fall back to .agentv/logs without an active run', () => { + process.env.AGENTV_RUN_DIR = undefined; + + expect(resolveDefaultProviderLogDir('copilot-cli', { evalCaseId: 'case-one' })).toBeUndefined(); + }); +}); diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index b25d4bd13..69d5329c9 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -140,8 +140,8 @@ function writeRunArtifactsWithPointers( timestamp: string, ): void { writeRunArtifacts(runDir, experiment, timestamp); - const outputsDir = path.join(runDir, 'alpha', 'outputs'); - mkdirSync(outputsDir, { recursive: true }); + const artifactDir = path.join(runDir, 'alpha'); + mkdirSync(artifactDir, { recursive: true }); const traceContent = Buffer.from( JSON.stringify({ schema_version: 'agentv.trace.v1', @@ -157,8 +157,8 @@ function writeRunArtifactsWithPointers( content: 'sidecar transcript', })}\n`, ); - writeFileSync(path.join(outputsDir, 'trace.json'), traceContent); - writeFileSync(path.join(outputsDir, 'transcript.jsonl'), transcriptContent); + writeFileSync(path.join(artifactDir, 'trace.json'), traceContent); + writeFileSync(path.join(artifactDir, 'transcript.jsonl'), transcriptContent); const traceSha = sha256Hex(traceContent); const transcriptSha = sha256Hex(transcriptContent); @@ -170,9 +170,9 @@ function writeRunArtifactsWithPointers( artifact_pointers: { trace: { ref: AGENTV_RESULTS_REFS.artifacts, - key: 'traces/alpha/outputs/trace.json', + key: 'traces/alpha/trace.json', object_version: `sha256:${traceSha}`, - path: 'alpha/outputs/trace.json', + path: 'alpha/trace.json', sha256: traceSha, size: traceContent.byteLength, schema_version: 'agentv.trace.v1', @@ -181,9 +181,9 @@ function writeRunArtifactsWithPointers( }, transcript: { ref: AGENTV_RESULTS_REFS.artifacts, - key: 'transcripts/alpha/outputs/transcript.jsonl', + key: 'transcripts/alpha/transcript.jsonl', object_version: `sha256:${transcriptSha}`, - path: 'alpha/outputs/transcript.jsonl', + path: 'alpha/transcript.jsonl', sha256: transcriptSha, size: transcriptContent.byteLength, schema_version: 'agentv.transcript.v1', @@ -1050,7 +1050,7 @@ describe('results repo write path', () => { git('git add README.md && git commit --quiet -m "seed results repo"', resultsRepoDir); const runTimestamp = '2026-06-17T11-00-00-000Z'; - const runDir = path.join(projectDir, '.agentv', 'results', 'runs', 'external', runTimestamp); + const runDir = path.join(projectDir, '.agentv', 'results', 'external', runTimestamp); writeRunArtifacts(runDir, 'external', '2026-06-17T11:00:00.000Z'); const published = await directPushResults({ @@ -1223,15 +1223,15 @@ describe('results repo write path', () => { ); expect(resultTree).toContain(`runs/${destinationPath}/index.jsonl`); expect(resultTree).toContain(`runs/${destinationPath}/benchmark.json`); - expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/outputs/trace.json`); - expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/outputs/transcript.jsonl`); + expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/trace.json`); + expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/transcript.jsonl`); const artifactTree = git( `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_REFS.artifacts}`, rootDir, ); - expect(artifactTree).toContain(`runs/${destinationPath}/alpha/outputs/trace.json`); - expect(artifactTree).toContain(`runs/${destinationPath}/alpha/outputs/transcript.jsonl`); + expect(artifactTree).toContain(`runs/${destinationPath}/alpha/trace.json`); + expect(artifactTree).toContain(`runs/${destinationPath}/alpha/transcript.jsonl`); expect(artifactTree).not.toContain(`runs/${destinationPath}/benchmark.json`); expect(artifactTree).not.toContain(`runs/${destinationPath}/index.jsonl`); @@ -1324,14 +1324,14 @@ describe('results repo write path', () => { ); expect(resultTree).toContain(`runs/${destinationPath}/index.jsonl`); expect(resultTree).toContain(`runs/${destinationPath}/benchmark.json`); - expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/outputs/trace.json`); - expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/outputs/transcript.jsonl`); + expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/trace.json`); + expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/transcript.jsonl`); const artifactTree = git( `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_REFS.artifacts}`, rootDir, ); - expect(artifactTree).toContain(`runs/${destinationPath}/alpha/outputs/trace.json`); - expect(artifactTree).toContain(`runs/${destinationPath}/alpha/outputs/transcript.jsonl`); + expect(artifactTree).toContain(`runs/${destinationPath}/alpha/trace.json`); + expect(artifactTree).toContain(`runs/${destinationPath}/alpha/transcript.jsonl`); await expect( directPushResults({ @@ -1947,7 +1947,7 @@ describe('results branch stable genesis', () => { describe('buildWipBranchName', () => { it('produces an agentv/wip// branch name', () => { - const runDir = '/some/path/.agentv/results/runs/default/2026-01-15T10-00-00'; + const runDir = '/some/path/.agentv/results/default/2026-01-15T10-00-00'; const branch = buildWipBranchName(runDir); expect(branch).toMatch(/^agentv\/wip\/[^/]+\/2026-01-15T10-00-00$/); }); diff --git a/packages/core/test/evaluation/trace-envelope.test.ts b/packages/core/test/evaluation/trace-envelope.test.ts index e4beafb6a..c9ba6880b 100644 --- a/packages/core/test/evaluation/trace-envelope.test.ts +++ b/packages/core/test/evaluation/trace-envelope.test.ts @@ -116,8 +116,8 @@ describe('execution trace artifact v1', () => { }, }, artifacts: { - trace_path: 'outputs/trace.json', - transcript_path: 'outputs/transcript.jsonl', + trace_path: 'trace.json', + transcript_path: 'transcript.jsonl', }, }, ); @@ -160,7 +160,7 @@ describe('execution trace artifact v1', () => { target: 'replay_coding_agent', }); expect(wire.trace.format).toBe('otlp_openinference_spans'); - expect(wire.artifacts?.trace_path).toBe('outputs/trace.json'); + expect(wire.artifacts?.trace_path).toBe('trace.json'); expect(wire.artifacts).not.toHaveProperty('execution_trace_path'); expect(wire.trace.spans[0]?.attributes['gen_ai.operation.name']).toBe('invoke_agent'); expect(wire.trace.spans[0]?.attributes['openinference.span.kind']).toBe('AGENT'); diff --git a/packages/core/test/evaluation/trace-normalization.test.ts b/packages/core/test/evaluation/trace-normalization.test.ts index 5f619a036..d85e2dd20 100644 --- a/packages/core/test/evaluation/trace-normalization.test.ts +++ b/packages/core/test/evaluation/trace-normalization.test.ts @@ -297,9 +297,9 @@ describe('Dashboard trace artifact normalization', () => { message: 'Preserved warning', }, ], - artifacts: { trace_path: 'outputs/trace.json' }, + artifacts: { trace_path: 'trace.json' }, }, - { runId: 'run-agentv', artifactPath: 'demo/agentv/outputs/trace.json' }, + { runId: 'run-agentv', artifactPath: 'demo/agentv/trace.json' }, ); expect(result.status).toBe('ok'); @@ -313,7 +313,7 @@ describe('Dashboard trace artifact normalization', () => { expect.objectContaining({ code: 'source_warning', message: 'Preserved warning' }), ]); expect(result.traceSession.artifact_links).toEqual([ - { name: 'trace_path', path: 'outputs/trace.json' }, + { name: 'trace_path', path: 'trace.json' }, ]); }); }); diff --git a/plugins/agentv-self/skills/image-compress-and-docs/SKILL.md b/plugins/agentv-self/skills/image-compress-and-docs/SKILL.md index a1dcf7fc3..29dd7ae7a 100644 --- a/plugins/agentv-self/skills/image-compress-and-docs/SKILL.md +++ b/plugins/agentv-self/skills/image-compress-and-docs/SKILL.md @@ -46,7 +46,7 @@ agent-browser --session docs-shots close kill $(lsof -ti:14800) 2>/dev/null ``` -**Screenshots with realistic data:** Dashboard screenshots must have populated data — multiple runs with varying pass rates and real targets. If results are sparse, create synthetic JSONL files in `.agentv/results/runs///index.jsonl` with realistic fields before launching Dashboard. +**Screenshots with realistic data:** Dashboard screenshots must have populated data — multiple runs with varying pass rates and real targets. If results are sparse, create synthetic JSONL files in `.agentv/results///index.jsonl` with realistic fields before launching Dashboard. Synthetic JSONL record format: ```json diff --git a/scripts/deploy-example-results-repos.ts b/scripts/deploy-example-results-repos.ts index 220103181..e353adb2e 100644 --- a/scripts/deploy-example-results-repos.ts +++ b/scripts/deploy-example-results-repos.ts @@ -449,7 +449,7 @@ function initializeResultsRepo(params: { Shared AgentV evaluation results repository. -AgentV stores run artifacts under \`.agentv/results/runs/\` and syncs them through normal git fetch/push operations. +AgentV stores run artifacts under \`.agentv/results/\` and syncs them through normal git fetch/push operations. `; changed = writeFileIfChanged(path.join(params.repoDir, 'README.md'), readme, params.dryRun) || changed; diff --git a/scripts/setup-dashboard-deployment.sh b/scripts/setup-dashboard-deployment.sh index 9d2c104bc..280e05ad1 100755 --- a/scripts/setup-dashboard-deployment.sh +++ b/scripts/setup-dashboard-deployment.sh @@ -114,7 +114,7 @@ EOF } write_project_config() { - mkdir -p "$project_dir/.agentv/results/runs" + mkdir -p "$project_dir/.agentv/results/default" local results_url results_url="$(repo_to_url "$results_repo")" if [[ -f "$source_dir/.agentv/targets.yaml" ]]; then diff --git a/skills-data/agentv-bench/SKILL.md b/skills-data/agentv-bench/SKILL.md index 21a8f7271..1efa305e4 100644 --- a/skills-data/agentv-bench/SKILL.md +++ b/skills-data/agentv-bench/SKILL.md @@ -123,7 +123,7 @@ Prefer deterministic graders over LLM graders whenever possible. If an assertion This section is one continuous sequence — don't stop partway through. -Each run produces a new `.agentv/results/runs//` directory automatically. Use timestamps to identify iterations when comparing runs. +Each run produces a new `.agentv/results/default//` directory automatically. Use timestamps to identify iterations when comparing runs. ### Choosing a run mode @@ -346,7 +346,7 @@ If a prompt file is referenced in both task input and grader configs, optimize f After improving: 1. Apply your changes to the agent's prompts/skills/config -2. Re-run all test cases (agentv creates a new `.agentv/results/runs//` directory automatically) +2. Re-run all test cases (agentv creates a new `.agentv/results/default//` directory automatically) 3. Compare against the previous iteration (Step 4). If running in automated mode, use the **automated keep/discard** logic below instead of manual judgment — it will decide whether to keep or revert the change for you. 4. Present results to the user (or log the decision if running automated keep/discard) 5. Stop when ANY of: diff --git a/skills-data/agentv-bench/agents/executor.md b/skills-data/agentv-bench/agents/executor.md index 2543e0ec1..5b6b4437d 100644 --- a/skills-data/agentv-bench/agents/executor.md +++ b/skills-data/agentv-bench/agents/executor.md @@ -14,7 +14,7 @@ You are the executor for an AgentV evaluation test case. Your job is to **perfor You are the target agent being evaluated. Do the task to the best of your ability — your output will be graded by a separate grader agent. **You will receive these parameters:** -- `test-dir`: Path to the test case directory (e.g., `.agentv/results/runs///`) +- `test-dir`: Path to the test case directory (e.g., `.agentv/results/default///`) ## Process diff --git a/skills-data/agentv-bench/agents/grader.md b/skills-data/agentv-bench/agents/grader.md index 97f88126e..099281164 100644 --- a/skills-data/agentv-bench/agents/grader.md +++ b/skills-data/agentv-bench/agents/grader.md @@ -18,7 +18,7 @@ You are the grader for an AgentV evaluation test case. You have two jobs: **grad - `eval-path`: Path to the eval YAML file - `test-id`: The test case ID - `response-file`: Path to the executor's response (e.g., `response.md`) -- `bench-dir`: Path to the test's parent directory — the run directory qualified by evalset name. Example: `.agentv/results/runs////`. The evalset name comes from the eval.yaml `name` field; when absent, it falls back to the eval file's basename (e.g. `my-suite.eval.yaml` → `my-suite`), matching CLI mode. The grader writes results under `{bench-dir}/{test-id}/...`. +- `bench-dir`: Path to the test's parent directory — the run directory qualified by evalset name. Example: `.agentv/results////`. The evalset name comes from the eval.yaml `name` field; when absent, it falls back to the eval file's basename (e.g. `my-suite.eval.yaml` → `my-suite`), matching CLI mode. The grader writes results under `{bench-dir}/{test-id}/...`. - `timing-file`: Path to `timing.json` (for execution-metrics/latency/cost assertions) ## Process diff --git a/skills-data/agentv-bench/references/autoresearch.md b/skills-data/agentv-bench/references/autoresearch.md index eea2f4045..037b3f469 100644 --- a/skills-data/agentv-bench/references/autoresearch.md +++ b/skills-data/agentv-bench/references/autoresearch.md @@ -56,14 +56,14 @@ Before proceeding to the next iteration, log the decision and rationale so the u Iteration 2: KEEP wins=3, losses=1, ties=6, meanDelta=+0.05 Rationale: candidate wins outweigh losses (3 > 1) - Baseline promoted: .agentv/results/runs/20250101-120000/index.jsonl + Baseline promoted: .agentv/results/default/20250101-120000/index.jsonl ``` ``` Iteration 3: DISCARD wins=1, losses=2, ties=7, meanDelta=-0.03 Rationale: candidate losses outweigh wins (2 > 1) - Reverted to baseline: .agentv/results/runs/20250101-110000/index.jsonl + Reverted to baseline: .agentv/results/default/20250101-110000/index.jsonl Next: try a different mutation ``` @@ -124,7 +124,7 @@ Shell out to `agentv eval --experiment autoresearch-` via the Each cycle is a standard eval run. Autoresearch session metadata lives in `_autoresearch/` within the experiment directory: ``` -.agentv/results/runs// +.agentv/results// _autoresearch/ iterations.jsonl # one line per cycle — data for chart + mutator trajectory.html # live-updating score trajectory chart @@ -187,7 +187,7 @@ Follow this step-by-step procedure to execute autoresearch: 1. Determine the **artifact path** (file or directory to optimize) and **eval path** (EVAL.yaml or evals.json). 2. Detect **artifact mode**: `file` if the artifact path is a file, `directory` if it's a directory. 3. Derive the **experiment name**: `autoresearch-` from the artifact filename/dirname, or use a user-provided name. -4. Set the experiment directory: `.agentv/results/runs//`. +4. Set the experiment directory: `.agentv/results//`. 5. Create the `_autoresearch/` subdirectory inside the experiment directory. 6. Record `initial_sha=$(git rev-parse HEAD)` — the commit before any mutations. 7. Copy `scripts/trajectory.html` to `_autoresearch/trajectory.html`. diff --git a/skills-data/agentv-bench/references/schemas.md b/skills-data/agentv-bench/references/schemas.md index b02ca0633..a2e430887 100644 --- a/skills-data/agentv-bench/references/schemas.md +++ b/skills-data/agentv-bench/references/schemas.md @@ -164,7 +164,7 @@ Output from the grader agent. Located at `/grading.json`. ## metrics.json -Output from the executor agent. Located at `/outputs/metrics.json`. +Output from the executor agent. Located at `/metrics.json`. ```json { diff --git a/skills-data/agentv-bench/references/subagent-pipeline.md b/skills-data/agentv-bench/references/subagent-pipeline.md index 78c773fbe..91b77e637 100644 --- a/skills-data/agentv-bench/references/subagent-pipeline.md +++ b/skills-data/agentv-bench/references/subagent-pipeline.md @@ -43,7 +43,7 @@ even in subagent mode. For evals with CLI targets, `pipeline run` handles input extraction, target invocation, and code grading in one step. When `--out` is omitted, the output directory defaults to -`.agentv/results/runs/` (same convention as `agentv eval`): +`.agentv/results/default/` (same convention as `agentv eval`): ```bash # Extract inputs and invoke all CLI targets in parallel: @@ -69,7 +69,7 @@ opted out via `subagent_mode_allowed: false` in `.agentv/targets.yaml`), fall ba ### Step 1: Extract inputs ```bash -# Defaults to .agentv/results/runs/ +# Defaults to .agentv/results/default/ agentv pipeline input evals/repro.eval.yaml ``` @@ -116,7 +116,7 @@ LLM grading → merge and validate). Use individual commands when you need control over each step with CLI targets: ```bash -# Step 1: Extract inputs (defaults to .agentv/results/runs/) +# Step 1: Extract inputs (defaults to .agentv/results/default/) agentv pipeline input evals/repro.eval.yaml # Step 2: run_tests.py invokes CLI targets (or use pipeline run instead) @@ -162,7 +162,7 @@ The path hierarchy mirrors the CLI mode: `` comes from the `name` the eval.yaml. The target is recorded in `manifest.json` — one run = one target. ``` -.agentv/results/runs/// +.agentv/results/// ├── manifest.json ← eval metadata, target, test_ids ├── index.jsonl ← per-test scores ├── benchmark.json ← aggregate statistics diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 50298c0b9..d823d930a 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -558,16 +558,16 @@ agentv eval assert --agent-output "..." --agent-input "..." agentv import claude --session-id # Re-run only execution errors from a previous run -agentv eval --retry-errors .agentv/results/runs//index.jsonl +agentv eval --retry-errors .agentv/results/default//index.jsonl # Validate eval file agentv validate # Compare results — N-way matrix from a canonical run manifest -agentv compare .agentv/results/runs//index.jsonl -agentv compare .agentv/results/runs//index.jsonl --baseline # CI regression gate -agentv compare .agentv/results/runs//index.jsonl --baseline --candidate # pairwise -agentv compare .agentv/results/runs//index.jsonl .agentv/results/runs//index.jsonl +agentv compare .agentv/results/default//index.jsonl +agentv compare .agentv/results/default//index.jsonl --baseline # CI regression gate +agentv compare .agentv/results/default//index.jsonl --baseline --candidate # pairwise +agentv compare .agentv/results/default//index.jsonl .agentv/results/default//index.jsonl # Author assertions directly in the eval file # Prefer simple assertions when they fit the criteria; use deterministic or LLM-based graders when needed From fe0d735e82e92fc6b5e6ba2b26c57ea7a3152044 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 23 Jun 2026 01:46:11 +0200 Subject: [PATCH 2/5] feat(results): use tag revisions for result tags Bead: av-vqe --- .../src/commands/results/remote-metadata.ts | 65 +++----- apps/cli/src/commands/results/remote.ts | 17 ++- apps/cli/src/commands/results/run-oplog.ts | 141 ------------------ apps/cli/src/commands/results/run-state.ts | 68 +++++++++ apps/cli/src/commands/results/run-tags.ts | 48 ++---- apps/cli/src/commands/results/serve.ts | 100 ++++++++++--- .../commands/results/remote-metadata.test.ts | 30 ++-- .../test/commands/results/run-oplog.test.ts | 108 -------------- .../test/commands/results/run-tags.test.ts | 16 +- apps/cli/test/commands/results/serve.test.ts | 119 +++++++++------ .../dashboard/src/components/AnalyticsTab.tsx | 7 +- apps/dashboard/src/lib/api.ts | 26 +++- apps/dashboard/src/lib/types.ts | 18 +-- .../src/content/docs/docs/tools/dashboard.mdx | 4 +- .../src/content/docs/docs/tools/results.mdx | 2 +- 15 files changed, 340 insertions(+), 429 deletions(-) delete mode 100644 apps/cli/src/commands/results/run-oplog.ts create mode 100644 apps/cli/src/commands/results/run-state.ts delete mode 100644 apps/cli/test/commands/results/run-oplog.test.ts diff --git a/apps/cli/src/commands/results/remote-metadata.ts b/apps/cli/src/commands/results/remote-metadata.ts index 7c26c699d..b6285708f 100644 --- a/apps/cli/src/commands/results/remote-metadata.ts +++ b/apps/cli/src/commands/results/remote-metadata.ts @@ -18,13 +18,7 @@ import { execFileSync } from 'node:child_process'; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import path from 'node:path'; -import { - type RunOplogWatermark, - buildRunIdFromRelativePath, - createRunTagsSetOperation, - normalizeRunOplogWatermark, - watermarkFromRunOperation, -} from './run-oplog.js'; +import { assertExpectedTagRevision, createTagRevision, normalizeTagRevision } from './run-state.js'; import { RUN_TAGS_FILENAME, normalizeTags } from './run-tags.js'; const RESULTS_RUNS_DIR = 'runs'; @@ -33,7 +27,7 @@ const REMOTE_METADATA_RUNS_DIR = path.join('metadata', 'runs'); interface TagsFile { readonly tags: string[]; readonly updatedAt?: string; - readonly oplogWatermark?: RunOplogWatermark; + readonly tagRevision: string; } interface RemoteRunMetadataPaths { @@ -57,7 +51,7 @@ export interface RemoteRunTagState { readonly pendingTags?: string[]; readonly dirty: boolean; readonly updatedAt?: string; - readonly oplogWatermark: RunOplogWatermark; + readonly tagRevision: string; readonly metadataPath: string; } @@ -126,7 +120,7 @@ function parseTagsFile(content: string): TagsFile | undefined { return { tags, updatedAt, - oplogWatermark: normalizeRunOplogWatermark(record.oplog_watermark, updatedAt), + tagRevision: normalizeTagRevision(record.tag_revision, tags, updatedAt), }; } @@ -135,23 +129,12 @@ function equalTags(a: readonly string[], b: readonly string[]): boolean { return a.every((tag, index) => tag === b[index]); } -function equalWatermarks( - a: RunOplogWatermark | undefined, - b: RunOplogWatermark | undefined, -): boolean { - return ( - a?.ref === b?.ref && a?.operation_id === b?.operation_id && a?.updated_at === b?.updated_at - ); -} - function equalTagFiles(a: TagsFile | undefined, b: TagsFile | undefined): boolean { if (a === undefined || b === undefined) { return a === b; } return ( - equalTags(a.tags, b.tags) && - a.updatedAt === b.updatedAt && - equalWatermarks(a.oplogWatermark, b.oplogWatermark) + equalTags(a.tags, b.tags) && a.updatedAt === b.updatedAt && a.tagRevision === b.tagRevision ); } @@ -226,22 +209,23 @@ function toRemoteRunTagState(context: RemoteRunTagsContext): RemoteRunTagState { const dirty = context.localOverlayTags ? !equalTagFiles(context.localOverlayTags, context.baseOverlayTags) : !equalTags(effectiveTags, remoteTags); - const watermark = - context.localOverlayTags?.oplogWatermark ?? - context.baseOverlayTags?.oplogWatermark ?? - context.artifactTags?.oplogWatermark ?? - normalizeRunOplogWatermark(undefined); + const updatedAt = + context.localOverlayTags?.updatedAt ?? + context.baseOverlayTags?.updatedAt ?? + context.artifactTags?.updatedAt; + const tagRevision = + context.localOverlayTags?.tagRevision ?? + context.baseOverlayTags?.tagRevision ?? + context.artifactTags?.tagRevision ?? + createTagRevision(effectiveTags, updatedAt); return { tags: effectiveTags, remoteTags, ...(dirty && { pendingTags: effectiveTags }), dirty, - updatedAt: - context.localOverlayTags?.updatedAt ?? - context.baseOverlayTags?.updatedAt ?? - context.artifactTags?.updatedAt, - oplogWatermark: watermark, + updatedAt, + tagRevision, metadataPath: context.paths.overlayTagsPath, }; } @@ -276,11 +260,14 @@ export function writeRemoteRunTags( manifestPath: string, tags: readonly string[], comparisonRef?: string, + expectedTagRevision?: string, ): RemoteRunTagState { assertWritableResultsRepo(repoDir); const cleaned = normalizeTags(tags); const context = readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef); + const currentState = toRemoteRunTagState(context); + assertExpectedTagRevision(expectedTagRevision, currentState.tagRevision); const remoteTags = context.baseOverlayTags?.tags ?? context.artifactTags?.tags ?? []; if ( @@ -292,16 +279,11 @@ export function writeRemoteRunTags( return readRemoteRunTags(repoDir, manifestPath, comparisonRef); } - const operation = createRunTagsSetOperation({ - runId: buildRunIdFromRelativePath(context.paths.runRelativePath), - runPath: context.paths.runRelativePath, - tags: cleaned, - actor: { kind: 'dashboard' }, - }); + const updatedAt = new Date().toISOString(); const entry = { tags: cleaned, - updated_at: operation.authored_at, - oplog_watermark: watermarkFromRunOperation(operation), + updated_at: updatedAt, + tag_revision: createTagRevision(cleaned, updatedAt), }; mkdirSync(path.dirname(context.paths.overlayTagsPath), { recursive: true }); writeFileSync(context.paths.overlayTagsPath, `${JSON.stringify(entry, null, 2)}\n`, 'utf8'); @@ -312,6 +294,7 @@ export function deleteRemoteRunTags( repoDir: string, manifestPath: string, comparisonRef?: string, + expectedTagRevision?: string, ): RemoteRunTagState { - return writeRemoteRunTags(repoDir, manifestPath, [], comparisonRef); + return writeRemoteRunTags(repoDir, manifestPath, [], comparisonRef, expectedTagRevision); } diff --git a/apps/cli/src/commands/results/remote.ts b/apps/cli/src/commands/results/remote.ts index dea570482..6dda5382c 100644 --- a/apps/cli/src/commands/results/remote.ts +++ b/apps/cli/src/commands/results/remote.ts @@ -538,6 +538,7 @@ export async function setRemoteRunTags( meta: Pick, tags: readonly string[], projectId?: string, + expectedTagRevision?: string, ): Promise { if (meta.source !== 'remote' && !meta.on_remote) { throw new Error('Remote metadata can only be set on remote runs'); @@ -551,13 +552,20 @@ export async function setRemoteRunTags( throw new Error('Remote metadata can only be set on remote runs'); } assertWritableResultsRepo(config.path); - return writeRemoteRunTags(config.path, manifestPath, tags, getResultsStorageRef(config)); + return writeRemoteRunTags( + config.path, + manifestPath, + tags, + getResultsStorageRef(config), + expectedTagRevision, + ); } export async function clearRemoteRunTags( cwd: string, meta: Pick, projectId?: string, + expectedTagRevision?: string, ): Promise { if (meta.source !== 'remote' && !meta.on_remote) { throw new Error('Remote metadata can only be removed from remote runs'); @@ -571,7 +579,12 @@ export async function clearRemoteRunTags( throw new Error('Remote metadata can only be removed from remote runs'); } assertWritableResultsRepo(config.path); - return deleteRemoteRunTags(config.path, manifestPath, getResultsStorageRef(config)); + return deleteRemoteRunTags( + config.path, + manifestPath, + getResultsStorageRef(config), + expectedTagRevision, + ); } export async function maybeAutoExportRunArtifacts( diff --git a/apps/cli/src/commands/results/run-oplog.ts b/apps/cli/src/commands/results/run-oplog.ts deleted file mode 100644 index c307e5841..000000000 --- a/apps/cli/src/commands/results/run-oplog.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { randomUUID } from 'node:crypto'; - -/** - * Minimal run operation-log contract used by Dashboard read models. - * - * The raw oplog storage branch is intentionally not implemented here. This - * module only centralizes the ref name, a small typed operation envelope for - * tag replacement, and the materialized final-state shape that readers consume. - */ - -export const RUN_OPLOG_REF = 'agentv/oplog/v1'; -export const RUN_OPERATION_SCHEMA_VERSION = 'agentv.run_operation.v1'; - -export type RunFinalStateLifecycle = 'active' | 'hidden' | 'deleted'; - -export interface RunOplogWatermark { - readonly ref: typeof RUN_OPLOG_REF; - readonly operation_id?: string; - readonly updated_at?: string; -} - -export interface RunFinalState { - readonly lifecycle: RunFinalStateLifecycle; - readonly tags: string[]; -} - -export interface RunReadStateFields { - readonly final_state: RunFinalState; - readonly oplog_watermark: RunOplogWatermark; -} - -export type RunOperationActorKind = 'dashboard' | 'cli' | 'ci' | 'agent' | 'unknown'; - -export interface RunOperationActor { - readonly kind: RunOperationActorKind; - readonly id?: string; -} - -export interface RunOperationSubject { - readonly run_id: string; - readonly run_path?: string; -} - -export interface RunTagsSetOperation { - readonly schema_version: typeof RUN_OPERATION_SCHEMA_VERSION; - readonly operation_id: string; - readonly operation_type: 'run.tags.set'; - readonly authored_at: string; - readonly actor: RunOperationActor; - readonly subject: RunOperationSubject; - readonly payload: { - readonly tags: string[]; - }; -} - -export type RunOperationEnvelope = RunTagsSetOperation; - -export function buildRunIdFromRelativePath(relativeRunPath: string): string { - const segments = relativeRunPath.split(/[\\/]+/).filter(Boolean); - if (segments.length >= 2) { - const experiment = segments.slice(0, -1).join('/'); - const runName = segments.at(-1) ?? relativeRunPath; - return experiment === 'default' ? runName : `${experiment}::${runName}`; - } - return segments[0] ?? relativeRunPath; -} - -export function createRunTagsSetOperation(input: { - readonly runId: string; - readonly runPath?: string; - readonly tags: readonly string[]; - readonly actor?: RunOperationActor; - readonly authoredAt?: string; - readonly operationId?: string; -}): RunTagsSetOperation { - return { - schema_version: RUN_OPERATION_SCHEMA_VERSION, - operation_id: input.operationId ?? randomUUID(), - operation_type: 'run.tags.set', - authored_at: input.authoredAt ?? new Date().toISOString(), - actor: input.actor ?? { kind: 'unknown' }, - subject: { - run_id: input.runId, - ...(input.runPath ? { run_path: input.runPath } : {}), - }, - payload: { - tags: [...input.tags], - }, - }; -} - -export function watermarkFromRunOperation(operation: RunOperationEnvelope): RunOplogWatermark { - return { - ref: RUN_OPLOG_REF, - operation_id: operation.operation_id, - updated_at: operation.authored_at, - }; -} - -export function normalizeRunOplogWatermark( - input: unknown, - fallbackUpdatedAt?: string, -): RunOplogWatermark { - if (input && typeof input === 'object') { - const record = input as Record; - const operationId = record.operation_id; - const updatedAt = record.updated_at; - return { - ref: RUN_OPLOG_REF, - ...(typeof operationId === 'string' && operationId ? { operation_id: operationId } : {}), - ...(typeof updatedAt === 'string' && updatedAt - ? { updated_at: updatedAt } - : fallbackUpdatedAt - ? { updated_at: fallbackUpdatedAt } - : {}), - }; - } - - return { - ref: RUN_OPLOG_REF, - ...(fallbackUpdatedAt ? { updated_at: fallbackUpdatedAt } : {}), - }; -} - -export function materializeRunState(input?: { - readonly lifecycle?: RunFinalStateLifecycle; - readonly tags?: readonly string[]; - readonly watermark?: RunOplogWatermark; - readonly updatedAt?: string; -}): RunReadStateFields { - const tags = [...(input?.tags ?? [])]; - const watermark = input?.watermark ?? normalizeRunOplogWatermark(undefined, input?.updatedAt); - - return { - final_state: { - lifecycle: input?.lifecycle ?? 'active', - tags, - }, - oplog_watermark: watermark, - }; -} diff --git a/apps/cli/src/commands/results/run-state.ts b/apps/cli/src/commands/results/run-state.ts new file mode 100644 index 000000000..cfc9c6588 --- /dev/null +++ b/apps/cli/src/commands/results/run-state.ts @@ -0,0 +1,68 @@ +import { createHash } from 'node:crypto'; + +export type RunFinalStateLifecycle = 'active' | 'hidden' | 'deleted'; + +export interface RunFinalState { + readonly lifecycle: RunFinalStateLifecycle; + readonly tags: string[]; +} + +export interface RunReadStateFields { + readonly final_state: RunFinalState; + readonly tag_revision: string; +} + +export class TagRevisionConflictError extends Error { + readonly expectedRevision: string; + readonly currentRevision: string; + + constructor(expectedRevision: string, currentRevision: string) { + super('Run tags changed. Refresh the run and try again.'); + this.name = 'TagRevisionConflictError'; + this.expectedRevision = expectedRevision; + this.currentRevision = currentRevision; + } +} + +export function createTagRevision(tags: readonly string[], updatedAt?: string): string { + const hash = createHash('sha256'); + hash.update('agentv.run_tags.v1\0'); + hash.update(JSON.stringify({ tags: [...tags], updated_at: updatedAt ?? '' })); + return `sha256:${hash.digest('hex')}`; +} + +export function normalizeTagRevision( + input: unknown, + tags: readonly string[], + updatedAt?: string, +): string { + return typeof input === 'string' && input.trim().length > 0 + ? input + : createTagRevision(tags, updatedAt); +} + +export function assertExpectedTagRevision( + expectedRevision: string | undefined, + currentRevision: string, +): void { + if (expectedRevision !== undefined && expectedRevision !== currentRevision) { + throw new TagRevisionConflictError(expectedRevision, currentRevision); + } +} + +export function materializeRunState(input?: { + readonly lifecycle?: RunFinalStateLifecycle; + readonly tags?: readonly string[]; + readonly tagRevision?: string; + readonly updatedAt?: string; +}): RunReadStateFields { + const tags = [...(input?.tags ?? [])]; + + return { + final_state: { + lifecycle: input?.lifecycle ?? 'active', + tags, + }, + tag_revision: input?.tagRevision ?? createTagRevision(tags, input?.updatedAt), + }; +} diff --git a/apps/cli/src/commands/results/run-tags.ts b/apps/cli/src/commands/results/run-tags.ts index 8f7e50459..992862293 100644 --- a/apps/cli/src/commands/results/run-tags.ts +++ b/apps/cli/src/commands/results/run-tags.ts @@ -10,7 +10,7 @@ * { * "tags": ["baseline", "v2-prompt"], * "updated_at": "2026-04-10T00:00:00.000Z", - * "oplog_watermark": { "ref": "agentv/oplog/v1" } + * "tag_revision": "sha256:..." * } * ``` * @@ -24,7 +24,7 @@ * - No control characters (\n, \t, DEL, etc.) * - Tags are deduplicated case-sensitively * - A run can have at most 20 tags - * - Writing an empty array records a clear/tombstone state with a watermark + * - Writing an empty array records an intentional clear state * * To extend (e.g. add colored labels or descriptions): add optional fields * to `RunTagsFile` and keep the schema additive so older files still parse. @@ -33,13 +33,7 @@ import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; import path from 'node:path'; -import { - type RunOplogWatermark, - buildRunIdFromRelativePath, - createRunTagsSetOperation, - normalizeRunOplogWatermark, - watermarkFromRunOperation, -} from './run-oplog.js'; +import { createTagRevision, normalizeTagRevision } from './run-state.js'; export const RUN_TAGS_FILENAME = 'tags.json'; @@ -54,8 +48,8 @@ export interface RunTagsFile { tags: string[]; /** ISO-8601 timestamp of last update. */ updated_at: string; - /** Watermark for the operation-log state this materialized tag list reflects. */ - oplog_watermark?: RunOplogWatermark; + /** Optimistic-concurrency token for this materialized tag list. */ + tag_revision: string; } /** Resolve the tags sidecar path given a run manifest (index.jsonl) path. */ @@ -63,24 +57,6 @@ export function runTagsPath(manifestPath: string): string { return path.join(path.dirname(manifestPath), RUN_TAGS_FILENAME); } -function inferRunRelativePath(manifestPath: string): string { - const runDir = path.dirname(manifestPath); - const segments = runDir.split(path.sep); - const resultsIndex = segments.lastIndexOf('results'); - if (resultsIndex >= 0 && resultsIndex < segments.length - 2) { - const runSegments = segments.slice(resultsIndex + 1); - const namespace = runSegments[0]; - if (namespace !== 'metadata' && namespace !== 'runs') { - return runSegments.join('/'); - } - } - const runsIndex = segments.lastIndexOf('runs'); - if (runsIndex >= 0 && runsIndex < segments.length - 1) { - return segments.slice(runsIndex + 1).join('/'); - } - return path.basename(runDir); -} - /** Read the tags for a run. Returns `undefined` if missing or unreadable. */ export function readRunTags(manifestPath: string): RunTagsFile | undefined { const fp = runTagsPath(manifestPath); @@ -97,7 +73,7 @@ export function readRunTags(manifestPath: string): RunTagsFile | undefined { return { tags, updated_at: updatedAt, - oplog_watermark: normalizeRunOplogWatermark(record.oplog_watermark, updatedAt || undefined), + tag_revision: normalizeTagRevision(record.tag_revision, tags, updatedAt || undefined), }; } catch { return undefined; @@ -110,17 +86,11 @@ export function readRunTags(manifestPath: string): RunTagsFile | undefined { */ export function writeRunTags(manifestPath: string, tags: readonly string[]): RunTagsFile { const cleaned = normalizeTags(tags); - const runPath = inferRunRelativePath(manifestPath); - const operation = createRunTagsSetOperation({ - runId: buildRunIdFromRelativePath(runPath), - runPath, - tags: cleaned, - actor: { kind: 'dashboard' }, - }); + const updatedAt = new Date().toISOString(); const entry: RunTagsFile = { tags: cleaned, - updated_at: operation.authored_at, - oplog_watermark: watermarkFromRunOperation(operation), + updated_at: updatedAt, + tag_revision: createTagRevision(cleaned, updatedAt), }; writeFileSync(runTagsPath(manifestPath), `${JSON.stringify(entry, null, 2)}\n`, 'utf8'); return entry; diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 06b43d6d9..50bca666c 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -107,10 +107,11 @@ import { } from './remote.js'; import { type RunFinalState, - type RunOplogWatermark, type RunReadStateFields, + TagRevisionConflictError, + assertExpectedTagRevision, materializeRunState, -} from './run-oplog.js'; +} from './run-state.js'; import { readRunTags, writeRunTags } from './run-tags.js'; import { type StudioConfig, loadStudioConfig, saveStudioConfig } from './studio-config.js'; @@ -983,7 +984,7 @@ interface RunTagFields { readonly pending_tags?: string[]; readonly metadata_dirty?: boolean; readonly final_state: RunFinalState; - readonly oplog_watermark: RunOplogWatermark; + readonly tag_revision: string; } // biome-ignore lint/suspicious/noExplicitAny: Hono Context generic varies by route @@ -1016,7 +1017,7 @@ async function readRunTagFields( ...(state.dirty && { pending_tags: state.pendingTags ?? state.tags }), ...materializeRunState({ tags: state.tags, - watermark: state.oplogWatermark, + tagRevision: state.tagRevision, updatedAt: state.updatedAt, }), }; @@ -1027,7 +1028,7 @@ async function readRunTagFields( const tagsEntry = readRunTags(meta.path); const runState = materializeRunState({ tags: tagsEntry?.tags ?? [], - watermark: tagsEntry?.oplog_watermark, + tagRevision: tagsEntry?.tag_revision, updatedAt: tagsEntry?.updated_at || undefined, }); return { @@ -1053,7 +1054,7 @@ async function readRunTagFields( ...(state.dirty && { pending_tags: state.pendingTags ?? state.tags }), ...materializeRunState({ tags: state.tags, - watermark: state.oplogWatermark, + tagRevision: state.tagRevision, updatedAt: state.updatedAt, }), }; @@ -1065,7 +1066,7 @@ function remoteTagMutationResponse(state: { readonly pendingTags?: string[]; readonly dirty: boolean; readonly updatedAt?: string; - readonly oplogWatermark: RunOplogWatermark; + readonly tagRevision: string; }) { return { tags: state.tags, @@ -1074,7 +1075,7 @@ function remoteTagMutationResponse(state: { ...(state.dirty && { pending_tags: state.pendingTags ?? state.tags }), ...materializeRunState({ tags: state.tags, - watermark: state.oplogWatermark, + tagRevision: state.tagRevision, updatedAt: state.updatedAt, }), updated_at: state.updatedAt ?? new Date().toISOString(), @@ -1084,16 +1085,19 @@ function remoteTagMutationResponse(state: { function localTagMutationResponse(input: { readonly tags: readonly string[]; readonly updatedAt?: string; - readonly watermark?: RunOplogWatermark; + readonly tagRevision?: string; }): RunReadStateFields { return materializeRunState({ tags: input.tags, - watermark: input.watermark, + tagRevision: input.tagRevision, updatedAt: input.updatedAt, }); } function remoteMetadataErrorStatus(error: unknown): 400 | 409 { + if (error instanceof TagRevisionConflictError) { + return 409; + } const message = error instanceof Error ? error.message : String(error); if ( message.includes('not configured') || @@ -1105,6 +1109,36 @@ function remoteMetadataErrorStatus(error: unknown): 400 | 409 { return 400; } +function tagMutationErrorBody(error: unknown) { + const message = error instanceof Error ? error.message : String(error); + if (error instanceof TagRevisionConflictError) { + return { + error: message, + expected_tag_revision: error.expectedRevision, + current_tag_revision: error.currentRevision, + }; + } + return { error: message }; +} + +function expectedTagRevisionFromRecord(record: Record): string | undefined { + const raw = record.expected_tag_revision ?? record.tag_revision ?? record.etag; + if (raw === undefined) return undefined; + if (typeof raw !== 'string') { + throw new Error('expected_tag_revision must be a string'); + } + return raw; +} + +function currentLocalTagRevision(manifestPath: string): string { + const tagsEntry = readRunTags(manifestPath); + return materializeRunState({ + tags: tagsEntry?.tags ?? [], + tagRevision: tagsEntry?.tag_revision, + updatedAt: tagsEntry?.updated_at || undefined, + }).tag_revision; +} + async function ensureRunReadable( searchDir: string, meta: SourcedResultFileMeta, @@ -1963,7 +1997,7 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont pending_tags?: string[]; metadata_dirty?: boolean; final_state: RunFinalState; - oplog_watermark: RunOplogWatermark; + tag_revision: string; source: 'local' | 'remote'; eval_count: number; quality_count: number; @@ -2404,17 +2438,30 @@ async function handleRunTagsPut(c: C, { searchDir, projectId }: DataContext) { if (!Array.isArray(tags)) { return c.json({ error: 'Missing tags array' }, 400); } + let expectedTagRevision: string | undefined; + try { + expectedTagRevision = expectedTagRevisionFromRecord(body as Record); + } catch (err) { + return c.json({ error: (err as Error).message }, 400); + } try { if (meta.on_remote) { - const state = await setRemoteRunTags(searchDir, meta, tags as string[], projectId); + const state = await setRemoteRunTags( + searchDir, + meta, + tags as string[], + projectId, + expectedTagRevision, + ); return c.json(remoteTagMutationResponse(state)); } + assertExpectedTagRevision(expectedTagRevision, currentLocalTagRevision(meta.path)); const entry = writeRunTags(meta.path, tags as string[]); const responseState = localTagMutationResponse({ tags: entry?.tags ?? [], updatedAt: entry?.updated_at, - watermark: entry?.oplog_watermark, + tagRevision: entry?.tag_revision, }); return c.json({ tags: entry?.tags ?? [], @@ -2422,7 +2469,7 @@ async function handleRunTagsPut(c: C, { searchDir, projectId }: DataContext) { updated_at: entry?.updated_at ?? new Date().toISOString(), }); } catch (err) { - return c.json({ error: (err as Error).message }, remoteMetadataErrorStatus(err)); + return c.json(tagMutationErrorBody(err), remoteMetadataErrorStatus(err)); } } @@ -2430,20 +2477,37 @@ async function handleRunTagsDelete(c: C, { searchDir, projectId }: DataContext) const filename = c.req.param('filename') ?? ''; const meta = await findRunById(searchDir, filename, projectId); if (!meta) return c.json({ error: 'Run not found' }, 404); + let expectedTagRevision: string | undefined; + try { + const text = await c.req.text(); + if (text.trim().length > 0) { + const body = JSON.parse(text) as unknown; + if (!body || typeof body !== 'object') { + return c.json({ error: 'Invalid payload' }, 400); + } + expectedTagRevision = expectedTagRevisionFromRecord(body as Record); + } + } catch (err) { + return c.json( + { error: err instanceof SyntaxError ? 'Invalid JSON' : (err as Error).message }, + 400, + ); + } try { if (meta.on_remote) { - const state = await clearRemoteRunTags(searchDir, meta, projectId); + const state = await clearRemoteRunTags(searchDir, meta, projectId, expectedTagRevision); return c.json({ ok: true, ...remoteTagMutationResponse(state), }); } + assertExpectedTagRevision(expectedTagRevision, currentLocalTagRevision(meta.path)); const entry = writeRunTags(meta.path, []); const responseState = localTagMutationResponse({ tags: entry.tags, updatedAt: entry.updated_at, - watermark: entry.oplog_watermark, + tagRevision: entry.tag_revision, }); return c.json({ ok: true, @@ -2452,7 +2516,7 @@ async function handleRunTagsDelete(c: C, { searchDir, projectId }: DataContext) updated_at: entry.updated_at, }); } catch (err) { - return c.json({ error: (err as Error).message }, remoteMetadataErrorStatus(err)); + return c.json(tagMutationErrorBody(err), remoteMetadataErrorStatus(err)); } } @@ -2782,7 +2846,7 @@ export function createApp( pending_tags?: string[]; metadata_dirty?: boolean; final_state: RunFinalState; - oplog_watermark: RunOplogWatermark; + tag_revision: string; source: 'local' | 'remote'; project_id: string; project_name: string; diff --git a/apps/cli/test/commands/results/remote-metadata.test.ts b/apps/cli/test/commands/results/remote-metadata.test.ts index 5fe26b51f..94280e7fe 100644 --- a/apps/cli/test/commands/results/remote-metadata.test.ts +++ b/apps/cli/test/commands/results/remote-metadata.test.ts @@ -11,7 +11,6 @@ import { readRemoteRunTags, writeRemoteRunTags, } from '../../../src/commands/results/remote-metadata.js'; -import { RUN_OPLOG_REF } from '../../../src/commands/results/run-oplog.js'; const RUN_TIMESTAMP = '2026-06-06T10-00-00-000Z'; @@ -83,8 +82,7 @@ describe('remote metadata tags', () => { expect(state.remoteTags).toEqual(['remote-baseline']); expect(state.pendingTags).toEqual(['pending', 'remote-baseline']); expect(state.dirty).toBe(true); - expect(state.oplogWatermark.ref).toBe(RUN_OPLOG_REF); - expect(state.oplogWatermark.operation_id).toBeString(); + expect(state.tagRevision).toStartWith('sha256:'); expect(state.metadataPath).toContain( path.join('metadata', 'runs', 'default', RUN_TIMESTAMP, 'tags.json'), ); @@ -96,7 +94,7 @@ describe('remote metadata tags', () => { expect(reloaded.tags).toEqual(['pending', 'remote-baseline']); expect(reloaded.pendingTags).toEqual(['pending', 'remote-baseline']); expect(reloaded.dirty).toBe(true); - expect(reloaded.oplogWatermark.operation_id).toBe(state.oplogWatermark.operation_id); + expect(reloaded.tagRevision).toBe(state.tagRevision); }); it('uses committed metadata overlays as the clean remote baseline', () => { @@ -112,7 +110,7 @@ describe('remote metadata tags', () => { expect(reloaded.remoteTags).toEqual(['accepted']); expect(reloaded.pendingTags).toBeUndefined(); expect(reloaded.dirty).toBe(false); - expect(reloaded.oplogWatermark.ref).toBe(RUN_OPLOG_REF); + expect(reloaded.tagRevision).toBe(state.tagRevision); }); it('preserves the storage ref when unchanged remote artifact tags clear a local overlay', () => { @@ -158,23 +156,35 @@ describe('remote metadata tags', () => { expect(readFileSync(state.metadataPath, 'utf8')).toContain('"tags": []'); }); - it('records an explicit clear watermark when the remote baseline is already empty', () => { + it('records an explicit clear revision when the remote baseline is already empty', () => { const manifestPath = seedRepo(repoDir, { artifactTags: [] }); const state = writeRemoteRunTags(repoDir, manifestPath, []); const metadata = JSON.parse(readFileSync(state.metadataPath, 'utf8')) as { tags: string[]; - oplog_watermark: { ref: string; operation_id?: string; updated_at?: string }; + tag_revision: string; }; expect(state.tags).toEqual([]); expect(state.remoteTags).toEqual([]); expect(state.pendingTags).toEqual([]); expect(state.dirty).toBe(true); - expect(state.oplogWatermark.ref).toBe(RUN_OPLOG_REF); - expect(state.oplogWatermark.operation_id).toBeString(); + expect(state.tagRevision).toStartWith('sha256:'); expect(metadata.tags).toEqual([]); - expect(metadata.oplog_watermark.operation_id).toBe(state.oplogWatermark.operation_id); + expect(metadata.tag_revision).toBe(state.tagRevision); + }); + + it('rejects stale tag revisions before writing an overlay', () => { + const manifestPath = seedRepo(repoDir); + const before = readRemoteRunTags(repoDir, manifestPath); + + expect(() => + writeRemoteRunTags(repoDir, manifestPath, ['stale'], undefined, 'sha256:stale'), + ).toThrow('Run tags changed. Refresh the run and try again.'); + + const after = readRemoteRunTags(repoDir, manifestPath); + expect(after.tagRevision).toBe(before.tagRevision); + expect(after.tags).toEqual(before.tags); }); it('rejects writes when the configured results path is not a git checkout', () => { diff --git a/apps/cli/test/commands/results/run-oplog.test.ts b/apps/cli/test/commands/results/run-oplog.test.ts deleted file mode 100644 index ff6990bb0..000000000 --- a/apps/cli/test/commands/results/run-oplog.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { describe, expect, it } from 'bun:test'; -import { execFileSync } from 'node:child_process'; - -import { - RUN_OPERATION_SCHEMA_VERSION, - RUN_OPLOG_REF, - buildRunIdFromRelativePath, - createRunTagsSetOperation, - materializeRunState, - watermarkFromRunOperation, -} from '../../../src/commands/results/run-oplog.js'; - -const PRIMARY_RESULTS_REF = 'agentv/results/v1'; -const ARTIFACTS_REF = 'agentv/artifacts/v1'; - -function refsHavePrefixConflict(left: string, right: string): boolean { - return left === right || left.startsWith(`${right}/`) || right.startsWith(`${left}/`); -} - -function isValidGitBranchRef(ref: string): boolean { - try { - execFileSync('git', ['check-ref-format', `refs/heads/${ref}`], { stdio: 'ignore' }); - return true; - } catch { - return false; - } -} - -describe('run operation log contract', () => { - it('defines the stable oplog ref', () => { - expect(RUN_OPLOG_REF).toBe('agentv/oplog/v1'); - }); - - it('keeps results, artifacts, and oplog refs non-prefix-conflicting', () => { - const refs = [PRIMARY_RESULTS_REF, ARTIFACTS_REF, RUN_OPLOG_REF]; - - for (const left of refs) { - expect(isValidGitBranchRef(left)).toBe(true); - } - - for (const [index, left] of refs.entries()) { - for (const right of refs.slice(index + 1)) { - expect(refsHavePrefixConflict(left, right)).toBe(false); - } - } - }); - - it('builds a typed tag replacement operation envelope', () => { - const operation = createRunTagsSetOperation({ - runId: 'smoke::2026-06-21T10-00-00-000Z', - runPath: 'smoke/2026-06-21T10-00-00-000Z', - tags: ['baseline', 'reviewed'], - actor: { kind: 'dashboard', id: 'local' }, - authoredAt: '2026-06-21T10:15:00.000Z', - operationId: 'op-123', - }); - - expect(operation).toEqual({ - schema_version: RUN_OPERATION_SCHEMA_VERSION, - operation_id: 'op-123', - operation_type: 'run.tags.set', - authored_at: '2026-06-21T10:15:00.000Z', - actor: { kind: 'dashboard', id: 'local' }, - subject: { - run_id: 'smoke::2026-06-21T10-00-00-000Z', - run_path: 'smoke/2026-06-21T10-00-00-000Z', - }, - payload: { - tags: ['baseline', 'reviewed'], - }, - }); - }); - - it('materializes final run state from tags and an operation watermark', () => { - const operation = createRunTagsSetOperation({ - runId: '2026-06-21T10-00-00-000Z', - tags: ['accepted'], - authoredAt: '2026-06-21T10:15:00.000Z', - operationId: 'op-456', - }); - - expect( - materializeRunState({ - tags: operation.payload.tags, - watermark: watermarkFromRunOperation(operation), - }), - ).toEqual({ - final_state: { - lifecycle: 'active', - tags: ['accepted'], - }, - oplog_watermark: { - ref: RUN_OPLOG_REF, - operation_id: 'op-456', - updated_at: '2026-06-21T10:15:00.000Z', - }, - }); - }); - - it('derives run IDs from results branch paths', () => { - expect(buildRunIdFromRelativePath('default/2026-06-21T10-00-00-000Z')).toBe( - '2026-06-21T10-00-00-000Z', - ); - expect(buildRunIdFromRelativePath('smoke/2026-06-21T10-00-00-000Z')).toBe( - 'smoke::2026-06-21T10-00-00-000Z', - ); - }); -}); diff --git a/apps/cli/test/commands/results/run-tags.test.ts b/apps/cli/test/commands/results/run-tags.test.ts index 2c555c259..7540ee586 100644 --- a/apps/cli/test/commands/results/run-tags.test.ts +++ b/apps/cli/test/commands/results/run-tags.test.ts @@ -3,7 +3,6 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { tmpdir } from 'node:os'; import path from 'node:path'; -import { RUN_OPLOG_REF } from '../../../src/commands/results/run-oplog.js'; import { deleteRunTags, readRunTags, @@ -27,7 +26,7 @@ describe('run tags sidecar', () => { rmSync(tempDir, { recursive: true, force: true }); }); - it('records empty tags as a clear tombstone with an oplog watermark', () => { + it('records empty tags as an explicit clear state with a tag revision', () => { writeRunTags(manifestPath, ['baseline']); const cleared = writeRunTags(manifestPath, []); @@ -35,10 +34,19 @@ describe('run tags sidecar', () => { expect(existsSync(runTagsPath(manifestPath))).toBe(true); expect(cleared.tags).toEqual([]); - expect(cleared.oplog_watermark?.ref).toBe(RUN_OPLOG_REF); - expect(cleared.oplog_watermark?.operation_id).toBeString(); + expect(cleared.tag_revision).toStartWith('sha256:'); expect(reloaded).toEqual(cleared); expect(readFileSync(runTagsPath(manifestPath), 'utf8')).toContain('"tags": []'); + expect(readFileSync(runTagsPath(manifestPath), 'utf8')).toContain('"tag_revision": "sha256:'); + }); + + it('changes tag_revision after replacement writes', () => { + const first = writeRunTags(manifestPath, ['baseline']); + const second = writeRunTags(manifestPath, ['candidate']); + + expect(first.tag_revision).toStartWith('sha256:'); + expect(second.tag_revision).toStartWith('sha256:'); + expect(second.tag_revision).not.toBe(first.tag_revision); }); it('keeps physical sidecar deletion explicit', () => { diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 7d3287de2..6822ba46d 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -16,7 +16,7 @@ import { fileURLToPath } from 'node:url'; import { AGENTV_RESULTS_ARTIFACTS_REF, addProject, saveProjectRegistry } from '@agentv/core'; -import { RUN_OPLOG_REF } from '../../../src/commands/results/run-oplog.js'; +import { createTagRevision } from '../../../src/commands/results/run-state.js'; import { createApp, loadResults, @@ -1058,7 +1058,7 @@ describe('serve app', () => { source: string; on_remote: boolean; final_state: { lifecycle: string; tags: string[] }; - oplog_watermark: { ref: string }; + tag_revision: string; }>; }; expect(data.runs).toHaveLength(1); @@ -1071,10 +1071,8 @@ describe('serve app', () => { lifecycle: 'active', tags: [], }, - oplog_watermark: { - ref: RUN_OPLOG_REF, - }, }); + expect(data.runs[0].tag_revision).toStartWith('sha256:'); }); it('exposes sanitized Phoenix external_trace metadata through run detail only', async () => { @@ -1138,24 +1136,22 @@ describe('serve app', () => { expect(serialized).not.toContain('token='); }); - it('exposes materialized final state and oplog watermark for local run tags', async () => { + it('exposes materialized final state and tag revision for local run tags', async () => { const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-00-00-000Z'; const runDir = path.join(runsDir, filename); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); + const updatedAt = '2026-06-21T10:15:00.000Z'; + const tagRevision = createTagRevision(['accepted'], updatedAt); writeFileSync( path.join(runDir, 'tags.json'), `${JSON.stringify( { tags: ['accepted'], - updated_at: '2026-06-21T10:15:00.000Z', - oplog_watermark: { - ref: RUN_OPLOG_REF, - operation_id: 'op-local-tags', - updated_at: '2026-06-21T10:15:00.000Z', - }, + updated_at: updatedAt, + tag_revision: tagRevision, }, null, 2, @@ -1170,7 +1166,7 @@ describe('serve app', () => { runs: Array<{ tags: string[]; final_state: { lifecycle: string; tags: string[] }; - oplog_watermark: { ref: string; operation_id?: string; updated_at?: string }; + tag_revision: string; }>; }; expect(listData.runs[0]).toMatchObject({ @@ -1179,11 +1175,7 @@ describe('serve app', () => { lifecycle: 'active', tags: ['accepted'], }, - oplog_watermark: { - ref: RUN_OPLOG_REF, - operation_id: 'op-local-tags', - updated_at: '2026-06-21T10:15:00.000Z', - }, + tag_revision: tagRevision, }); const detailRes = await app.request(`/api/runs/${encodeURIComponent(filename)}`); @@ -1191,7 +1183,7 @@ describe('serve app', () => { const detailData = (await detailRes.json()) as { tags: string[]; final_state: { lifecycle: string; tags: string[] }; - oplog_watermark: { ref: string; operation_id?: string; updated_at?: string }; + tag_revision: string; }; expect(detailData).toMatchObject({ tags: ['accepted'], @@ -1199,32 +1191,26 @@ describe('serve app', () => { lifecycle: 'active', tags: ['accepted'], }, - oplog_watermark: { - ref: RUN_OPLOG_REF, - operation_id: 'op-local-tags', - updated_at: '2026-06-21T10:15:00.000Z', - }, + tag_revision: tagRevision, }); }); - it('preserves a local tag clear watermark after DELETE /tags', async () => { + it('preserves a local tag clear state after DELETE /tags', async () => { const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-30-00-000Z'; const runDir = path.join(runsDir, filename); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); + const updatedAt = '2026-06-21T10:15:00.000Z'; + const tagRevision = createTagRevision(['accepted'], updatedAt); writeFileSync( path.join(runDir, 'tags.json'), `${JSON.stringify( { tags: ['accepted'], - updated_at: '2026-06-21T10:15:00.000Z', - oplog_watermark: { - ref: RUN_OPLOG_REF, - operation_id: 'op-before-clear', - updated_at: '2026-06-21T10:15:00.000Z', - }, + updated_at: updatedAt, + tag_revision: tagRevision, }, null, 2, @@ -1235,13 +1221,15 @@ describe('serve app', () => { const deleteRes = await app.request(`/api/runs/${encodeURIComponent(filename)}/tags`, { method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ expected_tag_revision: tagRevision }), }); expect(deleteRes.status).toBe(200); const deleteData = (await deleteRes.json()) as { ok: boolean; tags: string[]; final_state: { lifecycle: string; tags: string[] }; - oplog_watermark: { ref: string; operation_id?: string; updated_at?: string }; + tag_revision: string; updated_at: string; }; expect(deleteData.ok).toBe(true); @@ -1250,17 +1238,15 @@ describe('serve app', () => { lifecycle: 'active', tags: [], }); - expect(deleteData.oplog_watermark.ref).toBe(RUN_OPLOG_REF); - expect(deleteData.oplog_watermark.operation_id).toBeString(); - expect(deleteData.oplog_watermark.operation_id).not.toBe('op-before-clear'); - expect(deleteData.oplog_watermark.updated_at).toBe(deleteData.updated_at); + expect(deleteData.tag_revision).toStartWith('sha256:'); + expect(deleteData.tag_revision).not.toBe(tagRevision); const tagFile = JSON.parse(readFileSync(path.join(runDir, 'tags.json'), 'utf8')) as { tags: string[]; - oplog_watermark: { ref: string; operation_id?: string; updated_at?: string }; + tag_revision: string; }; expect(tagFile.tags).toEqual([]); - expect(tagFile.oplog_watermark.operation_id).toBe(deleteData.oplog_watermark.operation_id); + expect(tagFile.tag_revision).toBe(deleteData.tag_revision); const reloadedApp = createApp([], tempDir, tempDir, undefined, { studioDir }); const detailRes = await reloadedApp.request(`/api/runs/${encodeURIComponent(filename)}`); @@ -1268,7 +1254,7 @@ describe('serve app', () => { const detailData = (await detailRes.json()) as { tags: string[]; final_state: { lifecycle: string; tags: string[] }; - oplog_watermark: { ref: string; operation_id?: string; updated_at?: string }; + tag_revision: string; }; expect(detailData).toMatchObject({ tags: [], @@ -1276,12 +1262,57 @@ describe('serve app', () => { lifecycle: 'active', tags: [], }, - oplog_watermark: { - ref: RUN_OPLOG_REF, - operation_id: deleteData.oplog_watermark.operation_id, - updated_at: deleteData.oplog_watermark.updated_at, - }, + tag_revision: deleteData.tag_revision, + }); + }); + + it('rejects stale local tag writes with refresh-required details', async () => { + const runsDir = localResultsExperimentDir(tempDir); + mkdirSync(runsDir, { recursive: true }); + const filename = '2026-03-25T10-45-00-000Z'; + const runDir = path.join(runsDir, filename); + mkdirSync(runDir, { recursive: true }); + writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); + const updatedAt = '2026-06-21T10:15:00.000Z'; + const tagRevision = createTagRevision(['accepted'], updatedAt); + writeFileSync( + path.join(runDir, 'tags.json'), + `${JSON.stringify( + { + tags: ['accepted'], + updated_at: updatedAt, + tag_revision: tagRevision, + }, + null, + 2, + )}\n`, + ); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + + const staleRes = await app.request(`/api/runs/${encodeURIComponent(filename)}/tags`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ tags: ['stale'], expected_tag_revision: 'sha256:stale' }), + }); + expect(staleRes.status).toBe(409); + const staleData = (await staleRes.json()) as { + error: string; + expected_tag_revision: string; + current_tag_revision: string; + }; + expect(staleData).toEqual({ + error: 'Run tags changed. Refresh the run and try again.', + expected_tag_revision: 'sha256:stale', + current_tag_revision: tagRevision, }); + + const tagFile = JSON.parse(readFileSync(path.join(runDir, 'tags.json'), 'utf8')) as { + tags: string[]; + tag_revision: string; + }; + expect(tagFile.tags).toEqual(['accepted']); + expect(tagFile.tag_revision).toBe(tagRevision); }); it('computes pass_rate using the configured dashboard threshold', async () => { diff --git a/apps/dashboard/src/components/AnalyticsTab.tsx b/apps/dashboard/src/components/AnalyticsTab.tsx index 0378e4d76..0de71c52e 100644 --- a/apps/dashboard/src/components/AnalyticsTab.tsx +++ b/apps/dashboard/src/components/AnalyticsTab.tsx @@ -747,6 +747,7 @@ function PerRunRow({ void; @@ -814,7 +817,7 @@ function TagsEditor({ }, []); const saveMut = useMutation({ - mutationFn: () => saveRunTagsApi(runId, tags, projectId), + mutationFn: () => saveRunTagsApi(runId, tags, projectId, tagRevision), onSuccess: () => { qc.invalidateQueries({ queryKey: ['compare'] }); qc.invalidateQueries({ queryKey: ['runs'] }); @@ -829,7 +832,7 @@ function TagsEditor({ }); const clearMut = useMutation({ - mutationFn: () => deleteRunTagsApi(runId, projectId), + mutationFn: () => deleteRunTagsApi(runId, projectId, tagRevision), onSuccess: () => { qc.invalidateQueries({ queryKey: ['compare'] }); qc.invalidateQueries({ queryKey: ['runs'] }); diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index 869e1e81d..1d01b3634 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -692,12 +692,13 @@ export async function deleteRunApi(runId: string, projectId?: string): Promise { const url = projectId ? `${projectApiBase(projectId)}/runs/${encodeURIComponent(runId)}/tags` @@ -705,7 +706,10 @@ export async function saveRunTagsApi( const res = await fetch(url, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ tags }), + body: JSON.stringify({ + tags, + ...(expectedTagRevision ? { expected_tag_revision: expectedTagRevision } : {}), + }), }); if (!res.ok) { const err = await res.json().catch(() => ({ error: res.statusText })); @@ -714,12 +718,24 @@ export async function saveRunTagsApi( return res.json() as Promise; } -/** Clear the tags for a run while preserving the clear watermark. */ -export async function deleteRunTagsApi(runId: string, projectId?: string): Promise { +/** Clear the tags for a run, rejecting stale browser edits when a revision is provided. */ +export async function deleteRunTagsApi( + runId: string, + projectId?: string, + expectedTagRevision?: string, +): Promise { const url = projectId ? `${projectApiBase(projectId)}/runs/${encodeURIComponent(runId)}/tags` : `/api/runs/${encodeURIComponent(runId)}/tags`; - const res = await fetch(url, { method: 'DELETE' }); + const res = await fetch(url, { + method: 'DELETE', + ...(expectedTagRevision + ? { + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ expected_tag_revision: expectedTagRevision }), + } + : {}), + }); if (!res.ok) { const err = await res.json().catch(() => ({ error: res.statusText })); throw new Error((err as { error?: string }).error ?? `Failed to delete tags: ${res.status}`); diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index fdad164f5..a7066642e 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -50,10 +50,10 @@ export interface RunMeta { pending_tags?: string[]; /** True when local editable metadata differs from the fetched remote metadata. */ metadata_dirty?: boolean; - /** Materialized final run state consumed by readers instead of folding raw operations. */ + /** Materialized final run state from the result bundle and tag sidecar. */ final_state?: RunFinalState; - /** Operation-log watermark for the materialized final state. */ - oplog_watermark?: RunOplogWatermark; + /** Optimistic-concurrency token for mutable tag state. */ + tag_revision?: string; /** * Live execution status. Only present for Dashboard-launched runs that are * still being tracked in-memory — used to render a spinner in RunList @@ -63,12 +63,6 @@ export interface RunMeta { status?: 'starting' | 'running' | 'finished' | 'failed'; } -export interface RunOplogWatermark { - ref: string; - operation_id?: string; - updated_at?: string; -} - export interface RunFinalState { lifecycle: 'active' | 'hidden' | 'deleted'; tags: string[]; @@ -217,7 +211,7 @@ export interface RunDetailResponse { source: 'local' | 'remote'; source_label?: string; final_state?: RunFinalState; - oplog_watermark?: RunOplogWatermark; + tag_revision?: string; /** Live execution status when this run is still tracked in-memory by Dashboard. */ status?: 'starting' | 'running' | 'finished' | 'failed'; /** Path to the run workspace directory (relative to cwd when inside, otherwise absolute). Local runs only. */ @@ -343,7 +337,7 @@ export interface CompareRunEntry { pending_tags?: string[]; metadata_dirty?: boolean; final_state?: RunFinalState; - oplog_watermark?: RunOplogWatermark; + tag_revision?: string; source: 'local' | 'remote'; eval_count: number; quality_count?: number; @@ -368,7 +362,7 @@ export interface RunTagsResponse { pending_tags?: string[]; metadata_dirty?: boolean; final_state?: RunFinalState; - oplog_watermark?: RunOplogWatermark; + tag_revision: string; updated_at: string; } diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 195771902..117a13a01 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -161,9 +161,9 @@ Select 2+ rows with the checkboxes and click the sticky **Compare N** action to ### Retroactive tags -Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `index.jsonl` in the run workspace, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to record an empty tag state with an operation watermark. +Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `index.jsonl` in the timestamped result folder, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to record an empty tag state. The sidecar includes a `tag_revision`; if a stale browser tab submits tags after the run's tags changed, Dashboard rejects the write and asks you to refresh before retrying. -Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `metadata/runs/.../tags.json` in the configured results repo clone/branch. That overlay path is a remote-results implementation detail, not part of the local `.agentv/results///` layout. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. +Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `metadata/runs/.../tags.json` in the configured results repo clone/branch. That overlay path is a remote-results implementation detail, not part of the local `.agentv/results///` layout. Remote tag overlays use the same `tag_revision` stale-write check as local tags. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. Use tags to annotate ad-hoc variants, experiment cross-cuts, or status flags you didn't plan for up front — `baseline`, `v2-prompt`, `slow`, `after-retry-fix`, `regression`, etc. Unlike `experiment` — which groups runs and is baked into the JSONL at eval-run time — tags are mutable, multi-valued, and never touch the original run data. diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index 10c27873b..a26d448f2 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -229,7 +229,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.remote` with `repo.path: .` and `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo without requiring a machine-local Git remote name. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. `results.repo.path` without `results.repo.remote` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. AgentV manages any local Git remote alias internally. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.remote` with `repo.path: .` and `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo without requiring a machine-local Git remote name. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.repo.path` without `results.repo.remote` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. AgentV manages any local Git remote alias internally. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. - **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.repo.path` clone with `git` directly, then sync again. From 31ea6dfa85f56e4b505990cf007e7ec10f699a05 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 23 Jun 2026 02:39:30 +0200 Subject: [PATCH 3/5] fix(results): inherit experiment for combined runs Combined outputs now inherit a shared source experiment, including default, and mixed-experiment combines require an explicit new experiment name from the CLI or Dashboard. Bead: av-vqe --- apps/cli/src/commands/results/combine-run.ts | 82 ++++++++++++++-- apps/cli/src/commands/results/combine.ts | 42 +++++++- apps/cli/src/commands/results/serve.ts | 18 +++- .../cli/test/commands/results/combine.test.ts | 96 ++++++++++++++++++- apps/cli/test/commands/results/serve.test.ts | 64 +++++++++++++ apps/dashboard/src/components/RunList.tsx | 27 +++++- apps/dashboard/src/lib/api.test.ts | 33 ++++++- apps/dashboard/src/lib/api.ts | 2 + apps/dashboard/src/lib/types.ts | 1 + .../src/content/docs/docs/tools/dashboard.mdx | 2 +- .../src/content/docs/docs/tools/results.mdx | 2 + 11 files changed, 352 insertions(+), 17 deletions(-) diff --git a/apps/cli/src/commands/results/combine-run.ts b/apps/cli/src/commands/results/combine-run.ts index c114ef4a2..b44c4ed39 100644 --- a/apps/cli/src/commands/results/combine-run.ts +++ b/apps/cli/src/commands/results/combine-run.ts @@ -36,6 +36,7 @@ import { import { buildDefaultRunDirFromName, createRunDirName, + normalizeExperimentName, relativeRunPathFromCwd, resolveRunManifestPath, } from '../eval/result-layout.js'; @@ -53,6 +54,7 @@ export interface CombineRunSource { readonly id: string; readonly displayName: string; readonly manifestPath: string; + readonly experiment: string; readonly tags?: readonly string[]; } @@ -91,6 +93,7 @@ export interface CombineRunOptions { readonly cwd: string; readonly sources: readonly CombineRunSource[]; readonly outputDir?: string; + readonly experiment?: string; readonly displayName?: string; readonly duplicatePolicy: CombineDuplicatePolicy; readonly promptChoices?: ReadonlyMap; @@ -103,6 +106,7 @@ export interface CombineRunResult { readonly benchmarkPath: string; readonly timingPath: string; readonly displayName: string; + readonly experiment: string; readonly combinedFromRunIds: readonly string[]; readonly duplicateConflicts: readonly DuplicateConflict[]; readonly testCount: number; @@ -286,13 +290,17 @@ function defaultDisplayName(sources: readonly LoadedSource[]): string { return `Combined run (${sources.map((source) => source.displayName).join(' + ')})`; } -function defaultCombinedRunDir(cwd: string, startedAt: string | undefined): string { +function defaultCombinedRunDirForExperiment( + cwd: string, + experiment: string, + startedAt: string | undefined, +): string { const parsed = startedAt ? new Date(startedAt) : undefined; const timestamp = parsed && !Number.isNaN(parsed.getTime()) ? createRunDirName(parsed) : sanitizePathSegment(startedAt ?? 'unknown-time'); - return buildDefaultRunDirFromName(cwd, 'combined', timestamp); + return buildDefaultRunDirFromName(cwd, experiment, timestamp); } function uniqueRunDir(baseDir: string): string { @@ -310,7 +318,47 @@ function toRunId(cwd: string, runDir: string): string { return path.basename(runDir); } const parts = relative.split(path.posix.sep); - return parts.length > 1 ? `${parts[0]}::${parts.slice(1).join(path.posix.sep)}` : relative; + if (parts.length <= 1) { + return relative; + } + const experiment = parts[0]; + const timestamp = parts.slice(1).join(path.posix.sep); + return experiment === 'default' ? timestamp : `${experiment}::${timestamp}`; +} + +function experimentFromRelativeRunPath(relativeRunPath: string): string { + const experiment = relativeRunPath.split(path.posix.sep).filter(Boolean)[0]; + return normalizeExperimentName(experiment); +} + +function resolveCombinedExperiment( + sources: readonly LoadedSource[], + requestedExperiment: string | undefined, +): string { + const experiments = [...new Set(sources.map((source) => source.experiment))]; + const normalizedRequested = requestedExperiment?.trim() + ? normalizeExperimentName(requestedExperiment) + : undefined; + if (experiments.length === 1) { + const sourceExperiment = experiments[0]; + if (normalizedRequested === undefined) { + return sourceExperiment; + } + if (normalizedRequested !== sourceExperiment) { + throw new Error(`Combined runs from the same experiment must inherit "${sourceExperiment}".`); + } + return sourceExperiment; + } + + if (normalizedRequested !== undefined) { + return normalizedRequested; + } + + throw new Error( + `Combining runs from multiple experiments requires an experiment name. Source experiments: ${experiments + .sort() + .join(', ')}`, + ); } const MANIFEST_PATH_FIELDS = [ @@ -430,9 +478,14 @@ function rewriteArtifactPointers( }; } -function rewriteAndCopyRecord(row: SelectedRow, outputDir: string): ResultManifestRecord { +function rewriteAndCopyRecord( + row: SelectedRow, + outputDir: string, + experiment: string, +): ResultManifestRecord { const sourceBaseDir = path.dirname(row.source.manifestPath); const rewritten: Record = { ...row.record }; + rewritten.experiment = experiment; for (const field of MANIFEST_PATH_FIELDS) { rewritten[field] = copyReferencedArtifact( sourceBaseDir, @@ -489,7 +542,8 @@ export function buildCombineRunSources( return sourcePaths.map((sourcePath, index) => { const manifestPath = resolveResultSourcePath(sourcePath, cwd); const runDir = path.dirname(resolveRunManifestPath(manifestPath)); - if (!relativeRunPathFromCwd(cwd, runDir)) { + const relativeRunPath = relativeRunPathFromCwd(cwd, runDir); + if (!relativeRunPath) { throw new Error( `Run workspace is outside the canonical results layout: ${runDir}. Expected .agentv/results//`, ); @@ -498,6 +552,7 @@ export function buildCombineRunSources( id: options?.ids?.[index] ?? toRunId(cwd, runDir), displayName: options?.displayNames?.[index] ?? path.basename(runDir), manifestPath, + experiment: experimentFromRelativeRunPath(relativeRunPath), tags: options?.tags?.[index], }; }); @@ -510,17 +565,25 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult const loadedSources = loadSources(options.sources); const startedAt = earliestTimestamp(loadedSources.map((source) => source.startedAt)); + const experiment = resolveCombinedExperiment(loadedSources, options.experiment); const displayName = options.displayName?.trim() || defaultDisplayName(loadedSources); const runDir = uniqueRunDir( options.outputDir ? path.resolve(options.cwd, options.outputDir) - : defaultCombinedRunDir(options.cwd, startedAt), + : defaultCombinedRunDirForExperiment(options.cwd, experiment, startedAt), ); - if (!relativeRunPathFromCwd(options.cwd, runDir)) { + const relativeOutputRunPath = relativeRunPathFromCwd(options.cwd, runDir); + if (!relativeOutputRunPath) { throw new Error( `Output run workspace must use .agentv/results//: ${runDir}`, ); } + const outputExperiment = experimentFromRelativeRunPath(relativeOutputRunPath); + if (outputExperiment !== experiment) { + throw new Error( + `Output run workspace experiment "${outputExperiment}" must match combined experiment "${experiment}".`, + ); + } const { rows, conflicts } = selectRows( loadedSources, options.duplicatePolicy, @@ -535,7 +598,7 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult }); mkdirSync(runDir, { recursive: true }); - const records = rows.map((row) => rewriteAndCopyRecord(row, runDir)); + const records = rows.map((row) => rewriteAndCopyRecord(row, runDir, experiment)); const manifestPath = path.join(runDir, 'index.jsonl'); writeJsonl(manifestPath, records); @@ -549,6 +612,7 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult display_name: string; combined_from_run_ids: readonly string[]; combined_from_display_names: readonly string[]; + experiment: string; duplicate_policy: Exclude | 'prompt'; }; } = { @@ -557,6 +621,7 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult ...benchmark.metadata, timestamp: startedAt ?? benchmark.metadata.timestamp, display_name: displayName, + experiment, combined_from_run_ids: loadedSources.map((source) => source.id), combined_from_display_names: loadedSources.map((source) => source.displayName), duplicate_policy: options.duplicatePolicy, @@ -573,6 +638,7 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult benchmarkPath, timingPath, displayName, + experiment, combinedFromRunIds: loadedSources.map((source) => source.id), duplicateConflicts: conflicts, testCount: rows.length, diff --git a/apps/cli/src/commands/results/combine.ts b/apps/cli/src/commands/results/combine.ts index f10575dfa..2f96c4e38 100644 --- a/apps/cli/src/commands/results/combine.ts +++ b/apps/cli/src/commands/results/combine.ts @@ -12,6 +12,7 @@ import { command, oneOf, option, optional, restPositionals, string } from 'cmd-t import { type CombineDuplicatePolicy, + type CombineRunSource, buildCombineRunSources, combineRunSources, inspectRunSourceDuplicates, @@ -72,6 +73,22 @@ function parseDuplicatePolicy(policy: string | undefined): CombineDuplicatePolic return undefined; } +function uniqueSourceExperiments(sources: readonly CombineRunSource[]): string[] { + return [...new Set(sources.map((source) => source.experiment))].sort(); +} + +async function promptExperimentName(experiments: readonly string[]): Promise { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + try { + const answer = await rl.question( + `Source runs span experiments (${experiments.join(', ')}). New experiment name: `, + ); + return answer.trim() || undefined; + } finally { + rl.close(); + } +} + export const resultsCombineCommand = command({ name: 'combine', description: 'Combine two or more partial local run workspaces into a new run workspace', @@ -86,7 +103,13 @@ export const resultsCombineCommand = command({ long: 'output', short: 'o', description: - 'Output run workspace directory (defaults to .agentv/results/combined/)', + 'Output run workspace directory (defaults to .agentv/results//)', + }), + experiment: option({ + type: optional(string), + long: 'experiment', + description: + 'Experiment namespace for the combined run. Required when sources span multiple experiments.', }), displayName: option({ type: optional(string), @@ -108,6 +131,22 @@ export const resultsCombineCommand = command({ const cwd = process.cwd(); const sources = buildCombineRunSources(args.sources, cwd); + const sourceExperiments = uniqueSourceExperiments(sources); + let experiment = args.experiment; + if ( + !experiment && + sourceExperiments.length > 1 && + process.stdin.isTTY && + process.stdout.isTTY + ) { + experiment = await promptExperimentName(sourceExperiments); + } + if (!experiment && sourceExperiments.length > 1) { + console.error( + `Error: combining runs from multiple experiments requires --experiment . Source experiments: ${sourceExperiments.join(', ')}`, + ); + process.exit(1); + } const duplicatePolicy = defaultDuplicatePolicy(parseDuplicatePolicy(args.duplicatePolicy)); let promptChoices: Map | undefined; @@ -129,6 +168,7 @@ export const resultsCombineCommand = command({ cwd, sources, outputDir: args.output, + experiment, displayName: args.displayName, duplicatePolicy, promptChoices, diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 50bca666c..1029475d2 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -2590,6 +2590,10 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { if (displayNameValue !== undefined && typeof displayNameValue !== 'string') { return c.json({ error: 'display_name must be a string' }, 400); } + const experimentValue = payload.experiment; + if (experimentValue !== undefined && typeof experimentValue !== 'string') { + return c.json({ error: 'experiment must be a string' }, 400); + } const duplicatePolicyValue = payload.duplicate_policy; const duplicatePolicy = duplicatePolicyValue === undefined ? 'error' : String(duplicatePolicyValue); @@ -2598,6 +2602,7 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { } const displayName = displayNameValue?.trim(); + const experiment = experimentValue?.trim() || undefined; const metas: SourcedResultFileMeta[] = []; for (const runId of runIds) { @@ -2621,6 +2626,7 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { const combined = combineRunSources({ cwd: searchDir, sources, + experiment, displayName, duplicatePolicy: duplicatePolicy as Exclude, }); @@ -2631,6 +2637,7 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { ok: true, run_id: combined.runId, display_name: combined.displayName, + experiment: combined.experiment, combined_from_run_ids: combined.combinedFromRunIds, duplicate_conflicts: combined.duplicateConflicts, ...(tagEntry && { tags: tagEntry.tags }), @@ -2641,7 +2648,16 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { if (err instanceof CombineDuplicateError) { return c.json({ error: err.message, duplicates: err.conflicts }, 409); } - return c.json({ error: (err as Error).message }, 500); + const message = err instanceof Error ? err.message : String(err); + if ( + message.includes('requires an experiment name') || + message.includes('must inherit') || + message.includes('must match combined experiment') || + message.includes('Invalid experiment name') + ) { + return c.json({ error: message }, 400); + } + return c.json({ error: message }, 500); } } diff --git a/apps/cli/test/commands/results/combine.test.ts b/apps/cli/test/commands/results/combine.test.ts index be55be12f..618a628c9 100644 --- a/apps/cli/test/commands/results/combine.test.ts +++ b/apps/cli/test/commands/results/combine.test.ts @@ -44,8 +44,8 @@ describe('results combine', () => { rmSync(tempDir, { recursive: true, force: true }); }); - function seedRun(name: string, records: object[]): string { - const runDir = path.join(tempDir, '.agentv', 'results', 'default', name); + function seedRun(name: string, records: object[], experiment = 'default'): string { + const runDir = path.join(tempDir, '.agentv', 'results', experiment, name); mkdirSync(path.join(runDir, 'demo', 'test-a'), { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(...records), 'utf8'); writeFileSync(path.join(runDir, 'demo', 'test-a', 'grading.json'), '{"assertions":[]}\n'); @@ -79,11 +79,13 @@ describe('results combine', () => { duplicatePolicy: 'error', }); - expect(combined.runId).toBe('combined::2026-06-01T10-00-00-000Z'); + expect(combined.runId).toBe('2026-06-01T10-00-00-000Z'); + expect(combined.experiment).toBe('default'); expect(combined.testCount).toBe(2); const index = readFileSync(combined.manifestPath, 'utf8'); expect(index).toContain('"test_id":"test-a"'); expect(index).toContain('"test_id":"test-b"'); + expect(index).toContain('"experiment":"default"'); expect(index).toContain('"grading_path":"sources/source-1/demo/test-a/grading.json"'); expect( existsSync(path.join(combined.runDir, 'sources/source-1/demo/test-a/grading.json')), @@ -94,6 +96,94 @@ describe('results combine', () => { expect(benchmark.metadata.timestamp).toBe('2026-06-01T10:00:00.000Z'); }); + it('inherits a shared non-default source experiment', () => { + const first = seedRun('run-a', [result()], 'smoke'); + const second = seedRun( + 'run-b', + [ + result({ + timestamp: '2026-06-01T11:00:00.000Z', + test_id: 'test-b', + }), + ], + 'smoke', + ); + + const combined = combineRunSources({ + cwd: tempDir, + sources: buildCombineRunSources([first, second], tempDir), + duplicatePolicy: 'error', + }); + + expect(combined.experiment).toBe('smoke'); + expect(combined.runId).toBe('smoke::2026-06-01T10-00-00-000Z'); + expect(combined.runDir).toContain(path.join('.agentv', 'results', 'smoke')); + expect(readIndex(combined.manifestPath).map((record) => record.experiment)).toEqual([ + 'smoke', + 'smoke', + ]); + }); + + it('rejects overriding a shared source experiment', () => { + const first = seedRun('run-a', [result()], 'smoke'); + const second = seedRun( + 'run-b', + [ + result({ + timestamp: '2026-06-01T11:00:00.000Z', + test_id: 'test-b', + }), + ], + 'smoke', + ); + + expect(() => + combineRunSources({ + cwd: tempDir, + sources: buildCombineRunSources([first, second], tempDir), + experiment: 'other', + duplicatePolicy: 'error', + }), + ).toThrow('Combined runs from the same experiment must inherit "smoke"'); + }); + + it('requires an explicit experiment when source experiments differ', () => { + const first = seedRun('run-a', [result()], 'smoke'); + const second = seedRun( + 'run-b', + [ + result({ + timestamp: '2026-06-01T11:00:00.000Z', + test_id: 'test-b', + }), + ], + 'regression', + ); + const sources = buildCombineRunSources([first, second], tempDir); + + expect(() => + combineRunSources({ + cwd: tempDir, + sources, + duplicatePolicy: 'error', + }), + ).toThrow('Combining runs from multiple experiments requires an experiment name'); + + const combined = combineRunSources({ + cwd: tempDir, + sources, + experiment: 'smoke-regression', + duplicatePolicy: 'error', + }); + + expect(combined.experiment).toBe('smoke-regression'); + expect(combined.runId).toBe('smoke-regression::2026-06-01T10-00-00-000Z'); + expect(readIndex(combined.manifestPath).map((record) => record.experiment)).toEqual([ + 'smoke-regression', + 'smoke-regression', + ]); + }); + it('copies and rewrites artifact pointers when combining runs', () => { const first = seedRun('run-a', [ result({ diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 6822ba46d..f49dd814a 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -2549,9 +2549,12 @@ describe('serve app', () => { const data = (await res.json()) as { run_id: string; display_name: string; + experiment: string; combined_from_run_ids: string[]; }; expect(data.display_name).toBe('Combined Smoke'); + expect(data.experiment).toBe('default'); + expect(data.run_id).toBe('2026-03-18T10-00-01-000Z'); expect(data.combined_from_run_ids).toEqual([first.runId, second.runId]); const detailRes = await app.request(`/api/runs/${encodeURIComponent(data.run_id)}`); @@ -2571,9 +2574,60 @@ describe('serve app', () => { }; expect(benchmark.metadata.combined_from_run_ids).toEqual([first.runId, second.runId]); expect(benchmark.metadata.display_name).toBe('Combined Smoke'); + expect(benchmark.metadata).toMatchObject({ experiment: 'default' }); expect(benchmark.metadata.timestamp).toBe('2026-03-18T10:00:01.000Z'); }); + it('requires an explicit experiment when combining runs across experiments', async () => { + const first = seedRun('2026-06-01T10-00-00-000Z', [RESULT_A], { + experiment: 'smoke', + }); + const second = seedRun('2026-06-01T11-00-00-000Z', [RESULT_B], { + experiment: 'regression', + }); + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + + const rejected = await app.request('/api/runs/combine', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ run_ids: [first.runId, second.runId] }), + }); + expect(rejected.status).toBe(400); + const rejectedData = (await rejected.json()) as { error: string }; + expect(rejectedData.error).toContain( + 'Combining runs from multiple experiments requires an experiment name', + ); + + const accepted = await app.request('/api/runs/combine', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + run_ids: [first.runId, second.runId], + experiment: 'smoke-regression', + }), + }); + expect(accepted.status).toBe(201); + const acceptedData = (await accepted.json()) as { run_id: string; experiment: string }; + expect(acceptedData.experiment).toBe('smoke-regression'); + expect(acceptedData.run_id).toStartWith('smoke-regression::'); + expect(existsSync(localRunDirFromRunId(tempDir, acceptedData.run_id))).toBe(true); + + const detailRes = await app.request(`/api/runs/${encodeURIComponent(acceptedData.run_id)}`); + expect(detailRes.status).toBe(200); + await detailRes.json(); + const records = readFileSync( + path.join(localRunDirFromRunId(tempDir, acceptedData.run_id), 'index.jsonl'), + 'utf8', + ) + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { experiment?: string }); + expect(records.map((result) => result.experiment)).toEqual([ + 'smoke-regression', + 'smoke-regression', + ]); + }); + it('generates a display name when combine omits display_name', async () => { const first = seedRun('2026-06-01T10-00-00-000Z', [RESULT_A]); const second = seedRun('2026-06-01T11-00-00-000Z', [RESULT_B]); @@ -2631,6 +2685,16 @@ describe('serve app', () => { }), }); expect(invalidPolicy.status).toBe(400); + + const invalidExperiment = await app.request('/api/runs/combine', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + run_ids: [first.runId, 'missing-run'], + experiment: 123, + }), + }); + expect(invalidExperiment.status).toBe(400); }); it('rejects duplicate rows by default and combines with explicit latest policy', async () => { diff --git a/apps/dashboard/src/components/RunList.tsx b/apps/dashboard/src/components/RunList.tsx index 35764cb4d..c8fe545ca 100644 --- a/apps/dashboard/src/components/RunList.tsx +++ b/apps/dashboard/src/components/RunList.tsx @@ -154,6 +154,7 @@ export function RunList({ [enableCombine, runs], ); const selectedSet = new Set(selectedRunIds); + const runsById = useMemo(() => new Map(runs.map((run) => [run.filename, run])), [runs]); useEffect(() => { if (!isFetchingNextPage) { @@ -217,12 +218,34 @@ export function RunList({ async function handleCombine() { if (selectedRunIds.length < 2 || combineInFlight) return; const sourceRunIds = [...selectedRunIds]; + const selectedExperiments = [ + ...new Set( + sourceRunIds + .map((runId) => runsById.get(runId)?.experiment?.trim() || 'default') + .filter(Boolean), + ), + ].sort(); + let experiment: string | undefined; + if (selectedExperiments.length > 1) { + const answer = window.prompt( + `Selected runs span experiments (${selectedExperiments.join(', ')}). Enter a new experiment name for the combined run:`, + ); + if (answer === null) return; + experiment = answer.trim(); + if (!experiment) { + setActionFeedback({ + kind: 'error', + message: 'Experiment name is required to combine runs from multiple experiments.', + }); + return; + } + } setActionFeedback(null); setCombineInFlight(true); try { let result: CombineRunsResponse; try { - result = await combineRunsApi(sourceRunIds, 'error', projectId); + result = await combineRunsApi(sourceRunIds, 'error', projectId, undefined, experiment); } catch (err) { if (!(err instanceof CombineRunsApiError) || err.status !== 409) { throw err; @@ -232,7 +255,7 @@ export function RunList({ `${count} duplicate (test_id, target) pair${count === 1 ? '' : 's'} found. Replace duplicates with the latest timestamp?`, ); if (!confirmed) return; - result = await combineRunsApi(sourceRunIds, 'latest', projectId); + result = await combineRunsApi(sourceRunIds, 'latest', projectId, undefined, experiment); } setSelectedRunIds([]); setActionFeedback({ diff --git a/apps/dashboard/src/lib/api.test.ts b/apps/dashboard/src/lib/api.test.ts index 03ccacd08..1458043ae 100644 --- a/apps/dashboard/src/lib/api.test.ts +++ b/apps/dashboard/src/lib/api.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it } from 'bun:test'; -import { addProjectApi, browseFilesystemApi, removeProjectApi } from './api'; +import { addProjectApi, browseFilesystemApi, combineRunsApi, removeProjectApi } from './api'; const originalFetch = globalThis.fetch; @@ -88,4 +88,35 @@ describe('dashboard API boundary mapping', () => { await expect(removeProjectApi('missing')).rejects.toThrow('Project not found'); }); + + it('sends explicit experiment names for cross-experiment combine requests', async () => { + const calls = stubJsonResponse({ + ok: true, + run_id: 'smoke-regression::2026-06-01T10-00-00-000Z', + display_name: 'Combined run', + experiment: 'smoke-regression', + combined_from_run_ids: ['smoke::run-a', 'regression::run-b'], + }); + + await combineRunsApi( + ['smoke::run-a', 'regression::run-b'], + 'latest', + 'demo project', + undefined, + 'smoke-regression', + ); + + expect(calls[0]).toEqual({ + url: '/api/projects/demo%20project/runs/combine', + init: { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + run_ids: ['smoke::run-a', 'regression::run-b'], + duplicate_policy: 'latest', + experiment: 'smoke-regression', + }), + }, + }); + }); }); diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index 1d01b3634..cf87a579d 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -651,6 +651,7 @@ export async function combineRunsApi( duplicatePolicy: 'error' | 'latest', projectId?: string, displayName?: string, + experiment?: string, ): Promise { const url = projectId ? `${projectApiBase(projectId)}/runs/combine` : '/api/runs/combine'; const res = await fetch(url, { @@ -660,6 +661,7 @@ export async function combineRunsApi( run_ids: runIds, duplicate_policy: duplicatePolicy, ...(displayName?.trim() ? { display_name: displayName.trim() } : {}), + ...(experiment?.trim() ? { experiment: experiment.trim() } : {}), }), }); if (!res.ok) { diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index a7066642e..c96b37b03 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -381,6 +381,7 @@ export interface CombineRunsResponse { ok: true; run_id: string; display_name: string; + experiment: string; combined_from_run_ids: string[]; duplicate_conflicts?: CombineDuplicateConflict[]; tags?: string[]; diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 117a13a01..f92be83e4 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -109,7 +109,7 @@ In the per-test results table, click a test ID to open its checks, transcript, s ## Run management -In Recent Runs, select local completed runs to combine partial runs or delete stale run workspaces. Combine creates a new local run workspace and leaves the source runs in place. Delete removes the selected local run workspace directory, including sidecars such as `tags.json`; remote runs are read-only. +In Recent Runs, select local completed runs to combine partial runs or delete stale run workspaces. Combine creates a new local run workspace and leaves the source runs in place. If all selected runs are from one experiment, the combined run inherits that experiment, including `default`; if selected runs span experiments, Dashboard asks for a new experiment name before creating the combined run. Delete removes the selected local run workspace directory, including sidecars such as `tags.json`; remote runs are read-only. The same deletion primitive is available from the CLI: diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index a26d448f2..139468319 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -26,6 +26,8 @@ Remote result repository exchange is intentionally not part of `agentv results`. | `results show` | Display case-level rows from a run workspace | | `results validate` | Validate that a workspace or manifest resolves correctly | +`results combine` writes the new run under the source experiment when every selected source run belongs to the same experiment, including `default`. If the source runs span multiple experiments, pass `--experiment ` for the new combined run; AgentV does not silently write mixed-experiment combines under a `combined` namespace. + ## `results report` The `results report` command turns an existing run workspace or `index.jsonl` manifest into a self-contained HTML report for sharing, inspection, and human review. From 000ba7ef6bb38d5a6e71618eb2865c35dfc635bf Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 23 Jun 2026 03:45:28 +0200 Subject: [PATCH 4/5] fix(results): reserve non-run result namespaces --- apps/cli/src/commands/eval/result-layout.ts | 10 +++++++--- apps/cli/src/commands/inspect/filter.ts | 5 +++-- apps/cli/src/commands/inspect/search.ts | 3 ++- apps/cli/src/commands/inspect/utils.ts | 12 +++++++----- apps/cli/src/commands/results/export.ts | 11 +++++++---- .../test/commands/eval/result-layout.test.ts | 18 ++++++++++-------- apps/cli/test/commands/results/export.test.ts | 1 - apps/cli/test/commands/trace/trace.test.ts | 11 +++++++++++ scripts/deploy-example-results-repos.ts | 2 +- 9 files changed, 48 insertions(+), 25 deletions(-) diff --git a/apps/cli/src/commands/eval/result-layout.ts b/apps/cli/src/commands/eval/result-layout.ts index b518ecdee..dc02ec6d6 100644 --- a/apps/cli/src/commands/eval/result-layout.ts +++ b/apps/cli/src/commands/eval/result-layout.ts @@ -4,7 +4,11 @@ import path from 'node:path'; export const RESULT_INDEX_FILENAME = 'index.jsonl'; export const RESULTS_DIRNAME = 'results'; export const DEFAULT_EXPERIMENT_NAME = 'default'; -const RESERVED_EXPERIMENT_NAMES = new Set(['runs']); +export const RESERVED_RESULTS_NAMESPACES = new Set(['export', 'metadata', 'runs']); + +export function isReservedResultsNamespace(value: string | undefined): boolean { + return value !== undefined && RESERVED_RESULTS_NAMESPACES.has(value); +} export function normalizeExperimentName(experiment?: string): string { const trimmed = experiment?.trim(); @@ -16,7 +20,7 @@ export function normalizeExperimentName(experiment?: string): string { `Invalid experiment name "${trimmed}". Use only letters, numbers, ".", "_" and "-".`, ); } - if (RESERVED_EXPERIMENT_NAMES.has(trimmed)) { + if (isReservedResultsNamespace(trimmed)) { throw new Error(`Invalid experiment name "${trimmed}". This results namespace is reserved.`); } return trimmed; @@ -114,7 +118,7 @@ export function relativeRunPathFromCwd(cwd: string, runDir: string): string | un } const parts = relative.split(path.sep).filter(Boolean); - if (parts.length < 2 || RESERVED_EXPERIMENT_NAMES.has(parts[0] ?? '')) { + if (parts.length < 2 || isReservedResultsNamespace(parts[0])) { return undefined; } diff --git a/apps/cli/src/commands/inspect/filter.ts b/apps/cli/src/commands/inspect/filter.ts index 3b31139a5..338650fac 100644 --- a/apps/cli/src/commands/inspect/filter.ts +++ b/apps/cli/src/commands/inspect/filter.ts @@ -13,6 +13,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; import path from 'node:path'; +import { isReservedResultsNamespace } from '../eval/result-layout.js'; import { command, number, oneOf, option, optional, positional, string } from 'cmd-ts'; import { normalizeResultRow } from '../results/result-row-schema.js'; import { c, formatScore, padLeft, padRight } from './utils.js'; @@ -59,7 +60,7 @@ function collectCurrentResultIndexFiles(cwd: string): string[] { try { const entries = readdirSync(resultsDir, { withFileTypes: true }); for (const entry of entries) { - if (entry.name === 'runs') continue; + if (isReservedResultsNamespace(entry.name)) continue; const fullPath = path.join(resultsDir, entry.name); if (entry.isDirectory()) { files.push(...collectIndexFiles(fullPath)); @@ -118,7 +119,7 @@ function inferExperimentFromPath(filePath: string): string | undefined { if (resultsIdx === -1) return undefined; const runPath = parts.slice(resultsIdx + 1, -1); - if (runPath[0] === 'runs') return undefined; + if (isReservedResultsNamespace(runPath[0])) return undefined; if (runPath.length < 2) return undefined; const candidate = runPath[0]; diff --git a/apps/cli/src/commands/inspect/search.ts b/apps/cli/src/commands/inspect/search.ts index 82407c638..a2da8b251 100644 --- a/apps/cli/src/commands/inspect/search.ts +++ b/apps/cli/src/commands/inspect/search.ts @@ -15,6 +15,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; import path from 'node:path'; +import { isReservedResultsNamespace } from '../eval/result-layout.js'; import { command, oneOf, option, optional, positional, string } from 'cmd-ts'; import { toSnakeCaseDeep } from '../../utils/case-conversion.js'; import { c, padRight } from './utils.js'; @@ -62,7 +63,7 @@ function collectCurrentResultJsonlFiles(cwd: string): string[] { try { const entries = readdirSync(resultsDir, { withFileTypes: true }); for (const entry of entries) { - if (entry.name === 'runs') continue; + if (isReservedResultsNamespace(entry.name)) continue; const fullPath = path.join(resultsDir, entry.name); if (entry.isDirectory()) { files.push(...collectJsonlFiles(fullPath)); diff --git a/apps/cli/src/commands/inspect/utils.ts b/apps/cli/src/commands/inspect/utils.ts index 6bcb54544..0df6e0c53 100644 --- a/apps/cli/src/commands/inspect/utils.ts +++ b/apps/cli/src/commands/inspect/utils.ts @@ -5,6 +5,7 @@ import { DEFAULT_THRESHOLD, toCamelCaseDeep, toSnakeCaseDeep } from '@agentv/cor import { RESULT_INDEX_FILENAME, buildResultsRootDir, + isReservedResultsNamespace, resolveExistingRunPrimaryPath, resolveWorkspaceOrFilePath, } from '../eval/result-layout.js'; @@ -622,7 +623,7 @@ function listResultFilesFromRoot( const entries = readdirSync(runsDir, { withFileTypes: true }); for (const entry of entries) { if (entry.isDirectory()) { - if (options?.skipTopLevelDirs?.has(entry.name)) { + if (options?.skipTopLevelDirs?.has(entry.name) || isReservedResultsNamespace(entry.name)) { continue; } collectRunManifestPaths(runsDir, path.join(runsDir, entry.name), files); @@ -674,12 +675,13 @@ function listResultFilesFromRoot( /** * Enumerate canonical run manifests in `.agentv/results/`. * - * Reserved local namespaces such as `.agentv/results/runs/` and - * `.agentv/results/metadata/` are intentionally skipped by default discovery. + * Reserved local namespaces such as `.agentv/results/runs/`, + * `.agentv/results/metadata/`, and `.agentv/results/export/` are intentionally + * skipped by default discovery. */ export function listResultFiles(cwd: string, limit?: number): ResultFileMeta[] { const metas = listResultFilesFromRoot(buildResultsRootDir(cwd), { - skipTopLevelDirs: new Set(['metadata', 'runs']), + skipTopLevelDirs: new Set(['export', 'metadata', 'runs']), }).sort((a, b) => { const byTimestamp = b.timestamp.localeCompare(a.timestamp); return byTimestamp !== 0 ? byTimestamp : b.displayName.localeCompare(a.displayName); @@ -690,7 +692,7 @@ export function listResultFiles(cwd: string, limit?: number): ResultFileMeta[] { export function listResultFilesFromRunsDir(runsDir: string, limit?: number): ResultFileMeta[] { return listResultFilesFromRoot(runsDir, { limit, - skipTopLevelDirs: new Set(['metadata', 'runs']), + skipTopLevelDirs: new Set(['export', 'metadata', 'runs']), }); } diff --git a/apps/cli/src/commands/results/export.ts b/apps/cli/src/commands/results/export.ts index 769489c94..0df4c7678 100644 --- a/apps/cli/src/commands/results/export.ts +++ b/apps/cli/src/commands/results/export.ts @@ -29,7 +29,7 @@ import { command, flag, oneOf, option, optional, positional, string } from 'cmd- import type { EvaluationResult, ExportDuplicatePolicy, IndexArtifactEntry } from '@agentv/core'; import { parseJsonlResults, writeArtifactsFromResults } from '../eval/artifact-writer.js'; -import { RESULT_INDEX_FILENAME } from '../eval/result-layout.js'; +import { RESULT_INDEX_FILENAME, isReservedResultsNamespace } from '../eval/result-layout.js'; import { loadManifestResults } from './manifest.js'; import { type ProjectionBundle, @@ -72,9 +72,12 @@ export function deriveOutputDir(cwd: string, sourceFile: string): string { const runDir = path.dirname(sourceFile); const segments = path.normalize(runDir).split(path.sep).filter(Boolean); - const runsIndex = segments.lastIndexOf('runs'); - if (runsIndex >= 0 && runsIndex < segments.length - 1) { - return path.join(cwd, '.agentv', 'results', 'export', ...segments.slice(runsIndex + 1)); + const resultsIndex = segments.lastIndexOf('results'); + if (resultsIndex >= 0 && resultsIndex < segments.length - 2) { + const runSegments = segments.slice(resultsIndex + 1); + if (!isReservedResultsNamespace(runSegments[0])) { + return path.join(cwd, '.agentv', 'results', 'export', ...runSegments); + } } const parentDir = path.basename(runDir); diff --git a/apps/cli/test/commands/eval/result-layout.test.ts b/apps/cli/test/commands/eval/result-layout.test.ts index d45e3c740..97424c4c1 100644 --- a/apps/cli/test/commands/eval/result-layout.test.ts +++ b/apps/cli/test/commands/eval/result-layout.test.ts @@ -24,14 +24,16 @@ describe('result layout', () => { ); }); - it('reserves the deprecated runs namespace', () => { - expect(() => normalizeExperimentName('runs')).toThrow('reserved'); - expect( - relativeRunPathFromCwd( - '/repo', - path.join('/repo', '.agentv', 'results', 'runs', 'default', '2026-run'), - ), - ).toBeUndefined(); + it('reserves non-run namespaces at the results root', () => { + for (const namespace of ['export', 'metadata', 'runs']) { + expect(() => normalizeExperimentName(namespace)).toThrow('reserved'); + expect( + relativeRunPathFromCwd( + '/repo', + path.join('/repo', '.agentv', 'results', namespace, 'default', '2026-run'), + ), + ).toBeUndefined(); + } expect( relativeRunPathFromCwd( '/repo', diff --git a/apps/cli/test/commands/results/export.test.ts b/apps/cli/test/commands/results/export.test.ts index 97e5085ae..2056c16d7 100644 --- a/apps/cli/test/commands/results/export.test.ts +++ b/apps/cli/test/commands/results/export.test.ts @@ -221,7 +221,6 @@ describe('results export', () => { tempDir, '.agentv', 'results', - 'runs', 'with-skills', '2026-03-18T10-00-00-000Z', 'index.jsonl', diff --git a/apps/cli/test/commands/trace/trace.test.ts b/apps/cli/test/commands/trace/trace.test.ts index 21f36cc17..91b8934cf 100644 --- a/apps/cli/test/commands/trace/trace.test.ts +++ b/apps/cli/test/commands/trace/trace.test.ts @@ -331,6 +331,17 @@ describe('trace utils', () => { expect(metas).toEqual([]); }); + it('should skip local export workspaces by default', () => { + const exportDir = path.join(tempDir, '.agentv', 'results', 'export'); + const runDir = path.join(exportDir, 'default', '2026-02-20T21-38-05-833Z'); + mkdirSync(runDir, { recursive: true }); + writeFileSync(path.join(runDir, 'index.jsonl'), `${RESULT_WITH_TRACE}\n`); + + const metas = listResultFiles(tempDir); + + expect(metas).toEqual([]); + }); + it('should respect limit', () => { const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); mkdirSync(runsDir, { recursive: true }); diff --git a/scripts/deploy-example-results-repos.ts b/scripts/deploy-example-results-repos.ts index e353adb2e..24cce316c 100644 --- a/scripts/deploy-example-results-repos.ts +++ b/scripts/deploy-example-results-repos.ts @@ -457,7 +457,7 @@ AgentV stores run artifacts under \`.agentv/results/\` and syncs them through no changed = writeFileIfChanged( - path.join(params.repoDir, '.agentv', 'results', 'runs', '.gitkeep'), + path.join(params.repoDir, '.agentv', 'results', 'default', '.gitkeep'), '', params.dryRun, ) || changed; From 3aa28a8af4f07f2f1a352d7ddf4c5de3f1af2d85 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 23 Jun 2026 03:49:12 +0200 Subject: [PATCH 5/5] fix(results): update reserved namespace expectations --- apps/cli/src/commands/inspect/filter.ts | 2 +- apps/cli/src/commands/inspect/search.ts | 2 +- apps/cli/test/commands/inspect/filter.test.ts | 1 - apps/cli/test/eval.integration.test.ts | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/cli/src/commands/inspect/filter.ts b/apps/cli/src/commands/inspect/filter.ts index 338650fac..2b453db23 100644 --- a/apps/cli/src/commands/inspect/filter.ts +++ b/apps/cli/src/commands/inspect/filter.ts @@ -13,8 +13,8 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; import path from 'node:path'; -import { isReservedResultsNamespace } from '../eval/result-layout.js'; import { command, number, oneOf, option, optional, positional, string } from 'cmd-ts'; +import { isReservedResultsNamespace } from '../eval/result-layout.js'; import { normalizeResultRow } from '../results/result-row-schema.js'; import { c, formatScore, padLeft, padRight } from './utils.js'; diff --git a/apps/cli/src/commands/inspect/search.ts b/apps/cli/src/commands/inspect/search.ts index a2da8b251..a586d8a4e 100644 --- a/apps/cli/src/commands/inspect/search.ts +++ b/apps/cli/src/commands/inspect/search.ts @@ -15,9 +15,9 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; import path from 'node:path'; -import { isReservedResultsNamespace } from '../eval/result-layout.js'; import { command, oneOf, option, optional, positional, string } from 'cmd-ts'; import { toSnakeCaseDeep } from '../../utils/case-conversion.js'; +import { isReservedResultsNamespace } from '../eval/result-layout.js'; import { c, padRight } from './utils.js'; /** A single search match within a JSONL line. */ diff --git a/apps/cli/test/commands/inspect/filter.test.ts b/apps/cli/test/commands/inspect/filter.test.ts index 3a06e2d42..96abbadb5 100644 --- a/apps/cli/test/commands/inspect/filter.test.ts +++ b/apps/cli/test/commands/inspect/filter.test.ts @@ -149,7 +149,6 @@ describe('inspect filter', () => { tempDir, '.agentv', 'results', - 'runs', 'my-experiment', '2026-04-01T10-00-00-000Z', ); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 4d3f14cbd..29830509b 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -406,7 +406,7 @@ describe('agentv eval CLI', () => { await expectFileExists(path.join(outputDir, 'benchmark.json')); await expectFileExists(path.join(outputDir, 'timing.json')); for (const row of canonicalResults) { - expect(row.transcript_path).toMatch(/outputs\/transcript\.jsonl$/); + expect(row.transcript_path).toMatch(/transcript\.jsonl$/); await expectFileExists(path.join(outputDir, row.transcript_path as string)); } } finally {