From d7e1fba1dbe3e913492298e6e948a46c9b9941c4 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 21 Jun 2026 13:11:53 +0200 Subject: [PATCH 1/4] fix(results): publish remote artifact sidecar branch --- .../results/remote-auto-export.test.ts | 86 +++++- .../docs/docs/evaluation/running-evals.mdx | 8 + .../src/content/docs/docs/tools/results.mdx | 2 +- .../evaluation/result-artifact-contract.ts | 5 +- packages/core/src/evaluation/results-repo.ts | 277 +++++++++++++++--- .../core/test/evaluation/results-repo.test.ts | 218 +++++++++++++- 6 files changed, 542 insertions(+), 54 deletions(-) diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index 2d592f2a6..088030ad8 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -1,11 +1,12 @@ import { execSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, spyOn } from 'bun:test'; -import type { EvaluationResult } from '@agentv/core'; +import { AGENTV_RESULTS_ARTIFACTS_REF, type EvaluationResult } from '@agentv/core'; import { maybeAutoExportRunArtifacts } from '../../../src/commands/results/remote.js'; function cleanGitEnv(): Record { @@ -43,7 +44,7 @@ function initializeRemoteRepo(rootDir: string): string { function writeProjectConfig( projectDir: string, - params: { repo: string; path: string; autoPush: boolean }, + params: { repo: string; path: string; autoPush: boolean; branch?: string }, ) { mkdirSync(path.join(projectDir, '.agentv'), { recursive: true }); writeFileSync( @@ -51,7 +52,7 @@ function writeProjectConfig( `results: mode: github repo: ${JSON.stringify(params.repo)} - path: ${JSON.stringify(params.path)} +${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''} path: ${JSON.stringify(params.path)} auto_push: ${params.autoPush} `, ); @@ -71,6 +72,60 @@ function writeRunArtifacts(projectDir: string): string { return runDir; } +function sha256Hex(content: Buffer | string): string { + return createHash('sha256').update(content).digest('hex'); +} + +function writeRunArtifactsWithPointers(projectDir: string): string { + const runDir = path.join(projectDir, '.agentv', 'results', 'runs', 'default', 'run-002'); + const outputsDir = path.join(runDir, 'alpha', 'outputs'); + mkdirSync(outputsDir, { recursive: true }); + const traceContent = Buffer.from('{"schema_version":"agentv.trace.v1","spans":[]}\n'); + const transcriptContent = Buffer.from( + '{"schema_version":"agentv.transcript.v1","role":"assistant","content":"ok"}\n', + ); + writeFileSync(path.join(outputsDir, 'trace.json'), traceContent); + writeFileSync(path.join(outputsDir, 'transcript.jsonl'), transcriptContent); + const traceSha = sha256Hex(traceContent); + const transcriptSha = sha256Hex(transcriptContent); + writeFileSync( + path.join(runDir, 'index.jsonl'), + `${JSON.stringify({ + test_id: 'alpha', + score: 1, + artifact_pointers: { + trace: { + ref: AGENTV_RESULTS_ARTIFACTS_REF, + key: 'traces/alpha/outputs/trace.json', + object_version: `sha256:${traceSha}`, + path: 'alpha/outputs/trace.json', + sha256: traceSha, + size: traceContent.byteLength, + schema_version: 'agentv.trace.v1', + media_type: 'application/vnd.agentv.trace.v1+json', + family: 'traces', + }, + transcript: { + ref: AGENTV_RESULTS_ARTIFACTS_REF, + key: 'transcripts/alpha/outputs/transcript.jsonl', + object_version: `sha256:${transcriptSha}`, + path: 'alpha/outputs/transcript.jsonl', + sha256: transcriptSha, + size: transcriptContent.byteLength, + schema_version: 'agentv.transcript.v1', + media_type: 'application/x-ndjson', + family: 'transcripts', + }, + }, + })}\n`, + ); + writeFileSync( + path.join(runDir, 'benchmark.json'), + `${JSON.stringify({ eval_file: 'evals/example.eval.yaml', tests_run: 1 }, null, 2)}\n`, + ); + return runDir; +} + function payload(projectDir: string, runDir: string) { return { cwd: projectDir, @@ -152,6 +207,31 @@ describe('maybeAutoExportRunArtifacts', () => { ); }, 20_000); + it('pushes sidecar artifact payloads when artifact pointers name the artifact ref', async () => { + const remoteDir = initializeRemoteRepo(rootDir); + const runDir = writeRunArtifactsWithPointers(projectDir); + const resultsBranch = 'dogfood/remote-auto-export'; + writeProjectConfig(projectDir, { + repo: `file://${remoteDir}`, + path: cloneDir, + branch: resultsBranch, + autoPush: true, + }); + + const status = await maybeAutoExportRunArtifacts(payload(projectDir, runDir)); + + expect(status).toBe('published'); + expect( + git(`git --git-dir "${remoteDir}" ls-tree -r --name-only ${resultsBranch}`, rootDir), + ).toContain('runs/default/run-002/index.jsonl'); + const artifactTree = git( + `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_ARTIFACTS_REF}`, + rootDir, + ); + expect(artifactTree).toContain('runs/default/run-002/alpha/outputs/trace.json'); + expect(artifactTree).toContain('runs/default/run-002/alpha/outputs/transcript.jsonl'); + }, 20_000); + it('returns already_published when the final results branch is already up to date', async () => { const remoteDir = initializeRemoteRepo(rootDir); const runDir = writeRunArtifacts(projectDir); diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index ae18284b7..84c5eb59f 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -446,6 +446,14 @@ carry the storage `ref`, artifact `key`, canonical run-relative `path`, viewers and exports can migrate from git refs to object storage without changing the run record contract. +When automatic remote publishing sees pointers whose `ref` is +`agentv/artifacts/v1`, it also pushes those payload bytes to the +`agentv/artifacts/v1` branch in the same results remote at +`runs//`. The configured results branch remains the +self-contained run bundle for current readers, while the sidecar branch gives +artifact-aware consumers a concrete ref whose bytes can be verified with +`object_version` and `sha256`. + `outputs/trace.json` is the full-fidelity `agentv.trace.v1` sidecar. It stores the canonical span graph, source metadata, capture/redaction policy, conversion warnings, score provenance, and opaque evidence references. diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index add4afdfc..a83280643 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -171,7 +171,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo_path: .` with `branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. `repo_path` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo_path: .` with `branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing also pushes those bytes to that sidecar branch at `runs//` in the same remote; the configured results branch still contains the self-contained run bundle for existing readers. `repo_path` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. - **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.path` clone with `git` directly, then sync again. diff --git a/packages/core/src/evaluation/result-artifact-contract.ts b/packages/core/src/evaluation/result-artifact-contract.ts index a8f122bf5..7985a7a2d 100644 --- a/packages/core/src/evaluation/result-artifact-contract.ts +++ b/packages/core/src/evaluation/result-artifact-contract.ts @@ -5,7 +5,10 @@ * records. Local run workspaces still write their files under the existing * per-result artifact directories; these pointers describe where those same * AgentV-owned artifacts belong when projected to a results ref, sidecar ref, - * or object store. + * or object store. Git remote publishing keeps the configured results branch + * self-contained and mirrors pointer payloads whose `ref` is + * `agentv/artifacts/v1` onto that sidecar ref at + * `runs//`. */ export const AGENTV_RESULTS_PRIMARY_REF = 'agentv/results/v1' as const; diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index 11f7f5bd2..a76586805 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -1,4 +1,5 @@ import { execFile, spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, @@ -15,7 +16,10 @@ import { promisify } from 'node:util'; import { getAgentvDataDir } from '../paths.js'; import type { ResultsConfig } from './loaders/config-loader.js'; -import { AGENTV_RESULTS_PRIMARY_REF } from './result-artifact-contract.js'; +import { + AGENTV_RESULTS_ARTIFACTS_REF, + AGENTV_RESULTS_PRIMARY_REF, +} from './result-artifact-contract.js'; const execFileAsync = promisify(execFile); // Local working-tree run workspace inside the eval repo. Local commands @@ -43,6 +47,7 @@ const GIT_EMPTY_TREE = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'; // divergent root. See createOrphanResultsBranch. const RESULTS_REPO_GENESIS_MESSAGE = 'chore(results): initialize AgentV results branch'; const RESULTS_REPO_GENESIS_DATE = '@0 +0000'; +const RESULT_INDEX_FILENAME = 'index.jsonl'; export interface ResultsRepoLocalPaths { readonly rootDir: string; @@ -1464,6 +1469,122 @@ async function listSourceFiles(sourceDir: string): Promise { return entries; } +type ArtifactSidecarPointer = { + readonly path: string; + readonly sha256?: string; + readonly objectVersion?: string; +}; + +type PreparedArtifactSidecar = { + readonly sourceDir: string; + readonly cleanup: () => Promise; +}; + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function artifactSidecarPointers(record: unknown): ArtifactSidecarPointer[] { + if (!isRecord(record) || !isRecord(record.artifact_pointers)) { + return []; + } + + const pointers: ArtifactSidecarPointer[] = []; + for (const pointer of Object.values(record.artifact_pointers)) { + if (!isRecord(pointer)) { + continue; + } + if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== 'string') { + continue; + } + pointers.push({ + path: pointer.path, + ...(typeof pointer.sha256 === 'string' ? { sha256: pointer.sha256 } : {}), + ...(typeof pointer.object_version === 'string' + ? { objectVersion: pointer.object_version } + : {}), + }); + } + return pointers; +} + +function resolveArtifactPointerSourcePath(sourceDir: string, pointerPath: string): string { + const normalizedPointerPath = normalizeDestinationPath(pointerPath); + const sourceRoot = path.resolve(sourceDir); + const sourcePath = path.resolve(sourceRoot, ...normalizedPointerPath.split('/')); + const relative = path.relative(sourceRoot, sourcePath); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`Artifact pointer path escapes run directory: ${pointerPath}`); + } + return sourcePath; +} + +function verifyArtifactPointerChecksum(pointer: ArtifactSidecarPointer, content: Buffer): void { + const sha256 = createHash('sha256').update(content).digest('hex'); + if (pointer.sha256 && pointer.sha256 !== sha256) { + throw new Error(`Artifact pointer checksum mismatch for ${pointer.path}`); + } + if (pointer.objectVersion && pointer.objectVersion !== `sha256:${sha256}`) { + throw new Error(`Artifact pointer object_version mismatch for ${pointer.path}`); + } +} + +async function prepareArtifactSidecar(params: { + readonly sourceDir: string; +}): Promise { + const indexPath = path.join(params.sourceDir, RESULT_INDEX_FILENAME); + if (!existsSync(indexPath)) { + return undefined; + } + + const pointers: ArtifactSidecarPointer[] = []; + for (const line of readFileSync(indexPath, 'utf8').split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + try { + pointers.push(...artifactSidecarPointers(JSON.parse(trimmed))); + } catch {} + } + + if (pointers.length === 0) { + return undefined; + } + + const sidecarRoot = await mkdtemp(path.join(os.tmpdir(), 'agentv-artifact-sidecar-')); + const copied = new Set(); + + try { + for (const pointer of pointers) { + const relativePointerPath = normalizeDestinationPath(pointer.path); + if (copied.has(relativePointerPath)) { + continue; + } + const sourcePath = resolveArtifactPointerSourcePath(params.sourceDir, relativePointerPath); + const content = readFileSync(sourcePath); + verifyArtifactPointerChecksum(pointer, content); + const destinationPath = path.join(sidecarRoot, ...relativePointerPath.split('/')); + mkdirSync(path.dirname(destinationPath), { recursive: true }); + writeFileSync(destinationPath, content); + copied.add(relativePointerPath); + } + } catch (error) { + await rm(sidecarRoot, { recursive: true, force: true }).catch(() => undefined); + throw error; + } + + if (copied.size === 0) { + await rm(sidecarRoot, { recursive: true, force: true }).catch(() => undefined); + return undefined; + } + + return { + sourceDir: sidecarRoot, + cleanup: () => rm(sidecarRoot, { recursive: true, force: true }), + }; +} + async function getExistingRunTreePaths( repoDir: string, ref: string | undefined, @@ -1740,6 +1861,81 @@ async function pushDirectResultsToStorageBranch(params: { } } +async function commitAndMaybePushRunTree(params: { + readonly normalized: StorageBranchResultsConfig; + readonly repoDir: string; + readonly sourceDir: string; + readonly destinationPath: string; + readonly commitMessage: string; + readonly targetRunId: string; + readonly shouldPush: boolean; +}): Promise { + const result = await commitResultsRunWithTemporaryIndex({ + normalized: params.normalized, + repoDir: params.repoDir, + sourceDir: params.sourceDir, + destinationPath: params.destinationPath, + commitMessage: params.commitMessage, + targetRunId: params.targetRunId, + }); + + if (!params.shouldPush) { + updateStatusFile(params.normalized, { last_error: undefined }); + return result.commitCreated; + } + + if (!result.commitCreated) { + const localBranchExists = await gitRefExists( + params.repoDir, + `refs/heads/${params.normalized.branch}`, + ); + const hasUnpushed = result.upstreamRef + ? localBranchExists + ? await hasUnpushedCommits(params.repoDir, result.upstreamRef, params.normalized.branch) + : false + : localBranchExists; + if (!hasUnpushed) { + return false; + } + + const aheadPaths = result.upstreamRef + ? await getAheadPaths( + params.repoDir, + result.upstreamRef, + `refs/heads/${params.normalized.branch}`, + ) + : []; + if (result.upstreamRef && !areSafeResultsRepoPaths(aheadPaths)) { + const error = new Error('Results repo has non-results committed changes'); + updateStatusFile(params.normalized, { last_error: error.message }); + throw error; + } + await pushDirectResultsToStorageBranch({ + normalized: params.normalized, + repoDir: params.repoDir, + storageBranch: params.normalized.branch, + upstreamRef: result.upstreamRef, + sourceDir: params.sourceDir, + destinationPath: params.destinationPath, + commitMessage: params.commitMessage, + targetRunId: params.targetRunId, + }); + return true; + } + + await pushDirectResultsToStorageBranch({ + normalized: params.normalized, + repoDir: params.repoDir, + storageBranch: params.normalized.branch, + upstreamRef: result.upstreamRef, + sourceDir: params.sourceDir, + destinationPath: params.destinationPath, + commitMessage: params.commitMessage, + targetRunId: params.targetRunId, + }); + return true; +} + /** * Push results directly to the configured storage branch of the results repo. * Handles non-fast-forward conflicts by fetching, rebasing, and retrying. @@ -1764,61 +1960,46 @@ export async function directPushResults(params: { branch: storageBranch, }; const targetRunId = buildGitRunId(params.destinationPath); - - const result = await commitResultsRunWithTemporaryIndex({ - normalized: storageConfig, - repoDir, - sourceDir: params.sourceDir, - destinationPath: params.destinationPath, - commitMessage: params.commitMessage, - targetRunId, - }); - const shouldPush = normalized.auto_push || normalized.require_push; - if (!shouldPush) { - updateStatusFile(storageConfig, { last_error: undefined }); - return result.commitCreated; - } + const sidecar = await prepareArtifactSidecar({ sourceDir: params.sourceDir }); - if (!result.commitCreated) { - const hasUnpushed = result.upstreamRef - ? await hasUnpushedCommits(repoDir, result.upstreamRef, storageBranch) - : true; - if (hasUnpushed) { - const aheadPaths = result.upstreamRef - ? await getAheadPaths(repoDir, result.upstreamRef, `refs/heads/${storageBranch}`) - : []; - if (result.upstreamRef && !areSafeResultsRepoPaths(aheadPaths)) { - const error = new Error('Results repo has non-results committed changes'); - updateStatusFile(storageConfig, { last_error: error.message }); - throw error; - } - await pushDirectResultsToStorageBranch({ - normalized: storageConfig, + try { + let sidecarChanged = false; + if (sidecar) { + await fetchResultsRepo(repoDir, normalized.remote, AGENTV_RESULTS_ARTIFACTS_REF).catch( + (error) => { + if (normalized.require_push) { + throw error; + } + }, + ); + sidecarChanged = await commitAndMaybePushRunTree({ + normalized: { + ...normalized, + branch: AGENTV_RESULTS_ARTIFACTS_REF, + }, repoDir, - storageBranch, - upstreamRef: result.upstreamRef, - sourceDir: params.sourceDir, + sourceDir: sidecar.sourceDir, destinationPath: params.destinationPath, - commitMessage: params.commitMessage, + commitMessage: `chore(results): publish artifact sidecars for ${targetRunId}`, targetRunId, + shouldPush, }); - return true; } - return false; - } - await pushDirectResultsToStorageBranch({ - normalized: storageConfig, - repoDir, - storageBranch, - upstreamRef: result.upstreamRef, - sourceDir: params.sourceDir, - destinationPath: params.destinationPath, - commitMessage: params.commitMessage, - targetRunId, - }); - return true; + const primaryChanged = await commitAndMaybePushRunTree({ + normalized: storageConfig, + repoDir, + sourceDir: params.sourceDir, + destinationPath: params.destinationPath, + commitMessage: params.commitMessage, + targetRunId, + shouldPush, + }); + return primaryChanged || sidecarChanged; + } finally { + await sidecar?.cleanup().catch(() => undefined); + } } export interface GitListedRun { diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index c66852b8f..47ee71d2b 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -1,5 +1,14 @@ import { execSync } from 'node:child_process'; -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -43,6 +52,18 @@ function git(cmd: string, cwd: string): string { .trim(); } +function gitRaw(cmd: string, cwd: string): Buffer { + return execSync(cmd, { + cwd, + env: cleanGitEnv(), + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +function sha256Hex(content: Buffer | string): string { + return createHash('sha256').update(content).digest('hex'); +} + function createResultsConfig(repoDir: string, cloneDir: string): ResultsConfig { return { mode: 'github', @@ -112,6 +133,67 @@ function writeRunArtifacts(runDir: string, experiment: string, timestamp: string ); } +function writeRunArtifactsWithPointers( + runDir: string, + experiment: string, + timestamp: string, +): void { + writeRunArtifacts(runDir, experiment, timestamp); + const outputsDir = path.join(runDir, 'alpha', 'outputs'); + mkdirSync(outputsDir, { recursive: true }); + const traceContent = Buffer.from( + JSON.stringify({ + schema_version: 'agentv.trace.v1', + test_id: 'alpha', + spans: [], + }), + ); + const transcriptContent = Buffer.from( + `${JSON.stringify({ + schema_version: 'agentv.transcript.v1', + test_id: 'alpha', + role: 'assistant', + content: 'sidecar transcript', + })}\n`, + ); + writeFileSync(path.join(outputsDir, 'trace.json'), traceContent); + writeFileSync(path.join(outputsDir, 'transcript.jsonl'), transcriptContent); + + const traceSha = sha256Hex(traceContent); + const transcriptSha = sha256Hex(transcriptContent); + writeFileSync( + path.join(runDir, 'index.jsonl'), + `${JSON.stringify({ + test_id: 'alpha', + score: 1, + artifact_pointers: { + trace: { + ref: AGENTV_RESULTS_REFS.artifacts, + key: 'traces/alpha/outputs/trace.json', + object_version: `sha256:${traceSha}`, + path: 'alpha/outputs/trace.json', + sha256: traceSha, + size: traceContent.byteLength, + schema_version: 'agentv.trace.v1', + media_type: 'application/vnd.agentv.trace.v1+json', + family: 'traces', + }, + transcript: { + ref: AGENTV_RESULTS_REFS.artifacts, + key: 'transcripts/alpha/outputs/transcript.jsonl', + object_version: `sha256:${transcriptSha}`, + path: 'alpha/outputs/transcript.jsonl', + sha256: transcriptSha, + size: transcriptContent.byteLength, + schema_version: 'agentv.transcript.v1', + media_type: 'application/x-ndjson', + family: 'transcripts', + }, + }, + })}\n`, + ); +} + describe('listGitRuns', () => { let repoDir: string; @@ -554,6 +636,140 @@ describe('results repo write path', () => { ).toContain(`AgentV-Run: branch-storage::${runTimestamp}`); }, 20000); + it('pushes artifact pointer payloads to the sidecar artifact branch', async () => { + const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); + const storageBranch = initializeRemoteStorageBranch(seedDir, 'dogfood/results-sync'); + const cloneDir = path.join(rootDir, 'results-clone'); + const sourceDir = path.join(rootDir, 'source-run'); + const runTimestamp = '2026-06-21T12-00-00-000Z'; + const destinationPath = path.posix.join('sidecar', runTimestamp); + const config = { + ...createResultsConfig(remoteDir, cloneDir), + branch: storageBranch, + }; + writeRunArtifactsWithPointers(sourceDir, 'sidecar', '2026-06-21T12:00:00.000Z'); + + await expect( + directPushResults({ + config, + sourceDir, + destinationPath, + commitMessage: 'feat(results): sidecar - 1/1 PASS (1.000)', + }), + ).resolves.toBe(true); + + const resultTree = git( + `git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, + rootDir, + ); + expect(resultTree).toContain(`runs/${destinationPath}/index.jsonl`); + expect(resultTree).toContain(`runs/${destinationPath}/benchmark.json`); + + const artifactTree = git( + `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_REFS.artifacts}`, + rootDir, + ); + expect(artifactTree).toContain(`runs/${destinationPath}/alpha/outputs/trace.json`); + expect(artifactTree).toContain(`runs/${destinationPath}/alpha/outputs/transcript.jsonl`); + expect(artifactTree).not.toContain(`runs/${destinationPath}/benchmark.json`); + expect(artifactTree).not.toContain(`runs/${destinationPath}/index.jsonl`); + + const index = JSON.parse( + gitRaw( + `git --git-dir "${remoteDir}" show ${storageBranch}:runs/${destinationPath}/index.jsonl`, + rootDir, + ).toString('utf8'), + ); + for (const pointer of Object.values(index.artifact_pointers) as Array<{ + path: string; + sha256: string; + object_version: string; + }>) { + const bytes = gitRaw( + `git --git-dir "${remoteDir}" show ${AGENTV_RESULTS_REFS.artifacts}:runs/${destinationPath}/${pointer.path}`, + rootDir, + ); + const sha256 = sha256Hex(bytes); + expect(sha256).toBe(pointer.sha256); + expect(pointer.object_version).toBe(`sha256:${sha256}`); + } + + const resultHead = git(`git --git-dir "${remoteDir}" rev-parse ${storageBranch}`, rootDir); + const artifactHead = git( + `git --git-dir "${remoteDir}" rev-parse ${AGENTV_RESULTS_REFS.artifacts}`, + rootDir, + ); + await expect( + directPushResults({ + config, + sourceDir, + destinationPath, + commitMessage: 'feat(results): sidecar - 1/1 PASS (1.000)', + }), + ).resolves.toBe(false); + expect(git(`git --git-dir "${remoteDir}" rev-parse ${storageBranch}`, rootDir)).toBe( + resultHead, + ); + expect( + git(`git --git-dir "${remoteDir}" rev-parse ${AGENTV_RESULTS_REFS.artifacts}`, rootDir), + ).toBe(artifactHead); + }, 30000); + + it('backfills a missing artifact sidecar for an already-published run', async () => { + const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); + const storageBranch = 'dogfood/pr-1462-results-sync'; + const cloneDir = path.join(rootDir, 'results-clone'); + const sourceDir = path.join(rootDir, 'source-run'); + const runTimestamp = '2026-06-21T13-00-00-000Z'; + const destinationPath = path.posix.join('sidecar-backfill', runTimestamp); + const config = { + ...createResultsConfig(remoteDir, cloneDir), + branch: storageBranch, + }; + writeRunArtifactsWithPointers(sourceDir, 'sidecar-backfill', '2026-06-21T13:00:00.000Z'); + + git(`git switch --quiet --orphan ${storageBranch}`, seedDir); + git('git rm -rf --quiet . 2>/dev/null || true', seedDir); + const seededRunDir = path.join(seedDir, 'runs', ...destinationPath.split('/')); + mkdirSync(path.dirname(seededRunDir), { recursive: true }); + cpSync(sourceDir, seededRunDir, { recursive: true }); + git('git add runs && git commit --quiet -m "seed published run"', seedDir); + git(`git push --quiet origin HEAD:${storageBranch}`, seedDir); + git('git switch --quiet main', seedDir); + const seededResultsHead = git( + `git --git-dir "${remoteDir}" rev-parse ${storageBranch}`, + rootDir, + ); + + await expect( + directPushResults({ + config, + sourceDir, + destinationPath, + commitMessage: 'feat(results): sidecar backfill - 1/1 PASS (1.000)', + }), + ).resolves.toBe(true); + + expect(git(`git --git-dir "${remoteDir}" rev-parse ${storageBranch}`, rootDir)).toBe( + seededResultsHead, + ); + const artifactTree = git( + `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_REFS.artifacts}`, + rootDir, + ); + expect(artifactTree).toContain(`runs/${destinationPath}/alpha/outputs/trace.json`); + expect(artifactTree).toContain(`runs/${destinationPath}/alpha/outputs/transcript.jsonl`); + + await expect( + directPushResults({ + config, + sourceDir, + destinationPath, + commitMessage: 'feat(results): sidecar backfill - 1/1 PASS (1.000)', + }), + ).resolves.toBe(false); + }, 30000); + it('auto-creates a missing configured storage branch', async () => { const { remoteDir } = initializeRemoteRepo(rootDir); const cloneDir = path.join(rootDir, 'results-clone'); From 0d4b8da34dfa3fff4a60663cee17cdc5528addbc Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 21 Jun 2026 14:15:38 +0200 Subject: [PATCH 2/4] fix(results): make sidecar artifacts canonical --- apps/cli/src/commands/results/serve.ts | 206 ++++++++++++++-- .../results/remote-auto-export.test.ts | 23 +- apps/cli/test/commands/results/serve.test.ts | 137 ++++++++++- .../docs/docs/evaluation/running-evals.mdx | 14 +- .../src/content/docs/docs/tools/results.mdx | 2 +- .../evaluation/result-artifact-contract.ts | 8 +- packages/core/src/evaluation/results-repo.ts | 228 +++++++++++++++--- packages/core/src/index.ts | 2 + .../core/test/evaluation/results-repo.test.ts | 20 +- 9 files changed, 570 insertions(+), 70 deletions(-) diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index a846f0beb..1b3557463 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -49,6 +49,7 @@ import { fileURLToPath } from 'node:url'; import { command, flag, number, option, optional, positional, string } from 'cmd-ts'; import { + AGENTV_RESULTS_ARTIFACTS_REF, DEFAULT_CATEGORY, type EvaluationResult, addProject, @@ -56,6 +57,7 @@ import { loadConfig, loadProjectRegistry, removeProject, + readGitResultArtifact, syncProjects, touchProject, } from '@agentv/core'; @@ -88,6 +90,7 @@ import { findRunById, getRemoteResultsStatus, listMergedResultFiles, + loadNormalizedResultsConfig, readRemoteRunTagState, setRemoteRunTags, syncRemoteResults, @@ -342,39 +345,83 @@ function artifactPointerRef(pointer: unknown): string | undefined { return isRecord(pointer) ? nonEmptyString(pointer.ref) : undefined; } +function artifactPointerKey(pointer: unknown): string | undefined { + return isRecord(pointer) ? nonEmptyString(pointer.key) : undefined; +} + +function artifactPointerSha256(pointer: unknown): string | undefined { + return isRecord(pointer) ? nonEmptyString(pointer.sha256) : undefined; +} + +function artifactPointerObjectVersion(pointer: unknown): string | undefined { + return isRecord(pointer) ? nonEmptyString(pointer.object_version) : undefined; +} + interface ResolvedArtifactPointer { readonly path?: string; + readonly key?: string; readonly description?: string; readonly ref?: string; + readonly sha256?: string; + readonly objectVersion?: string; readonly unsupportedReason?: string; } function resolveRecordArtifactPointer( record: ResultManifestRecord, - kind: 'transcript' | 'answer', + kind: 'transcript' | 'answer' | 'trace', ): ResolvedArtifactPointer { + const pointer = + kind === 'transcript' + ? (record.transcript ?? record.artifacts?.transcript ?? record.artifact_pointers?.transcript) + : kind === 'trace' + ? (record.artifact_pointers?.trace ?? record.artifacts?.['trace']) + : record.artifacts?.answer; + const pointerPath = artifactPointerPath(pointer); + const description = artifactPointerDescription(pointer); + const ref = artifactPointerRef(pointer); + const key = artifactPointerKey(pointer); + const sha256 = artifactPointerSha256(pointer); + const objectVersion = artifactPointerObjectVersion(pointer); + if ((kind === 'transcript' || kind === 'trace') && pointerPath) { + return { + path: pointerPath, + description, + ref, + ...(key && { key }), + ...(sha256 && { sha256 }), + ...(objectVersion && { objectVersion }), + }; + } + + const recordWithTrace = record as ResultManifestRecord & { readonly trace_path?: string }; const directPath = kind === 'transcript' ? (record.transcript_path ?? record.artifacts?.transcript_path) - : (record.answer_path ?? record.artifacts?.answer_path ?? record.output_path); + : kind === 'trace' + ? (recordWithTrace.trace_path ?? nonEmptyString(record.artifacts?.['trace_path'])) + : (record.answer_path ?? record.artifacts?.answer_path ?? record.output_path); if (directPath) { return { path: directPath, description: directPath }; } - const pointer = - kind === 'transcript' - ? (record.transcript ?? record.artifacts?.transcript ?? record.artifact_pointers?.transcript) - : record.artifacts?.answer; - const pointerPath = artifactPointerPath(pointer); - const description = artifactPointerDescription(pointer); - const ref = artifactPointerRef(pointer); if (pointerPath) { - return { path: pointerPath, description, ref }; + return { + path: pointerPath, + description, + ref, + ...(key && { key }), + ...(sha256 && { sha256 }), + ...(objectVersion && { objectVersion }), + }; } if (pointer) { return { description, ref, + ...(key && { key }), + ...(sha256 && { sha256 }), + ...(objectVersion && { objectVersion }), unsupportedReason: description ? `${kind} artifact pointer does not include a local path (${description}).` : `${kind} artifact pointer does not include a local path.`, @@ -443,6 +490,101 @@ function readOptionalRunArtifactText( return readFileSync(resolved.absolutePath, 'utf8'); } +function normalizeArtifactRelativePath(relativePath: string): string | undefined { + const normalized = relativePath.split(path.sep).join('/'); + const segments = normalized.split('/').filter(Boolean); + if ( + segments.length === 0 || + normalized.startsWith('/') || + segments.some((segment) => segment === '..') + ) { + return undefined; + } + return segments.join('/'); +} + +function relativeRunPathFromManifest(repoDir: string, manifestPath: string): string | undefined { + const relativeManifestPath = path.relative(repoDir, manifestPath).split(path.sep).join('/'); + if ( + relativeManifestPath.length === 0 || + relativeManifestPath === manifestPath || + relativeManifestPath.startsWith('../') + ) { + return undefined; + } + + const parts = relativeManifestPath.split('/').filter(Boolean); + const runsIndex = parts.lastIndexOf('runs'); + if (runsIndex === -1 || parts.at(-1) !== 'index.jsonl') { + return undefined; + } + const runParts = parts.slice(runsIndex + 1, -1); + return runParts.length > 0 ? runParts.join('/') : undefined; +} + +function sidecarArtifactKeyForPointer( + repoDir: string, + manifestPath: string, + artifact: ResolvedArtifactPointer, +): string | undefined { + const publishedKey = artifact.key ? normalizeArtifactRelativePath(artifact.key) : undefined; + if (publishedKey?.startsWith('runs/')) { + return publishedKey; + } + if (!artifact.path) { + return undefined; + } + const relativeArtifactPath = normalizeArtifactRelativePath(artifact.path); + const relativeRunPath = relativeRunPathFromManifest(repoDir, manifestPath); + if (!relativeArtifactPath || !relativeRunPath) { + return undefined; + } + return ['runs', relativeRunPath, relativeArtifactPath].join('/'); +} + +async function readSidecarArtifactText( + searchDir: string, + projectId: string | undefined, + meta: SourcedResultFileMeta, + artifact: ResolvedArtifactPointer, +): Promise { + if (artifact.ref !== AGENTV_RESULTS_ARTIFACTS_REF) { + return undefined; + } + const config = await loadNormalizedResultsConfig(searchDir, projectId); + if (!config) { + return undefined; + } + const key = sidecarArtifactKeyForPointer(config.path, meta.path, artifact); + if (!key) { + return undefined; + } + const bytes = await readGitResultArtifact({ + repoDir: config.path, + key, + ref: AGENTV_RESULTS_ARTIFACTS_REF, + remote: config.remote, + ...(artifact.sha256 && { sha256: artifact.sha256 }), + ...(artifact.objectVersion && { objectVersion: artifact.objectVersion }), + }); + return bytes?.toString('utf8'); +} + +function artifactFileContentResponse(c: C, filePath: string, fileContent: string) { + if (c.req.query('raw') === '1' || c.req.query('download') === '1') { + c.header('Content-Type', inferRawContentType(filePath)); + if (c.req.query('download') === '1') { + c.header( + 'Content-Disposition', + `attachment; filename="${contentDispositionFilename(filePath)}"`, + ); + } + return c.body(fileContent); + } + const language = inferLanguage(filePath); + return c.json({ content: fileContent, language }); +} + function missingTranscriptMessage(): string { return [ 'This result does not include canonical outputs/transcript.jsonl metadata.', @@ -1062,6 +1204,7 @@ async function handleEvalFiles(c: C, { searchDir, projectId }: DataContext) { async function handleEvalFileContent(c: C, { searchDir, projectId }: DataContext) { const filename = c.req.param('filename') ?? ''; + const evalId = c.req.param('evalId'); const meta = await findRunById(searchDir, filename, projectId); if (!meta) return c.json({ error: 'Run not found' }, 404); @@ -1085,23 +1228,32 @@ async function handleEvalFileContent(c: C, { searchDir, projectId }: DataContext return c.json({ error: 'Path traversal not allowed' }, 403); } if (!resolvedFile.absolutePath) { + const records = parseResultManifest(readFileSync(meta.path, 'utf8')); + const record = records.find((r) => r.test_id === evalId); + if (record) { + const normalizedFilePath = normalizeArtifactRelativePath(filePath); + const artifact = [ + resolveRecordArtifactPointer(record, 'trace'), + resolveRecordArtifactPointer(record, 'transcript'), + ].find((candidate) => { + const pointerPath = candidate.path + ? normalizeArtifactRelativePath(candidate.path) + : undefined; + return pointerPath !== undefined && pointerPath === normalizedFilePath; + }); + if (artifact) { + const sidecarContent = await readSidecarArtifactText(searchDir, projectId, meta, artifact); + if (sidecarContent !== undefined) { + return artifactFileContentResponse(c, filePath, sidecarContent); + } + } + } return c.json({ error: 'File not found' }, 404); } try { const fileContent = readFileSync(resolvedFile.absolutePath, 'utf8'); - if (c.req.query('raw') === '1' || c.req.query('download') === '1') { - c.header('Content-Type', inferRawContentType(filePath)); - if (c.req.query('download') === '1') { - c.header( - 'Content-Disposition', - `attachment; filename="${contentDispositionFilename(filePath)}"`, - ); - } - return c.body(fileContent); - } - const language = inferLanguage(filePath); - return c.json({ content: fileContent, language }); + return artifactFileContentResponse(c, filePath, fileContent); } catch { return c.json({ error: 'Failed to read file' }, 500); } @@ -1140,7 +1292,14 @@ async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext) }); } - if (!resolvedTranscript.absolutePath) { + let content: string | undefined; + if (resolvedTranscript.absolutePath) { + content = readFileSync(resolvedTranscript.absolutePath, 'utf8'); + } else { + content = await readSidecarArtifactText(searchDir, projectId, meta, transcript); + } + + if (content === undefined) { const refMessage = transcript.ref ? ` on ${transcript.ref}` : ''; return c.json({ status: 'dangling', @@ -1150,7 +1309,6 @@ async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext) }); } - const content = readFileSync(resolvedTranscript.absolutePath, 'utf8'); const answerContent = readOptionalRunArtifactText(baseDir, answer); return c.json({ diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index 088030ad8..0bb735150 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -221,9 +221,26 @@ describe('maybeAutoExportRunArtifacts', () => { const status = await maybeAutoExportRunArtifacts(payload(projectDir, runDir)); expect(status).toBe('published'); - expect( - git(`git --git-dir "${remoteDir}" ls-tree -r --name-only ${resultsBranch}`, rootDir), - ).toContain('runs/default/run-002/index.jsonl'); + const resultTree = git( + `git --git-dir "${remoteDir}" ls-tree -r --name-only ${resultsBranch}`, + rootDir, + ); + expect(resultTree).toContain('runs/default/run-002/index.jsonl'); + expect(resultTree).toContain('runs/default/run-002/benchmark.json'); + expect(resultTree).not.toContain('runs/default/run-002/alpha/outputs/trace.json'); + expect(resultTree).not.toContain('runs/default/run-002/alpha/outputs/transcript.jsonl'); + const index = JSON.parse( + git( + `git --git-dir "${remoteDir}" show ${resultsBranch}:runs/default/run-002/index.jsonl`, + rootDir, + ), + ); + expect(index.artifact_pointers.trace.key).toBe( + 'runs/default/run-002/alpha/outputs/trace.json', + ); + expect(index.artifact_pointers.transcript.key).toBe( + 'runs/default/run-002/alpha/outputs/transcript.jsonl', + ); const artifactTree = git( `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_ARTIFACTS_REF}`, rootDir, diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 389089eaf..e51c8d6b5 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -14,7 +14,7 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { addProject, saveProjectRegistry } from '@agentv/core'; +import { AGENTV_RESULTS_ARTIFACTS_REF, addProject, saveProjectRegistry } from '@agentv/core'; import { RUN_OPLOG_REF } from '../../../src/commands/results/run-oplog.js'; import { @@ -2806,6 +2806,141 @@ describe('serve app', () => { expect(data.pointer).toContain('agentv/artifacts/v1'); }); + it('resolves remote sidecar transcript and trace artifacts only on demand', async () => { + const { remoteDir, cloneDir, seedDir } = initializeRemoteRepo(tempDir); + const resultsBranch = 'dogfood/serve-sidecar-contract'; + const experiment = 'sidecar-only'; + const timestamp = '2026-03-25T15-00-00-000Z'; + const runId = `remote::${experiment}::${timestamp}`; + const transcriptArtifactPath = 'demo/test-greeting/outputs/transcript.jsonl'; + const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; + const transcriptKey = `runs/${experiment}/${timestamp}/${transcriptArtifactPath}`; + const traceKey = `runs/${experiment}/${timestamp}/${traceArtifactPath}`; + const transcriptJsonl = `${JSON.stringify({ + schema_version: 'agentv.transcript.v1', + test_id: 'test-greeting', + target: 'gpt-4o', + message_index: 0, + role: 'assistant', + content: 'sidecar transcript body', + })}\n`; + const traceJson = `${JSON.stringify({ + schema_version: 'agentv.trace.v1', + test_id: 'test-greeting', + spans: [{ name: 'root' }], + })}\n`; + + git(`git switch --quiet --orphan ${resultsBranch}`, seedDir); + git('git rm -rf --quiet . 2>/dev/null || true', seedDir); + const runDir = path.join(seedDir, 'runs', experiment, timestamp); + mkdirSync(runDir, { recursive: true }); + writeFileSync( + path.join(runDir, 'index.jsonl'), + toJsonl({ + ...RESULT_A, + experiment, + transcript_path: transcriptArtifactPath, + trace_path: traceArtifactPath, + artifact_pointers: { + trace: { + ref: AGENTV_RESULTS_ARTIFACTS_REF, + key: traceKey, + path: traceArtifactPath, + }, + transcript: { + ref: AGENTV_RESULTS_ARTIFACTS_REF, + key: transcriptKey, + path: transcriptArtifactPath, + }, + }, + }), + ); + writeFileSync( + path.join(runDir, 'benchmark.json'), + JSON.stringify( + { + metadata: { + timestamp: '2026-03-25T15:00:00.000Z', + experiment, + targets: ['gpt-4o'], + tests_run: ['test-greeting'], + }, + run_summary: { 'gpt-4o': { pass_rate: { mean: 1 } } }, + }, + null, + 2, + ), + ); + git('git add runs && git commit --quiet -m "seed metadata-only results"', seedDir); + git(`git push --quiet origin HEAD:${resultsBranch}`, seedDir); + + git(`git switch --quiet --orphan ${AGENTV_RESULTS_ARTIFACTS_REF}`, seedDir); + git('git rm -rf --quiet . 2>/dev/null || true', seedDir); + const transcriptPath = path.join(seedDir, ...transcriptKey.split('/')); + const tracePath = path.join(seedDir, ...traceKey.split('/')); + mkdirSync(path.dirname(transcriptPath), { recursive: true }); + writeFileSync(transcriptPath, transcriptJsonl); + writeFileSync(tracePath, traceJson); + git('git add runs && git commit --quiet -m "seed artifact sidecars"', seedDir); + git(`git push --quiet origin HEAD:${AGENTV_RESULTS_ARTIFACTS_REF}`, seedDir); + git('git switch --quiet main', seedDir); + + mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); + writeFileSync( + path.join(tempDir, '.agentv', 'config.yaml'), + `results: + mode: github + repo: ${JSON.stringify(`file://${remoteDir}`)} + branch: ${JSON.stringify(resultsBranch)} + path: ${JSON.stringify(cloneDir)} + auto_push: false +`, + ); + + const artifactRemoteRef = `refs/remotes/origin/${AGENTV_RESULTS_ARTIFACTS_REF}`; + const artifactRefLookup = () => + git( + `git -C "${cloneDir}" show-ref --verify --quiet ${artifactRemoteRef} && echo present || true`, + tempDir, + ); + expect(artifactRefLookup()).toBe(''); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const listRes = await app.request('/api/runs'); + expect(listRes.status).toBe(200); + expect(artifactRefLookup()).toBe(''); + + const detailRes = await app.request(`/api/runs/${encodeURIComponent(runId)}`); + expect(detailRes.status).toBe(200); + expect(await detailRes.text()).not.toContain('sidecar transcript body'); + expect(artifactRefLookup()).toBe(''); + expect(existsSync(path.join(cloneDir, ...transcriptKey.split('/')))).toBe(false); + expect(existsSync(path.join(cloneDir, ...traceKey.split('/')))).toBe(false); + + const transcriptRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/transcript`, + ); + expect(transcriptRes.status).toBe(200); + const transcriptData = (await transcriptRes.json()) as { + status: string; + content: string; + transcript_path: string; + }; + expect(transcriptData.status).toBe('ok'); + expect(transcriptData.transcript_path).toBe(transcriptArtifactPath); + expect(transcriptData.content).toBe(transcriptJsonl); + expect(artifactRefLookup()).toBe('present'); + expect(existsSync(path.join(cloneDir, ...transcriptKey.split('/')))).toBe(false); + + const traceRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/files/${traceArtifactPath}?raw=1`, + ); + expect(traceRes.status).toBe(200); + expect(traceRes.headers.get('content-type')).toContain('application/json'); + expect(await traceRes.text()).toBe(traceJson); + expect(existsSync(path.join(cloneDir, ...traceKey.split('/')))).toBe(false); + }); + it('returns a clear missing state when no transcript pointer is recorded', async () => { const runId = writeLocalRunArtifact( tempDir, diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 84c5eb59f..d838d1a0d 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -449,10 +449,16 @@ the run record contract. When automatic remote publishing sees pointers whose `ref` is `agentv/artifacts/v1`, it also pushes those payload bytes to the `agentv/artifacts/v1` branch in the same results remote at -`runs//`. The configured results branch remains the -self-contained run bundle for current readers, while the sidecar branch gives -artifact-aware consumers a concrete ref whose bytes can be verified with -`object_version` and `sha256`. +`runs//` and rewrites the published pointer `key` to +that backend object key. The configured results branch is the metadata/control +plane for `index.jsonl`, `benchmark.json`, tags, and pointers; it does not +duplicate canonical trace/transcript payload bodies when those rows name +`agentv/artifacts/v1`. Local pre-publish run workspaces can still contain the +files beside the manifest, and Dashboard resolves the published pointers lazily +when a transcript or trace view requests the payload. AgentV keeps this explicit +pointer/backend contract instead of using Git LFS as the core abstraction so +S3, B2, or other object stores can use the same `key`, `object_version`, +`sha256`, `size`, `media_type`, and `schema_version` fields later. `outputs/trace.json` is the full-fidelity `agentv.trace.v1` sidecar. It stores the canonical span graph, source metadata, capture/redaction policy, diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index a83280643..eb1e134b8 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -171,7 +171,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo_path: .` with `branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing also pushes those bytes to that sidecar branch at `runs//` in the same remote; the configured results branch still contains the self-contained run bundle for existing readers. `repo_path` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo_path: .` with `branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. `repo_path` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. - **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.path` clone with `git` directly, then sync again. diff --git a/packages/core/src/evaluation/result-artifact-contract.ts b/packages/core/src/evaluation/result-artifact-contract.ts index 7985a7a2d..b61d581ca 100644 --- a/packages/core/src/evaluation/result-artifact-contract.ts +++ b/packages/core/src/evaluation/result-artifact-contract.ts @@ -5,10 +5,10 @@ * records. Local run workspaces still write their files under the existing * per-result artifact directories; these pointers describe where those same * AgentV-owned artifacts belong when projected to a results ref, sidecar ref, - * or object store. Git remote publishing keeps the configured results branch - * self-contained and mirrors pointer payloads whose `ref` is - * `agentv/artifacts/v1` onto that sidecar ref at - * `runs//`. + * or object store. Git remote publishing treats the configured results branch + * as the metadata/control plane and stores transcript or trace payload bytes + * whose `ref` is `agentv/artifacts/v1` on that artifact ref at the published + * pointer `key` (`runs//` for the git backend). */ export const AGENTV_RESULTS_PRIMARY_REF = 'agentv/results/v1' as const; diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index a76586805..fedce270f 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -310,16 +310,13 @@ async function fetchResultsRepo( remote = 'origin', branch?: string, ): Promise { - await runGit(['fetch', remote, '--prune'], { cwd: repoDir }); - // A shallow `--single-branch --branch ` clone (e.g. the containerized - // deploy) has a fetch refspec covering only the default branch, so the prune - // fetch above never learns about the results branch. Fetch it explicitly by - // name into its remote-tracking ref so the branch is detected and appended to - // rather than re-rooted as a fresh orphan. Tolerate absence: the branch simply - // may not exist on the remote yet (first publish). if (branch) { await fetchResultsBranchRef(repoDir, remote, branch); + await runGit(['remote', 'prune', remote], { cwd: repoDir, check: false }); + return; } + + await runGit(['fetch', remote, '--prune'], { cwd: repoDir }); } async function fetchResultsBranchRef( @@ -333,6 +330,21 @@ async function fetchResultsBranchRef( }); } +async function fetchResultsArtifactRef( + repoDir: string, + remote: string, + branch: string, +): Promise { + const refspec = `+refs/heads/${branch}:refs/remotes/${remote}/${branch}`; + await runGit(['fetch', '--filter=blob:none', remote, refspec], { + cwd: repoDir, + check: false, + }); + if (!(await gitRefExists(repoDir, `refs/remotes/${remote}/${branch}`))) { + await fetchResultsBranchRef(repoDir, remote, branch); + } +} + function remoteBranchRef(branch: string, remote = 'origin'): string { return `${remote}/${branch}`; } @@ -1508,6 +1520,33 @@ function artifactSidecarPointers(record: unknown): ArtifactSidecarPointer[] { return pointers; } +function artifactSidecarKey(destinationPath: string, pointerPath: string): string { + return path.posix.join( + RESULTS_REPO_RUNS_DIR, + normalizeDestinationPath(destinationPath), + normalizeDestinationPath(pointerPath), + ); +} + +function collectArtifactSidecarPointers(sourceDir: string): ArtifactSidecarPointer[] { + const indexPath = path.join(sourceDir, RESULT_INDEX_FILENAME); + if (!existsSync(indexPath)) { + return []; + } + + const pointers: ArtifactSidecarPointer[] = []; + for (const line of readFileSync(indexPath, 'utf8').split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + try { + pointers.push(...artifactSidecarPointers(JSON.parse(trimmed))); + } catch {} + } + return pointers; +} + function resolveArtifactPointerSourcePath(sourceDir: string, pointerPath: string): string { const normalizedPointerPath = normalizeDestinationPath(pointerPath); const sourceRoot = path.resolve(sourceDir); @@ -1531,39 +1570,29 @@ function verifyArtifactPointerChecksum(pointer: ArtifactSidecarPointer, content: async function prepareArtifactSidecar(params: { readonly sourceDir: string; + readonly pointers: readonly ArtifactSidecarPointer[]; }): Promise { - const indexPath = path.join(params.sourceDir, RESULT_INDEX_FILENAME); - if (!existsSync(indexPath)) { - return undefined; - } - - const pointers: ArtifactSidecarPointer[] = []; - for (const line of readFileSync(indexPath, 'utf8').split(/\r?\n/)) { - const trimmed = line.trim(); - if (!trimmed) { - continue; - } - try { - pointers.push(...artifactSidecarPointers(JSON.parse(trimmed))); - } catch {} - } - - if (pointers.length === 0) { + if (params.pointers.length === 0) { return undefined; } const sidecarRoot = await mkdtemp(path.join(os.tmpdir(), 'agentv-artifact-sidecar-')); const copied = new Set(); + const contentByPath = new Map(); try { - for (const pointer of pointers) { + for (const pointer of params.pointers) { const relativePointerPath = normalizeDestinationPath(pointer.path); + let content = contentByPath.get(relativePointerPath); + if (!content) { + const sourcePath = resolveArtifactPointerSourcePath(params.sourceDir, relativePointerPath); + content = readFileSync(sourcePath); + contentByPath.set(relativePointerPath, content); + } + verifyArtifactPointerChecksum(pointer, content); if (copied.has(relativePointerPath)) { continue; } - const sourcePath = resolveArtifactPointerSourcePath(params.sourceDir, relativePointerPath); - const content = readFileSync(sourcePath); - verifyArtifactPointerChecksum(pointer, content); const destinationPath = path.join(sidecarRoot, ...relativePointerPath.split('/')); mkdirSync(path.dirname(destinationPath), { recursive: true }); writeFileSync(destinationPath, content); @@ -1585,6 +1614,86 @@ async function prepareArtifactSidecar(params: { }; } +function rewritePublishedIndexLine(line: string, destinationPath: string): string { + if (!line.trim()) { + return line; + } + + let record: unknown; + try { + record = JSON.parse(line); + } catch { + return line; + } + + if (!isRecord(record) || !isRecord(record.artifact_pointers)) { + return line; + } + + let changed = false; + for (const pointer of Object.values(record.artifact_pointers)) { + if (!isRecord(pointer)) { + continue; + } + if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== 'string') { + continue; + } + const key = artifactSidecarKey(destinationPath, pointer.path); + if (pointer.key !== key) { + pointer.key = key; + changed = true; + } + } + + return changed ? JSON.stringify(record) : line; +} + +async function preparePublishedResultsSource(params: { + readonly sourceDir: string; + readonly destinationPath: string; + readonly pointers: readonly ArtifactSidecarPointer[]; +}): Promise { + if (params.pointers.length === 0) { + return undefined; + } + + const publishedRoot = await mkdtemp(path.join(os.tmpdir(), 'agentv-results-published-')); + const omittedPaths = new Set( + params.pointers.map((pointer) => normalizeDestinationPath(pointer.path)), + ); + + try { + const sourceFiles = await listSourceFiles(params.sourceDir); + for (const sourceFile of sourceFiles) { + const relativeFile = path.relative(params.sourceDir, sourceFile).split(path.sep).join('/'); + const destinationFile = path.join(publishedRoot, ...relativeFile.split('/')); + if (relativeFile === RESULT_INDEX_FILENAME) { + const original = readFileSync(sourceFile, 'utf8'); + const rewritten = original + .split(/\r?\n/) + .map((line) => rewritePublishedIndexLine(line, params.destinationPath)) + .join('\n'); + mkdirSync(path.dirname(destinationFile), { recursive: true }); + writeFileSync(destinationFile, rewritten); + continue; + } + if (omittedPaths.has(relativeFile)) { + continue; + } + mkdirSync(path.dirname(destinationFile), { recursive: true }); + await cp(sourceFile, destinationFile, { dereference: false }); + } + } catch (error) { + await rm(publishedRoot, { recursive: true, force: true }).catch(() => undefined); + throw error; + } + + return { + sourceDir: publishedRoot, + cleanup: () => rm(publishedRoot, { recursive: true, force: true }), + }; +} + async function getExistingRunTreePaths( repoDir: string, ref: string | undefined, @@ -1961,9 +2070,20 @@ export async function directPushResults(params: { }; const targetRunId = buildGitRunId(params.destinationPath); const shouldPush = normalized.auto_push || normalized.require_push; - const sidecar = await prepareArtifactSidecar({ sourceDir: params.sourceDir }); + const sidecarPointers = collectArtifactSidecarPointers(params.sourceDir); + const sidecar = await prepareArtifactSidecar({ + sourceDir: params.sourceDir, + pointers: sidecarPointers, + }); + let publishedResultsSource: PreparedArtifactSidecar | undefined; try { + publishedResultsSource = await preparePublishedResultsSource({ + sourceDir: params.sourceDir, + destinationPath: params.destinationPath, + pointers: sidecarPointers, + }); + let sidecarChanged = false; if (sidecar) { await fetchResultsRepo(repoDir, normalized.remote, AGENTV_RESULTS_ARTIFACTS_REF).catch( @@ -1990,7 +2110,7 @@ export async function directPushResults(params: { const primaryChanged = await commitAndMaybePushRunTree({ normalized: storageConfig, repoDir, - sourceDir: params.sourceDir, + sourceDir: publishedResultsSource?.sourceDir ?? params.sourceDir, destinationPath: params.destinationPath, commitMessage: params.commitMessage, targetRunId, @@ -1998,6 +2118,7 @@ export async function directPushResults(params: { }); return primaryChanged || sidecarChanged; } finally { + await publishedResultsSource?.cleanup().catch(() => undefined); await sidecar?.cleanup().catch(() => undefined); } } @@ -2165,6 +2286,53 @@ function parseGitBatchBlobs(output: Buffer): GitBatchBlob[] { return blobs; } +export interface GitResultArtifactReadParams { + readonly repoDir: string; + readonly key: string; + readonly ref?: string; + readonly remote?: string; + readonly sha256?: string; + readonly objectVersion?: string; +} + +export async function readGitResultArtifact( + params: GitResultArtifactReadParams, +): Promise { + const artifactRef = params.ref ?? AGENTV_RESULTS_ARTIFACTS_REF; + const normalizedKey = normalizeDestinationPath(params.key); + const remote = params.remote ?? 'origin'; + if (!artifactRef.startsWith('refs/')) { + await fetchResultsArtifactRef(params.repoDir, remote, artifactRef).catch(() => undefined); + } + + const candidateRefs = [ + `refs/remotes/${remote}/${artifactRef}`, + `refs/heads/${artifactRef}`, + artifactRef, + ].filter((candidate, index, all) => all.indexOf(candidate) === index); + + for (const candidateRef of candidateRefs) { + const blobs = parseGitBatchBlobs( + await runGitBatch(params.repoDir, `${candidateRef}:${normalizedKey}\n`), + ); + const blob = blobs[0]; + if (!blob) { + continue; + } + verifyArtifactPointerChecksum( + { + path: normalizedKey, + ...(params.sha256 ? { sha256: params.sha256 } : {}), + ...(params.objectVersion ? { objectVersion: params.objectVersion } : {}), + }, + blob.content, + ); + return Buffer.from(blob.content); + } + + return undefined; +} + // ── WIP (work-in-progress) branch helpers ───────────────────────────────── // // Periodic best-effort checkpoints push the partial run output to a unique diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c0e10451d..498285a01 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -120,7 +120,9 @@ export { deleteWipBranch, listGitRuns, materializeGitRun, + readGitResultArtifact, type CheckedOutResultsRepoBranch, + type GitResultArtifactReadParams, type GitListedRun, type NormalizedResultsConfig, type PreparedResultsRepoBranch, diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index 47ee71d2b..826936ef0 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -664,6 +664,8 @@ describe('results repo write path', () => { ); expect(resultTree).toContain(`runs/${destinationPath}/index.jsonl`); expect(resultTree).toContain(`runs/${destinationPath}/benchmark.json`); + expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/outputs/trace.json`); + expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/outputs/transcript.jsonl`); const artifactTree = git( `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_REFS.artifacts}`, @@ -681,12 +683,14 @@ describe('results repo write path', () => { ).toString('utf8'), ); for (const pointer of Object.values(index.artifact_pointers) as Array<{ + key: string; path: string; sha256: string; object_version: string; }>) { + expect(pointer.key).toBe(`runs/${destinationPath}/${pointer.path}`); const bytes = gitRaw( - `git --git-dir "${remoteDir}" show ${AGENTV_RESULTS_REFS.artifacts}:runs/${destinationPath}/${pointer.path}`, + `git --git-dir "${remoteDir}" show ${AGENTV_RESULTS_REFS.artifacts}:${pointer.key}`, rootDir, ); const sha256 = sha256Hex(bytes); @@ -750,9 +754,19 @@ describe('results repo write path', () => { }), ).resolves.toBe(true); - expect(git(`git --git-dir "${remoteDir}" rev-parse ${storageBranch}`, rootDir)).toBe( - seededResultsHead, + const migratedResultsHead = git( + `git --git-dir "${remoteDir}" rev-parse ${storageBranch}`, + rootDir, ); + expect(migratedResultsHead).not.toBe(seededResultsHead); + const resultTree = git( + `git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, + rootDir, + ); + expect(resultTree).toContain(`runs/${destinationPath}/index.jsonl`); + expect(resultTree).toContain(`runs/${destinationPath}/benchmark.json`); + expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/outputs/trace.json`); + expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/outputs/transcript.jsonl`); const artifactTree = git( `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_REFS.artifacts}`, rootDir, From f840d646e73cf20772f4c748f7c7c6a0ef5056ce Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 21 Jun 2026 14:20:59 +0200 Subject: [PATCH 3/4] chore(results): satisfy sidecar lint --- apps/cli/src/commands/results/serve.ts | 6 +++--- apps/cli/test/commands/results/remote-auto-export.test.ts | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 1b3557463..72e99c679 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -56,8 +56,8 @@ import { getProject, loadConfig, loadProjectRegistry, - removeProject, readGitResultArtifact, + removeProject, syncProjects, touchProject, } from '@agentv/core'; @@ -375,7 +375,7 @@ function resolveRecordArtifactPointer( kind === 'transcript' ? (record.transcript ?? record.artifacts?.transcript ?? record.artifact_pointers?.transcript) : kind === 'trace' - ? (record.artifact_pointers?.trace ?? record.artifacts?.['trace']) + ? (record.artifact_pointers?.trace ?? record.artifacts?.trace) : record.artifacts?.answer; const pointerPath = artifactPointerPath(pointer); const description = artifactPointerDescription(pointer); @@ -399,7 +399,7 @@ function resolveRecordArtifactPointer( kind === 'transcript' ? (record.transcript_path ?? record.artifacts?.transcript_path) : kind === 'trace' - ? (recordWithTrace.trace_path ?? nonEmptyString(record.artifacts?.['trace_path'])) + ? (recordWithTrace.trace_path ?? nonEmptyString(record.artifacts?.trace_path)) : (record.answer_path ?? record.artifacts?.answer_path ?? record.output_path); if (directPath) { return { path: directPath, description: directPath }; diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index 0bb735150..d1244b75b 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -235,9 +235,7 @@ describe('maybeAutoExportRunArtifacts', () => { rootDir, ), ); - expect(index.artifact_pointers.trace.key).toBe( - 'runs/default/run-002/alpha/outputs/trace.json', - ); + expect(index.artifact_pointers.trace.key).toBe('runs/default/run-002/alpha/outputs/trace.json'); expect(index.artifact_pointers.transcript.key).toBe( 'runs/default/run-002/alpha/outputs/transcript.jsonl', ); From 6c612017bd6132c7596fbda38f684d10a73b45fe Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 21 Jun 2026 14:37:08 +0200 Subject: [PATCH 4/4] chore(ci): retrigger PR checks