From 2d30b1df5d0b944e65912481ea756c1313600958 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 14:21:47 +1000 Subject: [PATCH 01/12] Support dashboard remote result branches --- Dockerfile | 7 +- .../src/commands/results/remote-metadata.ts | 28 +- apps/cli/src/commands/results/remote.ts | 60 +++- apps/cli/src/commands/results/serve.ts | 21 +- apps/cli/test/commands/results/serve.test.ts | 80 +++++ .../src/content/docs/docs/tools/dashboard.mdx | 2 + packages/core/src/evaluation/results-repo.ts | 330 +++++++++++++++++- .../core/src/evaluation/workspace/setup.ts | 11 +- .../core/test/evaluation/results-repo.test.ts | 220 ++++++++++++ .../test/evaluation/workspace/setup.test.ts | 116 ++++++ scripts/setup-dashboard-deployment.sh | 23 +- 11 files changed, 857 insertions(+), 41 deletions(-) create mode 100644 packages/core/test/evaluation/workspace/setup.test.ts diff --git a/Dockerfile b/Dockerfile index d7d1eab96..e18d6cdf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /app COPY package.json bun.lock ./ COPY packages/core/package.json packages/core/ COPY packages/sdk/package.json packages/sdk/ +COPY packages/phoenix-adapter/package.json packages/phoenix-adapter/ COPY apps/cli/package.json apps/cli/ COPY apps/dashboard/package.json apps/dashboard/ COPY apps/web/package.json apps/web/ @@ -49,8 +50,8 @@ RUN addgroup --system agentv && adduser --system --ingroup agentv agentv # Install uv + Python to a shared location (not user-specific) COPY --from=ghcr.io/astral-sh/uv:0.6.12 /uv /usr/local/bin/uv ENV UV_PYTHON_INSTALL_DIR=/opt/python -RUN uv python install 3.12 -ENV PATH="/opt/python/cpython-3.12.9-linux-aarch64-gnu/bin:${PATH}" +RUN uv python install 3.12 && ln -s /opt/python/cpython-3.12.9-linux-*-gnu /opt/python/current +ENV PATH="/opt/python/current/bin:${PATH}" # Copy pruned node_modules and built artifacts COPY --from=build /app/package.json /app/bun.lock ./ @@ -59,6 +60,8 @@ COPY --from=build /app/packages/core/dist ./packages/core/dist COPY --from=build /app/packages/core/package.json ./packages/core/ COPY --from=build /app/packages/sdk/dist ./packages/sdk/dist COPY --from=build /app/packages/sdk/package.json ./packages/sdk/ +COPY --from=build /app/packages/phoenix-adapter/dist ./packages/phoenix-adapter/dist +COPY --from=build /app/packages/phoenix-adapter/package.json ./packages/phoenix-adapter/ COPY --from=build /app/apps/cli/dist ./apps/cli/dist COPY --from=build /app/apps/cli/package.json ./apps/cli/ COPY --from=build /app/apps/cli/node_modules ./apps/cli/node_modules diff --git a/apps/cli/src/commands/results/remote-metadata.ts b/apps/cli/src/commands/results/remote-metadata.ts index 21513efae..e41287ffb 100644 --- a/apps/cli/src/commands/results/remote-metadata.ts +++ b/apps/cli/src/commands/results/remote-metadata.ts @@ -199,9 +199,12 @@ function resolveRemoteRunMetadataPaths( }; } -function readRemoteRunTagsContext(repoDir: string, manifestPath: string): RemoteRunTagsContext { +function readRemoteRunTagsContext( + repoDir: string, + manifestPath: string, + comparisonRef = resolveComparisonRef(repoDir), +): RemoteRunTagsContext { const paths = resolveRemoteRunMetadataPaths(repoDir, manifestPath); - const comparisonRef = resolveComparisonRef(repoDir); const artifactTags = readTagsFile(paths.artifactTagsPath) ?? readTagsFromGit(repoDir, comparisonRef, paths.artifactTagsGitPath); @@ -258,8 +261,12 @@ export function isResultsRepoWorktreeDirty(repoDir: string): boolean { return status !== undefined && status.trim().length > 0; } -export function readRemoteRunTags(repoDir: string, manifestPath: string): RemoteRunTagState { - const context = readRemoteRunTagsContext(repoDir, manifestPath); +export function readRemoteRunTags( + repoDir: string, + manifestPath: string, + comparisonRef?: string, +): RemoteRunTagState { + const context = readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef); return toRemoteRunTagState(context); } @@ -267,11 +274,12 @@ export function writeRemoteRunTags( repoDir: string, manifestPath: string, tags: readonly string[], + comparisonRef?: string, ): RemoteRunTagState { assertWritableResultsRepo(repoDir); const cleaned = normalizeTags(tags); - const context = readRemoteRunTagsContext(repoDir, manifestPath); + const context = readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef); const remoteTags = context.baseOverlayTags?.tags ?? context.artifactTags?.tags ?? []; if ( @@ -296,9 +304,13 @@ export function writeRemoteRunTags( }; mkdirSync(path.dirname(context.paths.overlayTagsPath), { recursive: true }); writeFileSync(context.paths.overlayTagsPath, `${JSON.stringify(entry, null, 2)}\n`, 'utf8'); - return readRemoteRunTags(repoDir, manifestPath); + return readRemoteRunTags(repoDir, manifestPath, comparisonRef); } -export function deleteRemoteRunTags(repoDir: string, manifestPath: string): RemoteRunTagState { - return writeRemoteRunTags(repoDir, manifestPath, []); +export function deleteRemoteRunTags( + repoDir: string, + manifestPath: string, + comparisonRef?: string, +): RemoteRunTagState { + return writeRemoteRunTags(repoDir, manifestPath, [], comparisonRef); } diff --git a/apps/cli/src/commands/results/remote.ts b/apps/cli/src/commands/results/remote.ts index c4617ebc3..bba92030a 100644 --- a/apps/cli/src/commands/results/remote.ts +++ b/apps/cli/src/commands/results/remote.ts @@ -118,6 +118,38 @@ export interface RemoteResultsStatus extends ResultsRepoStatus { readonly run_count: number; } +function relativeLocalRunPath(cwd: string, manifestPath: string): string | undefined { + const runsRoot = path.resolve(cwd, '.agentv', 'results', 'runs'); + const manifestDir = path.resolve(path.dirname(manifestPath)); + const relativeRunPath = path.relative(runsRoot, manifestDir); + if ( + relativeRunPath.length === 0 || + relativeRunPath.startsWith('..') || + path.isAbsolute(relativeRunPath) + ) { + return undefined; + } + return relativeRunPath.split(path.sep).join(path.posix.sep); +} + +function remoteMetadataManifestPath( + cwd: string, + config: NormalizedResultsConfig, + meta: Pick, +): string | undefined { + if (meta.source === 'remote') { + return meta.path; + } + if (!meta.on_remote) { + return undefined; + } + const relativeRunPath = relativeLocalRunPath(cwd, meta.path); + if (!relativeRunPath) { + return undefined; + } + return path.join(config.path, 'runs', ...relativeRunPath.split('/'), 'index.jsonl'); +} + export interface ResultsPublishOverrides { readonly repo?: string; readonly repo_url?: string; @@ -488,15 +520,17 @@ export async function ensureRemoteRunAvailable( export async function readRemoteRunTagState( cwd: string, - meta: Pick, + meta: Pick, projectId?: string, ): Promise { - if (meta.source !== 'remote') return undefined; + if (meta.source !== 'remote' && !meta.on_remote) return undefined; const config = await loadNormalizedResultsConfig(cwd, projectId); if (!config) return undefined; + const manifestPath = remoteMetadataManifestPath(cwd, config, meta); + if (!manifestPath) return undefined; try { - return readRemoteRunTags(config.path, meta.path); + return readRemoteRunTags(config.path, manifestPath, getResultsStorageRef(config)); } catch { return undefined; } @@ -504,35 +538,43 @@ export async function readRemoteRunTagState( export async function setRemoteRunTags( cwd: string, - meta: Pick, + meta: Pick, tags: readonly string[], projectId?: string, ): Promise { - if (meta.source !== 'remote') { + if (meta.source !== 'remote' && !meta.on_remote) { throw new Error('Remote metadata can only be set on remote runs'); } const config = await loadNormalizedResultsConfig(cwd, projectId); if (!config) { throw new Error('Writable results repo is not configured for remote metadata'); } + const manifestPath = remoteMetadataManifestPath(cwd, config, meta); + if (!manifestPath) { + throw new Error('Remote metadata can only be set on remote runs'); + } assertWritableResultsRepo(config.path); - return writeRemoteRunTags(config.path, meta.path, tags); + return writeRemoteRunTags(config.path, manifestPath, tags, getResultsStorageRef(config)); } export async function clearRemoteRunTags( cwd: string, - meta: Pick, + meta: Pick, projectId?: string, ): Promise { - if (meta.source !== 'remote') { + if (meta.source !== 'remote' && !meta.on_remote) { throw new Error('Remote metadata can only be removed from remote runs'); } const config = await loadNormalizedResultsConfig(cwd, projectId); if (!config) { throw new Error('Writable results repo is not configured for remote metadata'); } + const manifestPath = remoteMetadataManifestPath(cwd, config, meta); + if (!manifestPath) { + throw new Error('Remote metadata can only be removed from remote runs'); + } assertWritableResultsRepo(config.path); - return deleteRemoteRunTags(config.path, meta.path); + return deleteRemoteRunTags(config.path, manifestPath, getResultsStorageRef(config)); } export async function maybeAutoExportRunArtifacts( diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index bb5409107..991143b0e 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -722,6 +722,23 @@ async function readRunTagFields( meta: SourcedResultFileMeta, projectId?: string, ): Promise { + if (meta.on_remote) { + const state = await readRemoteRunTagState(searchDir, meta, projectId); + if (state) { + return { + tags: state.tags, + remote_tags: state.remoteTags, + metadata_dirty: state.dirty, + ...(state.dirty && { pending_tags: state.pendingTags ?? state.tags }), + ...materializeRunState({ + tags: state.tags, + watermark: state.oplogWatermark, + updatedAt: state.updatedAt, + }), + }; + } + } + if (meta.source === 'local') { const tagsEntry = readRunTags(meta.path); const runState = materializeRunState({ @@ -2093,7 +2110,7 @@ async function handleRunTagsPut(c: C, { searchDir, projectId }: DataContext) { return c.json({ error: 'Missing tags array' }, 400); } try { - if (meta.source === 'remote') { + if (meta.on_remote) { const state = await setRemoteRunTags(searchDir, meta, tags as string[], projectId); return c.json(remoteTagMutationResponse(state)); } @@ -2119,7 +2136,7 @@ async function handleRunTagsDelete(c: C, { searchDir, projectId }: DataContext) const meta = await findRunById(searchDir, filename, projectId); if (!meta) return c.json({ error: 'Run not found' }, 404); try { - if (meta.source === 'remote') { + if (meta.on_remote) { const state = await clearRemoteRunTags(searchDir, meta, projectId); return c.json({ ok: true, diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 95702419b..9e7be606e 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -1595,6 +1595,86 @@ describe('serve app', () => { }); }, 15000); + it('edits synced local run tags through the remote metadata overlay', async () => { + const { remoteDir, cloneDir } = initializeRemoteRepo(tempDir); + const experiment = 'green-uat'; + const timestamp = '2026-03-26T10-45-00-000Z'; + const runId = writeRemoteRunArtifact(cloneDir, experiment, timestamp, RESULT_A); + writeLocalRunArtifact(tempDir, experiment, timestamp, RESULT_A); + + mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); + writeFileSync( + path.join(tempDir, '.agentv', 'config.yaml'), + `results: + mode: github + repo: file://${remoteDir} + path: ${cloneDir} +`, + ); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const putRes = await app.request(`/api/runs/${encodeURIComponent(runId)}/tags`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ tags: ['needs-review'] }), + }); + + expect(putRes.status).toBe(200); + const putData = (await putRes.json()) as { + tags: string[]; + remote_tags: string[]; + pending_tags: string[]; + metadata_dirty: boolean; + }; + expect(putData).toMatchObject({ + tags: ['needs-review'], + remote_tags: [], + pending_tags: ['needs-review'], + metadata_dirty: true, + }); + + const localTagsPath = path.join( + tempDir, + '.agentv', + 'results', + 'runs', + experiment, + timestamp, + 'tags.json', + ); + const overlayTagsPath = path.join( + cloneDir, + 'metadata', + 'runs', + experiment, + timestamp, + 'tags.json', + ); + expect(existsSync(localTagsPath)).toBe(false); + expect(existsSync(overlayTagsPath)).toBe(true); + + const listRes = await app.request('/api/runs'); + expect(listRes.status).toBe(200); + const listData = (await listRes.json()) as { + runs: Array<{ + filename: string; + source: string; + on_remote: boolean; + tags: string[]; + pending_tags: string[]; + metadata_dirty: boolean; + }>; + }; + expect(listData.runs[0]).toMatchObject({ + filename: runId, + source: 'local', + on_remote: true, + tags: ['needs-review'], + pending_tags: ['needs-review'], + metadata_dirty: true, + }); + }, 15000); + it('computes git-native remote run list totals from materialized index rows', async () => { const { remoteDir, cloneDir } = initializeRemoteRepo(tempDir); const secondPass = { diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 72a72c359..63d5546c8 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -249,6 +249,8 @@ AGENTV_HOME=/tmp/agentv-home-a agentv dashboard --port 3117 AGENTV_HOME=/tmp/agentv-home-b agentv dashboard --port 3118 ``` +For local Docker development, prefer building from the latest AgentV checkout and running `bun` inside that image instead of depending on a globally installed or npm-installed `agentv` binary. A setup script should mount a writable `AGENTV_HOME`, project checkouts, and any results repo paths, publish the dashboard port on `0.0.0.0` when Tailscale access is needed, then start the dashboard with the local build, for example `bun apps/cli/dist/cli.js dashboard`. Keep this Docker path separate from Kubernetes or Helm deployment assets unless those assets are actually needed to model the local runtime. + The landing page shows a card for each project with run count, pass rate, and last run time. AgentV Dashboard projects dashboard showing project cards with pass rates diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index 0d98a6c01..295a8cb0c 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -734,6 +734,67 @@ async function getAheadBehindForRefs( }; } +async function readGitText( + repoDir: string, + ref: string | undefined, + gitPath: string, +): Promise { + if (!ref) { + return undefined; + } + const { stdout } = await runGit(['show', `${ref}:${gitPath}`], { + cwd: repoDir, + check: false, + }); + return stdout.length > 0 ? stdout : undefined; +} + +async function getStorageBranchWorktreeDirtyPaths( + repoDir: string, + config: NormalizedResultsConfig & { readonly branch: string }, +): Promise<{ dirtyPaths: string[]; conflictedPaths: string[] }> { + const { stdout: porcelain } = await runGit( + ['status', '--porcelain=v1', '--untracked-files=all'], + { + cwd: repoDir, + check: false, + }, + ); + const parsed = parseGitPorcelainPaths(porcelain); + const safeDirtyPaths = parsed.dirtyPaths.filter(isSafeResultsRepoPath); + const conflictedPaths = parsed.conflictedPaths.filter(isSafeResultsRepoPath); + if (safeDirtyPaths.length === 0) { + return { dirtyPaths: [], conflictedPaths }; + } + + const localRef = `refs/heads/${config.branch}`; + const remoteRef = remoteBranchRef(config.branch, config.remote); + const comparisonRef = (await gitRefExists(repoDir, localRef)) + ? localRef + : (await gitRefExists(repoDir, remoteRef)) + ? remoteRef + : undefined; + const dirtyPaths: string[] = []; + + for (const gitPath of safeDirtyPaths) { + const absolutePath = path.join(repoDir, ...gitPath.split('/')); + if (!existsSync(absolutePath)) { + if ((await readGitText(repoDir, comparisonRef, gitPath)) !== undefined) { + dirtyPaths.push(gitPath); + } + continue; + } + + const localContent = readFileSync(absolutePath, 'utf8'); + const branchContent = await readGitText(repoDir, comparisonRef, gitPath); + if (branchContent === undefined || localContent !== branchContent) { + dirtyPaths.push(gitPath); + } + } + + return { dirtyPaths: dirtyPaths.sort(), conflictedPaths }; +} + async function inspectResultsStorageBranchGit( repoDir: string, config: NormalizedResultsConfig, @@ -749,6 +810,10 @@ async function inspectResultsStorageBranchGit( const upstream = remoteBranchRef(config.branch, config.remote); const localExists = await gitRefExists(repoDir, localRef); const remoteExists = await gitRefExists(repoDir, upstream); + const worktree = await getStorageBranchWorktreeDirtyPaths(repoDir, { + ...config, + branch: config.branch, + }); const { ahead = 0, behind = 0 } = localExists && remoteExists ? await getAheadBehindForRefs(repoDir, localRef, upstream) @@ -758,7 +823,11 @@ async function inspectResultsStorageBranchGit( }; let syncStatus: ResultsRepoSyncStatus = 'clean'; - if (ahead > 0 && behind > 0) { + if (worktree.conflictedPaths.length > 0) { + syncStatus = 'conflicted'; + } else if (worktree.dirtyPaths.length > 0) { + syncStatus = 'dirty'; + } else if (ahead > 0 && behind > 0) { syncStatus = 'diverged'; } else if (behind > 0) { syncStatus = 'behind'; @@ -772,11 +841,33 @@ async function inspectResultsStorageBranchGit( ...(remoteExists && { upstream }), ahead, behind, - dirtyPaths: [], - conflictedPaths: [], + dirtyPaths: worktree.dirtyPaths, + conflictedPaths: worktree.conflictedPaths, }; } +async function fastForwardStorageBranchRef( + repoDir: string, + normalized: StorageBranchResultsConfig, + upstream: string, +): Promise { + await assertValidResultsBranchName(repoDir, normalized.branch); + await ensureResultsBranchNotCheckedOut(repoDir, normalized); + + const localRef = `refs/heads/${normalized.branch}`; + const localExists = await gitRefExists(repoDir, localRef); + const updateArgs = ['update-ref', localRef, upstream]; + if (localExists) { + const { stdout: localSha } = await runGit(['rev-parse', localRef], { cwd: repoDir }); + updateArgs.push(localSha.trim()); + } + await runGit(updateArgs, { cwd: repoDir }); + await runGit(['branch', '--set-upstream-to', upstream, normalized.branch], { + cwd: repoDir, + check: false, + }); +} + async function hasInProgressGitConflict(repoDir: string): Promise { const markers = ['MERGE_HEAD', 'CHERRY_PICK_HEAD', 'REVERT_HEAD', 'REBASE_HEAD']; for (const marker of markers) { @@ -1105,6 +1196,116 @@ export async function syncResultsRepoForProject(config: ResultsConfig): Promise< throw error; } } + let inspection = await inspectResultsStorageBranchGit(repoDir, normalized); + const fastForwardBehindStorageBranch = async (): Promise => { + if ((inspection.behind ?? 0) <= 0 || (inspection.ahead ?? 0) > 0) { + return undefined; + } + + if (!inspection.upstream) { + const status = withGitInspection(getResultsRepoStatus(normalized), inspection); + updateStatusFile(normalized, { + last_error: 'Results repo has no upstream branch to pull from', + }); + return withBlockedStatus(status, 'Results repo has no upstream branch to pull from', { + pullPerformed, + pushPerformed, + commitCreated, + }); + } + + if (!normalized.branch) { + return undefined; + } + + try { + await fastForwardStorageBranchRef( + repoDir, + { ...normalized, branch: normalized.branch }, + inspection.upstream, + ); + pullPerformed = true; + inspection = await inspectResultsStorageBranchGit(repoDir, normalized); + return undefined; + } catch (error) { + inspection = await inspectResultsStorageBranchGit(repoDir, normalized); + const status = withGitInspection(getResultsRepoStatus(normalized), inspection); + const reason = `Results repo could not be fast-forwarded: ${getStatusMessage(error)}`; + updateStatusFile(normalized, { last_error: reason }); + return withBlockedStatus(status, reason, { + pullPerformed, + pushPerformed, + commitCreated, + }); + } + }; + + if (inspection.syncStatus === 'conflicted') { + const status = withGitInspection(getResultsRepoStatus(normalized), inspection); + updateStatusFile(normalized, { + last_error: 'Results repo has unresolved git conflicts', + }); + return withBlockedStatus(status, 'Results repo has unresolved git conflicts', { + pullPerformed, + pushPerformed, + commitCreated, + }); + } + + if ((inspection.ahead ?? 0) > 0 && (inspection.behind ?? 0) > 0) { + const status = withGitInspection(getResultsRepoStatus(normalized), inspection); + updateStatusFile(normalized, { + last_error: 'Results repo local and remote histories have diverged', + }); + return withBlockedStatus(status, 'Results repo local and remote histories have diverged', { + pullPerformed, + pushPerformed, + commitCreated, + }); + } + + if (inspection.syncStatus === 'dirty') { + if (!normalized.auto_push && !normalized.require_push) { + const status = withGitInspection(getResultsRepoStatus(normalized), inspection); + updateStatusFile(normalized, { + last_error: 'Results repo has uncommitted changes and auto_push is disabled', + }); + return withBlockedStatus( + status, + 'Results repo has uncommitted changes and auto_push is disabled', + { + pullPerformed, + pushPerformed, + commitCreated, + }, + ); + } + + if ((inspection.behind ?? 0) > 0) { + const blockedStatus = await fastForwardBehindStorageBranch(); + if (blockedStatus) { + return blockedStatus; + } + } + + if (normalized.branch) { + commitCreated = await commitStorageBranchWorktreePaths({ + normalized: { ...normalized, branch: normalized.branch }, + repoDir, + paths: inspection.dirtyPaths, + commitMessage: 'chore(results): sync local result metadata', + }); + inspection = await inspectResultsStorageBranchGit(repoDir, normalized); + } + } + + if ((inspection.behind ?? 0) > 0 && (inspection.ahead ?? 0) === 0) { + const blockedStatus = await fastForwardBehindStorageBranch(); + if (blockedStatus) { + return blockedStatus; + } + } + if (normalized.auto_push || normalized.require_push) { if (normalized.branch) { const localRef = `refs/heads/${normalized.branch}`; @@ -1120,12 +1321,18 @@ export async function syncResultsRepoForProject(config: ResultsConfig): Promise< { cwd: repoDir }, ); pushPerformed = true; + await fetchResultsRepo(repoDir, normalized.remote, normalized.branch).catch( + () => undefined, + ); } catch (error) { updateStatusFile(normalized, { last_error: getStatusMessage(error) }); if (normalized.require_push) { throw error; } - const status = await getResultsRepoSyncStatus(normalized); + const status = withGitInspection( + getResultsRepoStatus(normalized), + await inspectResultsStorageBranchGit(repoDir, normalized), + ); return withBlockedStatus( status, `Results repo push was rejected: ${getStatusMessage(error)}`, @@ -1143,11 +1350,11 @@ export async function syncResultsRepoForProject(config: ResultsConfig): Promise< last_synced_at: new Date().toISOString(), last_error: undefined, }); - return withActionFlags(await getResultsRepoSyncStatus(normalized), { - pullPerformed, - pushPerformed, - commitCreated, - }); + const status = withGitInspection( + getResultsRepoStatus(normalized), + await inspectResultsStorageBranchGit(repoDir, normalized), + ); + return withActionFlags(status, { pullPerformed, pushPerformed, commitCreated }); } await fetchResultsRepo(repoDir, normalized.remote, normalized.branch); await checkoutConfiguredResultsBranch(repoDir, normalized); @@ -1846,6 +2053,111 @@ async function ensureResultsBranchNotCheckedOut( await runGit(['checkout', '--detach'], { cwd: repoDir }); } +async function commitStorageBranchWorktreePaths(params: { + readonly normalized: StorageBranchResultsConfig; + readonly repoDir: string; + readonly paths: readonly string[]; + readonly commitMessage: string; +}): Promise { + const { normalized } = params; + const paths = [...new Set(params.paths.filter(isSafeResultsRepoPath))].sort(); + if (paths.length === 0) { + return false; + } + + await assertValidResultsBranchName(params.repoDir, normalized.branch); + await ensureResultsBranchNotCheckedOut(params.repoDir, normalized); + + let base = await resolveStorageBranchBase({ + repoDir: params.repoDir, + normalized, + }); + if (!base.baseRef) { + await createOrphanResultsBranch(params.repoDir, normalized.branch); + base = await resolveStorageBranchBase({ + repoDir: params.repoDir, + normalized, + }); + } + + const indexRoot = await mkdtemp(path.join(os.tmpdir(), 'agentv-results-index-')); + const indexFile = path.join(indexRoot, 'index'); + const indexEnv = { GIT_INDEX_FILE: indexFile }; + + try { + if (base.baseRef) { + await runGit(['read-tree', base.baseRef], { cwd: params.repoDir, env: indexEnv }); + } else { + await runGit(['read-tree', '--empty'], { cwd: params.repoDir, env: indexEnv }); + } + + for (const gitPath of paths) { + const sourcePath = path.join(params.repoDir, ...gitPath.split('/')); + if (!existsSync(sourcePath)) { + await runGit(['update-index', '--force-remove', '--', gitPath], { + cwd: params.repoDir, + env: indexEnv, + check: false, + }); + continue; + } + + const fileStat = await lstat(sourcePath); + let mode = fileStat.mode & 0o111 ? '100755' : '100644'; + if (fileStat.isSymbolicLink()) { + mode = '120000'; + } + const { stdout: blob } = await runGit(['hash-object', '-w', '--no-filters', sourcePath], { + cwd: params.repoDir, + }); + await runGit(['update-index', '--add', '--cacheinfo', `${mode},${blob.trim()},${gitPath}`], { + cwd: params.repoDir, + env: indexEnv, + }); + } + + const { stdout: newTreeStdout } = await runGit(['write-tree'], { + cwd: params.repoDir, + env: indexEnv, + }); + const newTree = newTreeStdout.trim(); + if (base.baseTree && newTree === base.baseTree) { + return false; + } + + await ensureResultsRepoCommitIdentity(params.repoDir); + const commitArgs = [ + 'commit-tree', + newTree, + ...(base.baseCommit ? ['-p', base.baseCommit] : []), + '-m', + params.commitMessage, + ]; + const { stdout: commitStdout } = await runGit(commitArgs, { cwd: params.repoDir }); + const commitSha = commitStdout.trim(); + await runGit( + [ + 'update-ref', + `refs/heads/${normalized.branch}`, + commitSha, + ...(base.localExists ? [base.baseCommit ?? ''] : []), + ].filter(Boolean), + { cwd: params.repoDir }, + ); + + if (base.remoteExists) { + await runGit(['branch', '--set-upstream-to', base.remoteRef, normalized.branch], { + cwd: params.repoDir, + check: false, + }); + } + + return true; + } finally { + await rm(indexRoot, { recursive: true, force: true }).catch(() => undefined); + } +} + async function commitResultsRunWithTemporaryIndex(params: { readonly normalized: StorageBranchResultsConfig; readonly repoDir: string; diff --git a/packages/core/src/evaluation/workspace/setup.ts b/packages/core/src/evaluation/workspace/setup.ts index 4a2dbced4..bc4a57fed 100644 --- a/packages/core/src/evaluation/workspace/setup.ts +++ b/packages/core/src/evaluation/workspace/setup.ts @@ -396,6 +396,7 @@ export async function prepareSharedWorkspaceSetup( setupLog(`acquiring ${slotsNeeded} workspace pool slot(s) (pool capacity: ${poolMaxSlots})`); poolManager = new WorkspacePoolManager(getWorkspacePoolRoot()); const poolRepoManager = new RepoManager(verbose); + repoManager = poolRepoManager; for (let i = 0; i < slotsNeeded; i++) { const slot = await poolManager.acquireWorkspace({ @@ -457,9 +458,15 @@ export async function prepareSharedWorkspaceSetup( const needsPerRepoCheck = hasReposToMaterialize && useStaticWorkspace && !staticMaterialised && isYamlConfiguredPath; repoManager = - needsRepoMaterialisation || needsPerRepoCheck ? new RepoManager(verbose) : undefined; + repoManager ?? + (needsRepoMaterialisation || needsPerRepoCheck ? new RepoManager(verbose) : undefined); - if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos) { + if ( + (needsRepoMaterialisation || needsPerRepoCheck) && + repoManager && + sharedWorkspacePath && + suiteWorkspace?.repos + ) { try { if (needsPerRepoCheck) { for (const repo of suiteWorkspace.repos) { diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index 7d09e835f..70e958fe0 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -688,6 +688,226 @@ describe('results repo write path', () => { ); }, 20000); + it('commits repo_path metadata overlays to the configured storage branch during sync', async () => { + const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); + const storageBranch = initializeRemoteStorageBranch(seedDir, DEFAULT_RESULTS_BRANCH); + const projectDir = path.join(rootDir, 'source-project-metadata-sync'); + git(`git clone --quiet "${remoteDir}" "${projectDir}"`, rootDir); + git('git config user.email "test@example.com"', projectDir); + git('git config user.name "Test User"', projectDir); + git(`git fetch --quiet origin ${storageBranch}`, projectDir); + + const tagPath = path.join( + projectDir, + 'metadata', + 'runs', + 'default', + '2026-06-22T00-19-03-060Z', + 'tags.json', + ); + mkdirSync(path.dirname(tagPath), { recursive: true }); + writeFileSync( + tagPath, + `${JSON.stringify({ tags: ['dogfood'], updated_at: '2026-06-22T00:00:00.000Z' }, null, 2)}\n`, + ); + + const config: ResultsConfig = { + repo_path: projectDir, + branch: storageBranch, + remote: 'origin', + sync: { auto_push: true }, + }; + + await expect(getResultsRepoSyncStatus(config)).resolves.toMatchObject({ + sync_status: 'dirty', + dirty_paths: ['metadata/runs/default/2026-06-22T00-19-03-060Z/tags.json'], + }); + + const status = await syncResultsRepoForProject(config); + + expect(status).toMatchObject({ + sync_status: 'clean', + commit_created: true, + push_performed: true, + blocked: false, + branch: storageBranch, + upstream: `origin/${storageBranch}`, + }); + expect( + git(`git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, rootDir), + ).toContain('metadata/runs/default/2026-06-22T00-19-03-060Z/tags.json'); + await expect(getResultsRepoSyncStatus(config)).resolves.toMatchObject({ + sync_status: 'clean', + dirty_paths: [], + }); + expect(git('git branch --show-current', projectDir)).toBe('main'); + }, 20000); + + it('fast-forwards a clean repo_path storage branch during project sync', async () => { + const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); + const storageBranch = initializeRemoteStorageBranch(seedDir, DEFAULT_RESULTS_BRANCH); + const projectDir = path.join(rootDir, 'source-project-repo-path-ff'); + git(`git clone --quiet "${remoteDir}" "${projectDir}"`, rootDir); + git( + `git fetch --quiet origin refs/heads/${storageBranch}:refs/heads/${storageBranch}`, + projectDir, + ); + + git(`git switch --quiet ${storageBranch}`, seedDir); + const remoteTagPath = path.join( + seedDir, + 'metadata', + 'runs', + 'remote-only', + '2026-06-22T00-00-00-000Z', + 'tags.json', + ); + mkdirSync(path.dirname(remoteTagPath), { recursive: true }); + writeFileSync(remoteTagPath, `${JSON.stringify({ tags: ['remote'] }, null, 2)}\n`); + git('git add metadata && git commit --quiet -m "remote tag metadata"', seedDir); + git(`git push --quiet origin HEAD:${storageBranch}`, seedDir); + + const config: ResultsConfig = { + repo_path: projectDir, + branch: storageBranch, + remote: 'origin', + sync: { auto_push: false }, + }; + + const status = await syncResultsRepoForProject(config); + + expect(status).toMatchObject({ + sync_status: 'clean', + pull_performed: true, + push_performed: false, + commit_created: false, + blocked: false, + branch: storageBranch, + upstream: `origin/${storageBranch}`, + }); + expect(git(`git rev-parse ${storageBranch}`, projectDir)).toBe( + git(`git --git-dir "${remoteDir}" rev-parse ${storageBranch}`, rootDir), + ); + expect(git('git branch --show-current', projectDir)).toBe('main'); + }, 20000); + + it('fast-forwards repo_path metadata overlays before committing and pushing them', async () => { + const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); + const storageBranch = initializeRemoteStorageBranch(seedDir, DEFAULT_RESULTS_BRANCH); + const projectDir = path.join(rootDir, 'source-project-repo-path-dirty-ff'); + git(`git clone --quiet "${remoteDir}" "${projectDir}"`, rootDir); + git('git config user.email "test@example.com"', projectDir); + git('git config user.name "Test User"', projectDir); + git( + `git fetch --quiet origin refs/heads/${storageBranch}:refs/heads/${storageBranch}`, + projectDir, + ); + + git(`git switch --quiet ${storageBranch}`, seedDir); + const remoteTagPath = path.join( + seedDir, + 'metadata', + 'runs', + 'remote-only', + '2026-06-22T00-00-00-000Z', + 'tags.json', + ); + mkdirSync(path.dirname(remoteTagPath), { recursive: true }); + writeFileSync(remoteTagPath, `${JSON.stringify({ tags: ['remote'] }, null, 2)}\n`); + git('git add metadata && git commit --quiet -m "remote tag metadata"', seedDir); + git(`git push --quiet origin HEAD:${storageBranch}`, seedDir); + + const localTagPath = path.join( + projectDir, + 'metadata', + 'runs', + 'local', + '2026-06-22T01-00-00-000Z', + 'tags.json', + ); + mkdirSync(path.dirname(localTagPath), { recursive: true }); + writeFileSync(localTagPath, `${JSON.stringify({ tags: ['local'] }, null, 2)}\n`); + + const config: ResultsConfig = { + repo_path: projectDir, + branch: storageBranch, + remote: 'origin', + sync: { auto_push: true }, + }; + + const status = await syncResultsRepoForProject(config); + + expect(status).toMatchObject({ + sync_status: 'clean', + pull_performed: true, + push_performed: true, + commit_created: true, + blocked: false, + branch: storageBranch, + upstream: `origin/${storageBranch}`, + }); + const remoteFiles = git( + `git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, + rootDir, + ); + expect(remoteFiles).toContain('metadata/runs/remote-only/2026-06-22T00-00-00-000Z/tags.json'); + expect(remoteFiles).toContain('metadata/runs/local/2026-06-22T01-00-00-000Z/tags.json'); + expect(git('git branch --show-current', projectDir)).toBe('main'); + }, 20000); + + it('reports repo_path metadata push rejection without dropping the local commit', async () => { + const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); + const storageBranch = initializeRemoteStorageBranch(seedDir, DEFAULT_RESULTS_BRANCH); + const projectDir = path.join(rootDir, 'source-project-repo-path-rejected-push'); + git(`git clone --quiet "${remoteDir}" "${projectDir}"`, rootDir); + git('git config user.email "test@example.com"', projectDir); + git('git config user.name "Test User"', projectDir); + git( + `git fetch --quiet origin refs/heads/${storageBranch}:refs/heads/${storageBranch}`, + projectDir, + ); + + const tagPath = path.join( + projectDir, + 'metadata', + 'runs', + 'local', + '2026-06-22T02-00-00-000Z', + 'tags.json', + ); + mkdirSync(path.dirname(tagPath), { recursive: true }); + writeFileSync(tagPath, `${JSON.stringify({ tags: ['local'] }, null, 2)}\n`); + + const hookPath = path.join(remoteDir, 'hooks', 'pre-receive'); + writeFileSync(hookPath, '#!/usr/bin/env sh\necho "reject metadata push" >&2\nexit 1\n'); + chmodSync(hookPath, 0o755); + + const status = await syncResultsRepoForProject({ + repo_path: projectDir, + branch: storageBranch, + remote: 'origin', + sync: { auto_push: true }, + }); + + expect(status).toMatchObject({ + sync_status: 'ahead', + pull_performed: false, + push_performed: false, + commit_created: true, + blocked: true, + branch: storageBranch, + upstream: `origin/${storageBranch}`, + }); + expect(status.block_reason).toContain('Results repo push was rejected'); + expect(git(`git ls-tree -r --name-only ${storageBranch}`, projectDir)).toContain( + 'metadata/runs/local/2026-06-22T02-00-00-000Z/tags.json', + ); + expect( + git(`git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, rootDir), + ).not.toContain('metadata/runs/local/2026-06-22T02-00-00-000Z/tags.json'); + expect(git('git branch --show-current', projectDir)).toBe('main'); + }, 20000); + it('publishes to an explicit external local repo path', async () => { const projectDir = path.join(rootDir, 'project'); const resultsRepoDir = path.join(rootDir, 'local-results-repo'); diff --git a/packages/core/test/evaluation/workspace/setup.test.ts b/packages/core/test/evaluation/workspace/setup.test.ts new file mode 100644 index 000000000..b5f61790e --- /dev/null +++ b/packages/core/test/evaluation/workspace/setup.test.ts @@ -0,0 +1,116 @@ +import { execSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import type { EvalTest } from '../../../src/evaluation/types.js'; +import { + type SharedWorkspaceSetup, + prepareSharedWorkspaceSetup, + releaseSharedWorkspaceSetup, +} from '../../../src/evaluation/workspace/setup.js'; + +function cleanGitEnv(): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined && !(key.startsWith('GIT_') && key !== 'GIT_SSH_COMMAND')) { + env[key] = value; + } + } + return env; +} + +const EXEC_OPTS = { stdio: 'ignore' as const, env: cleanGitEnv() }; + +function createTestRepo(dir: string, files: Record): string { + mkdirSync(dir, { recursive: true }); + execSync('git init', { cwd: dir, ...EXEC_OPTS }); + execSync('git config user.email "test@test.com"', { cwd: dir, ...EXEC_OPTS }); + execSync('git config user.name "Test"', { cwd: dir, ...EXEC_OPTS }); + for (const [name, content] of Object.entries(files)) { + const filePath = path.join(dir, name); + mkdirSync(path.dirname(filePath), { recursive: true }); + writeFileSync(filePath, content); + } + execSync('git add -A && git commit -m "initial"', { cwd: dir, ...EXEC_OPTS }); + return execSync('git rev-parse HEAD', { cwd: dir, env: cleanGitEnv() }).toString().trim(); +} + +describe('prepareSharedWorkspaceSetup', () => { + let tmpDir: string; + let savedAgentvHome: string | undefined; + let savedAgentvDataDir: string | undefined; + let setup: SharedWorkspaceSetup | undefined; + + beforeEach(async () => { + tmpDir = await mkdtemp(path.join(os.tmpdir(), 'agentv-workspace-setup-')); + savedAgentvHome = process.env.AGENTV_HOME; + savedAgentvDataDir = process.env.AGENTV_DATA_DIR; + process.env.AGENTV_HOME = path.join(tmpDir, 'agentv-home'); + process.env.AGENTV_DATA_DIR = path.join(tmpDir, 'agentv-data'); + }); + + afterEach(async () => { + if (setup) { + await releaseSharedWorkspaceSetup(setup); + setup = undefined; + } + if (savedAgentvHome === undefined) { + process.env.AGENTV_HOME = undefined; + } else { + process.env.AGENTV_HOME = savedAgentvHome; + } + if (savedAgentvDataDir === undefined) { + process.env.AGENTV_DATA_DIR = undefined; + } else { + process.env.AGENTV_DATA_DIR = savedAgentvDataDir; + } + await rm(tmpDir, { recursive: true, force: true }); + }); + + it('retains repo manager for pooled repo workspaces so reset hooks can reset nested repos', async () => { + const sourceRepo = path.join(tmpDir, 'source-repo'); + const cleanCommit = createTestRepo(sourceRepo, { 'tracked.txt': 'clean\n' }); + const repos = [ + { + path: './repo-a', + repo: `file://${sourceRepo}`, + commit: cleanCommit, + }, + ]; + const evalCase: EvalTest = { + id: 'case-1', + question: 'test', + criteria: 'ok', + workspace: { + mode: 'pooled', + hooks: { after_each: { reset: 'fast' } }, + repos, + }, + }; + + setup = await prepareSharedWorkspaceSetup({ + evalRunId: 'test-pooled-repo-reset', + evalCases: [evalCase], + evalDir: tmpDir, + workers: 1, + }); + + expect(setup.repoManager).toBeDefined(); + expect(setup.sharedWorkspacePath).toBeDefined(); + if (!setup.repoManager || !setup.sharedWorkspacePath) { + throw new Error('Expected pooled setup to include repo manager and workspace path'); + } + + const repoDir = path.join(setup.sharedWorkspacePath, 'repo-a'); + await writeFile(path.join(repoDir, 'tracked.txt'), 'dirty\n'); + await writeFile(path.join(repoDir, 'stale.txt'), 'stale\n'); + + await setup.repoManager.reset(repos, setup.sharedWorkspacePath, 'fast'); + + expect(readFileSync(path.join(repoDir, 'tracked.txt'), 'utf8')).toBe('clean\n'); + expect(existsSync(path.join(repoDir, 'stale.txt'))).toBe(false); + }, 30_000); +}); diff --git a/scripts/setup-dashboard-deployment.sh b/scripts/setup-dashboard-deployment.sh index bf044f5cf..9392cc64b 100755 --- a/scripts/setup-dashboard-deployment.sh +++ b/scripts/setup-dashboard-deployment.sh @@ -115,6 +115,8 @@ EOF write_project_config() { mkdir -p "$project_dir/.agentv/results/runs" + local results_url + results_url="$(repo_to_url "$results_repo")" if [[ -f "$source_dir/.agentv/targets.yaml" ]]; then cp "$source_dir/.agentv/targets.yaml" "$project_dir/.agentv/targets.yaml" fi @@ -127,11 +129,12 @@ eval_patterns: - "examples/**/dataset*.yaml" results: - mode: github - repo: $results_repo + repo_url: $results_url + branch: agentv/results/v1 path: /data/results/agentv-evalresults - auto_push: false - branch_prefix: eval-results + sync: + auto_push: false + require_push: false dashboard: project_dashboard: true @@ -161,11 +164,13 @@ const nextEntry = { name: "AgentV Examples", path: "/data/projects/agentv-examples", results: { - mode: "github", - repo: resultsRepo, + repo_url: resultsRepo, + branch: "agentv/results/v1", path: "/data/results/agentv-evalresults", - auto_push: false, - branch_prefix: "eval-results", + sync: { + auto_push: false, + require_push: false, + }, }, added_at: existingEntry && typeof existingEntry.added_at === "string" ? existingEntry.added_at : now, @@ -175,7 +180,7 @@ config.projects = [...projects.filter((project) => { return !(project && typeof project === "object" && project.id === "agentv-examples"); }), nextEntry]; writeFileSync(configPath, stringify(config), "utf8"); -' "$home_dir/config.yaml" "$now" "$results_repo" +' "$home_dir/config.yaml" "$now" "$(repo_to_url "$results_repo")" } stage_examples_project() { From c1a37bd81753cb5b20787886b046c2e92274b5e3 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 16:10:58 +1000 Subject: [PATCH 02/12] feat(dashboard): unify sidecar artifact tree Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apps/cli/src/commands/results/serve.ts | 522 ++++++++++++++----- apps/cli/test/commands/results/serve.test.ts | 33 +- apps/dashboard/src/components/FileTree.tsx | 27 + apps/dashboard/src/lib/types.ts | 6 + 4 files changed, 468 insertions(+), 120 deletions(-) diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 991143b0e..e69359011 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -245,9 +245,29 @@ interface FileNode { name: string; path: string; type: 'file' | 'dir'; + kind?: string; + storage?: string; + ref?: string; + key?: string; + sha256?: string; + media_type?: string; children?: FileNode[]; } +type ArtifactCatalogStorage = 'local' | 'git'; + +interface ArtifactCatalogEntry { + readonly displayPath: string; + readonly kind: 'transcript' | 'trace' | 'answer' | 'artifact'; + readonly storage: ArtifactCatalogStorage; + readonly ref?: string; + readonly key?: string; + readonly path?: string; + readonly sha256?: string; + readonly mediaType?: string; + readonly objectVersion?: string; +} + function buildFileTree(dirPath: string, relativeTo: string): FileNode[] { if (!existsSync(dirPath) || !statSync(dirPath).isDirectory()) { return []; @@ -269,10 +289,75 @@ function buildFileTree(dirPath: string, relativeTo: string): FileNode[] { children: buildFileTree(fullPath, relativeTo), }; } - return { name: entry.name, path: relPath, type: 'file' as const }; + return { + name: entry.name, + path: relPath.split(path.sep).join('/'), + type: 'file' as const, + storage: 'local', + }; }); } +function flattenFileTree(nodes: readonly FileNode[]): FileNode[] { + const files: FileNode[] = []; + for (const node of nodes) { + if (node.type === 'file') { + files.push(node); + } else if (node.children) { + files.push(...flattenFileTree(node.children)); + } + } + return files; +} + +function ensureCatalogFileNode(root: FileNode[], entry: ArtifactCatalogEntry): void { + const segments = entry.displayPath.split('/').filter(Boolean); + if (segments.length === 0) return; + + let siblings = root; + let currentPath = ''; + for (let i = 0; i < segments.length; i += 1) { + const segment = segments[i]; + if (!segment) continue; + currentPath = currentPath ? `${currentPath}/${segment}` : segment; + const isFile = i === segments.length - 1; + let node = siblings.find((candidate) => candidate.name === segment); + if (!node) { + node = { + name: segment, + path: currentPath, + type: isFile ? 'file' : 'dir', + ...(isFile + ? { + kind: entry.kind, + storage: entry.storage, + ...(entry.ref && { ref: entry.ref }), + ...(entry.key && { key: entry.key }), + ...(entry.sha256 && { sha256: entry.sha256 }), + ...(entry.mediaType && { media_type: entry.mediaType }), + } + : { children: [] }), + }; + siblings.push(node); + siblings.sort((a, b) => { + if (a.type !== b.type) return a.type === 'dir' ? -1 : 1; + return a.name.localeCompare(b.name); + }); + } + if (isFile) { + node.kind = entry.kind; + node.storage = entry.storage; + if (entry.ref) node.ref = entry.ref; + if (entry.key) node.key = entry.key; + if (entry.sha256) node.sha256 = entry.sha256; + if (entry.mediaType) node.media_type = entry.mediaType; + return; + } + node.children ??= []; + siblings = node.children; + } +} + function inferLanguage(filePath: string): string { const ext = path.extname(filePath).toLowerCase(); const langMap: Record = { @@ -362,6 +447,10 @@ function artifactPointerSha256(pointer: unknown): string | undefined { return isRecord(pointer) ? nonEmptyString(pointer.sha256) : undefined; } +function artifactPointerMediaType(pointer: unknown): string | undefined { + return isRecord(pointer) ? nonEmptyString(pointer.media_type) : undefined; +} + function artifactPointerObjectVersion(pointer: unknown): string | undefined { return isRecord(pointer) ? nonEmptyString(pointer.object_version) : undefined; } @@ -372,6 +461,7 @@ interface ResolvedArtifactPointer { readonly description?: string; readonly ref?: string; readonly sha256?: string; + readonly mediaType?: string; readonly objectVersion?: string; readonly unsupportedReason?: string; } @@ -391,6 +481,7 @@ function resolveRecordArtifactPointer( const ref = artifactPointerRef(pointer); const key = artifactPointerKey(pointer); const sha256 = artifactPointerSha256(pointer); + const mediaType = artifactPointerMediaType(pointer); const objectVersion = artifactPointerObjectVersion(pointer); if ((kind === 'transcript' || kind === 'trace') && pointerPath) { return { @@ -399,6 +490,7 @@ function resolveRecordArtifactPointer( ref, ...(key && { key }), ...(sha256 && { sha256 }), + ...(mediaType && { mediaType }), ...(objectVersion && { objectVersion }), }; } @@ -421,6 +513,7 @@ function resolveRecordArtifactPointer( ref, ...(key && { key }), ...(sha256 && { sha256 }), + ...(mediaType && { mediaType }), ...(objectVersion && { objectVersion }), }; } @@ -430,6 +523,7 @@ function resolveRecordArtifactPointer( ref, ...(key && { key }), ...(sha256 && { sha256 }), + ...(mediaType && { mediaType }), ...(objectVersion && { objectVersion }), unsupportedReason: description ? `${kind} artifact pointer does not include a local path (${description}).` @@ -489,16 +583,6 @@ function resolveReadableRunArtifactFile( return { absolutePath: realArtifact }; } -function readOptionalRunArtifactText( - baseDir: string, - artifact: ResolvedArtifactPointer, -): string | undefined { - if (!artifact.path) return undefined; - const resolved = resolveReadableRunArtifactFile(baseDir, artifact.path); - if (!resolved.absolutePath) return undefined; - return readFileSync(resolved.absolutePath, 'utf8'); -} - function normalizeArtifactRelativePath(relativePath: string): string | undefined { const normalized = relativePath.split(path.sep).join('/'); const segments = normalized.split('/').filter(Boolean); @@ -512,6 +596,20 @@ function normalizeArtifactRelativePath(relativePath: string): string | undefined return segments.join('/'); } +function relativeRunPathFromNormalizedManifestPath(manifestPath: string): string | undefined { + const parts = manifestPath.split('/').filter(Boolean); + const runsIndex = parts.lastIndexOf('runs'); + if (runsIndex === -1 || parts.at(-1) !== 'index.jsonl') { + return undefined; + } + const runParts = parts.slice(runsIndex + 1, -1); + return runParts.length > 0 ? runParts.join('/') : undefined; +} + +function relativeRunPathFromManifestPath(manifestPath: string): string | undefined { + return relativeRunPathFromNormalizedManifestPath(manifestPath.split(path.sep).join('/')); +} + function relativeRunPathFromManifest(repoDir: string, manifestPath: string): string | undefined { const relativeManifestPath = path.relative(repoDir, manifestPath).split(path.sep).join('/'); if ( @@ -522,13 +620,7 @@ function relativeRunPathFromManifest(repoDir: string, manifestPath: string): str return undefined; } - const parts = relativeManifestPath.split('/').filter(Boolean); - const runsIndex = parts.lastIndexOf('runs'); - if (runsIndex === -1 || parts.at(-1) !== 'index.jsonl') { - return undefined; - } - const runParts = parts.slice(runsIndex + 1, -1); - return runParts.length > 0 ? runParts.join('/') : undefined; + return relativeRunPathFromNormalizedManifestPath(relativeManifestPath); } function sidecarArtifactKeyForPointer( @@ -541,7 +633,7 @@ function sidecarArtifactKeyForPointer( return publishedKey; } if (!artifact.path) { - return undefined; + return publishedKey; } const relativeArtifactPath = normalizeArtifactRelativePath(artifact.path); const relativeRunPath = relativeRunPathFromManifest(repoDir, manifestPath); @@ -579,6 +671,193 @@ async function readSidecarArtifactText( return bytes?.toString('utf8'); } +function addDirectArtifactCatalogEntry( + entries: ArtifactCatalogEntry[], + seen: Set, + displayPath: string | undefined, + kind: ArtifactCatalogEntry['kind'], +): void { + const normalized = displayPath ? normalizeArtifactRelativePath(displayPath) : undefined; + if (!normalized || seen.has(normalized)) return; + seen.add(normalized); + entries.push({ + displayPath: normalized, + kind, + storage: 'local', + path: normalized, + }); +} + +function addPointerArtifactCatalogEntry( + entries: ArtifactCatalogEntry[], + seen: Set, + artifact: ResolvedArtifactPointer, + kind: ArtifactCatalogEntry['kind'], + runPath?: string, +): void { + const artifactPath = artifact.path ? normalizeArtifactRelativePath(artifact.path) : undefined; + const displayPath = artifactPath ?? displayPathFromArtifactKey(artifact.key, runPath); + if (!displayPath || seen.has(displayPath)) return; + seen.add(displayPath); + entries.push({ + displayPath, + kind, + storage: artifact.ref ? 'git' : 'local', + ...(artifactPath && { path: artifactPath }), + ...(artifact.ref && { ref: artifact.ref }), + ...(artifact.key && { key: artifact.key }), + ...(artifact.sha256 && { sha256: artifact.sha256 }), + ...(artifact.mediaType && { mediaType: artifact.mediaType }), + ...(artifact.objectVersion && { objectVersion: artifact.objectVersion }), + }); +} + +function displayPathFromArtifactKey(key: string | undefined, runPath: string | undefined) { + const normalizedKey = key ? normalizeArtifactRelativePath(key) : undefined; + if (!normalizedKey) return undefined; + if (!runPath) return normalizedKey; + const runPrefix = `runs/${runPath}/`; + if (!normalizedKey.startsWith(runPrefix)) return normalizedKey; + return normalizeArtifactRelativePath(normalizedKey.slice(runPrefix.length)) ?? normalizedKey; +} + +function buildResultArtifactCatalog( + record: ResultManifestRecord, + options?: { readonly runPath?: string }, +): ArtifactCatalogEntry[] { + const entries: ArtifactCatalogEntry[] = []; + const seen = new Set(); + const transcript = resolveRecordArtifactPointer(record, 'transcript'); + const trace = resolveRecordArtifactPointer(record, 'trace'); + const answer = resolveRecordArtifactPointer(record, 'answer'); + const recordWithTrace = record as ResultManifestRecord & { readonly trace_path?: string }; + + addPointerArtifactCatalogEntry(entries, seen, transcript, 'transcript', options?.runPath); + addPointerArtifactCatalogEntry(entries, seen, trace, 'trace', options?.runPath); + addPointerArtifactCatalogEntry(entries, seen, answer, 'answer', options?.runPath); + + addDirectArtifactCatalogEntry(entries, seen, record.grading_path, 'artifact'); + addDirectArtifactCatalogEntry(entries, seen, record.timing_path, 'artifact'); + addDirectArtifactCatalogEntry(entries, seen, record.input_path, 'artifact'); + addDirectArtifactCatalogEntry(entries, seen, record.output_path, 'artifact'); + addDirectArtifactCatalogEntry(entries, seen, record.response_path, 'artifact'); + addDirectArtifactCatalogEntry(entries, seen, record.answer_path, 'answer'); + addDirectArtifactCatalogEntry(entries, seen, record.transcript_path, 'transcript'); + addDirectArtifactCatalogEntry(entries, seen, recordWithTrace.trace_path, 'trace'); + addDirectArtifactCatalogEntry(entries, seen, record.eval_path, 'artifact'); + addDirectArtifactCatalogEntry(entries, seen, record.targets_path, 'artifact'); + + return entries; +} + +function resultArtifactTreeRootPaths( + record: ResultManifestRecord, + catalog: readonly ArtifactCatalogEntry[], +): string[] { + return [ + ...catalog.filter((entry) => entry.storage === 'local').map((entry) => entry.displayPath), + record.task_dir, + record.files_path, + record.graders_path, + ].filter((p, index, all): p is string => !!p && all.indexOf(p) === index); +} + +function buildLocalResultArtifactTree( + baseDir: string, + record: ResultManifestRecord, + catalog: readonly ArtifactCatalogEntry[], +): FileNode[] { + const knownPaths = resultArtifactTreeRootPaths(record, catalog); + if (knownPaths.length === 0) return []; + + const artifactDirs = knownPaths.map((p) => path.dirname(p)); + let commonDir = artifactDirs[0]; + for (const dir of artifactDirs) { + while (!dir.startsWith(commonDir)) { + commonDir = path.dirname(commonDir); + } + } + + return buildFileTree(path.join(baseDir, commonDir), baseDir); +} + +function catalogEntryForDiscoveredLocalFile( + nodes: readonly FileNode[], + displayPath: string, +): ArtifactCatalogEntry | undefined { + const normalized = normalizeArtifactRelativePath(displayPath); + if (!normalized) return undefined; + const file = flattenFileTree(nodes).find((node) => node.path === normalized); + return file + ? { displayPath: normalized, kind: 'artifact', storage: 'local', path: normalized } + : undefined; +} + +function findArtifactCatalogEntry( + catalog: readonly ArtifactCatalogEntry[], + displayPath: string, +): ArtifactCatalogEntry | undefined { + const normalized = normalizeArtifactRelativePath(displayPath); + return normalized + ? catalog.find((entry) => normalizeArtifactRelativePath(entry.displayPath) === normalized) + : undefined; +} + +function findPointerArtifactCatalogEntry( + catalog: readonly ArtifactCatalogEntry[], + artifact: ResolvedArtifactPointer, + kind: ArtifactCatalogEntry['kind'], + runPath: string | undefined, +): ArtifactCatalogEntry | undefined { + const displayPath = + (artifact.path ? normalizeArtifactRelativePath(artifact.path) : undefined) ?? + displayPathFromArtifactKey(artifact.key, runPath); + if (displayPath) { + const entry = findArtifactCatalogEntry(catalog, displayPath); + if (entry) return entry; + } + return catalog.find( + (entry) => + entry.kind === kind && + artifact.key !== undefined && + entry.key !== undefined && + entry.key === artifact.key, + ); +} + +async function readArtifactCatalogEntryText( + searchDir: string, + projectId: string | undefined, + meta: SourcedResultFileMeta, + entry: ArtifactCatalogEntry, +): Promise<{ content?: string; error?: string }> { + const baseDir = path.dirname(meta.path); + if (entry.storage === 'local') { + const resolved = resolveReadableRunArtifactFile(baseDir, entry.displayPath); + if (resolved.error) return { error: resolved.error }; + if (!resolved.absolutePath) return {}; + return { content: readFileSync(resolved.absolutePath, 'utf8') }; + } + + const localResolved = resolveReadableRunArtifactFile(baseDir, entry.displayPath); + if (localResolved.error) return { error: localResolved.error }; + if (localResolved.absolutePath) { + return { content: readFileSync(localResolved.absolutePath, 'utf8') }; + } + + if (entry.ref !== AGENTV_RESULTS_ARTIFACTS_REF) { + return {}; + } + const content = await readSidecarArtifactText(searchDir, projectId, meta, { + path: entry.path ?? entry.displayPath, + ref: entry.ref, + ...(entry.key && { key: entry.key }), + ...(entry.sha256 && { sha256: entry.sha256 }), + ...(entry.objectVersion && { objectVersion: entry.objectVersion }), + }); + return { content }; +} + function artifactFileContentResponse(c: C, filePath: string, fileContent: string) { if (c.req.query('raw') === '1' || c.req.query('download') === '1') { c.header('Content-Type', inferRawContentType(filePath)); @@ -1292,37 +1571,16 @@ async function handleEvalFiles(c: C, { searchDir, projectId }: DataContext) { if (!record) return c.json({ error: 'Eval not found' }, 404); const baseDir = path.dirname(meta.path); - const transcriptArtifact = resolveRecordArtifactPointer(record, 'transcript'); - const answerArtifact = resolveRecordArtifactPointer(record, 'answer'); - const knownPaths = [ - record.grading_path, - record.timing_path, - record.input_path, - record.output_path, - record.response_path, - record.answer_path, - record.transcript_path, - transcriptArtifact.path, - answerArtifact.path, - record.task_dir, - record.eval_path, - record.targets_path, - record.files_path, - record.graders_path, - ].filter((p, index, all): p is string => !!p && all.indexOf(p) === index); - - if (knownPaths.length === 0) return c.json({ files: [] }); - - const artifactDirs = knownPaths.map((p) => path.dirname(p)); - let commonDir = artifactDirs[0]; - for (const dir of artifactDirs) { - while (!dir.startsWith(commonDir)) { - commonDir = path.dirname(commonDir); + const catalog = buildResultArtifactCatalog(record, { + runPath: relativeRunPathFromManifestPath(meta.path), + }); + const files = buildLocalResultArtifactTree(baseDir, record, catalog); + const localFilePaths = new Set(flattenFileTree(files).map((file) => file.path)); + for (const entry of catalog) { + if (entry.storage === 'git' || !localFilePaths.has(entry.displayPath)) { + ensureCatalogFileNode(files, entry); } } - - const artifactAbsDir = path.join(baseDir, commonDir); - const files = buildFileTree(artifactAbsDir, baseDir); return c.json({ files }); } catch { return c.json({ error: 'Failed to load file tree' }, 500); @@ -1349,41 +1607,27 @@ async function handleEvalFileContent(c: C, { searchDir, projectId }: DataContext if (!filePath) return c.json({ error: 'No file path specified' }, 400); await ensureRunReadable(searchDir, meta, projectId); - const baseDir = path.dirname(meta.path); - const resolvedFile = resolveReadableRunArtifactFile(baseDir, filePath); - if (resolvedFile.error) { - return c.json({ error: 'Path traversal not allowed' }, 403); - } - if (!resolvedFile.absolutePath) { - const records = parseResultManifest(readFileSync(meta.path, 'utf8')); - const record = records.find((r) => r.test_id === evalId); - if (record) { - const normalizedFilePath = normalizeArtifactRelativePath(filePath); - const artifact = [ - resolveRecordArtifactPointer(record, 'trace'), - resolveRecordArtifactPointer(record, 'transcript'), - ].find((candidate) => { - const pointerPath = candidate.path - ? normalizeArtifactRelativePath(candidate.path) - : undefined; - return pointerPath !== undefined && pointerPath === normalizedFilePath; - }); - if (artifact) { - const sidecarContent = await readSidecarArtifactText(searchDir, projectId, meta, artifact); - if (sidecarContent !== undefined) { - return artifactFileContentResponse(c, filePath, sidecarContent); - } - } - } + const records = parseResultManifest(readFileSync(meta.path, 'utf8')); + const record = records.find((r) => r.test_id === evalId); + if (!record) return c.json({ error: 'Eval not found' }, 404); + const catalog = buildResultArtifactCatalog(record, { + runPath: relativeRunPathFromManifestPath(meta.path), + }); + const entry = + findArtifactCatalogEntry(catalog, filePath) ?? + catalogEntryForDiscoveredLocalFile( + buildLocalResultArtifactTree(path.dirname(meta.path), record, catalog), + filePath, + ); + if (!entry) { return c.json({ error: 'File not found' }, 404); } - - try { - const fileContent = readFileSync(resolvedFile.absolutePath, 'utf8'); - return artifactFileContentResponse(c, filePath, fileContent); - } catch { - return c.json({ error: 'Failed to read file' }, 500); + const resolved = await readArtifactCatalogEntryText(searchDir, projectId, meta, entry); + if (resolved.error) { + return c.json({ error: 'Path traversal not allowed' }, 403); } + if (resolved.content === undefined) return c.json({ error: 'File not found' }, 404); + return artifactFileContentResponse(c, entry.displayPath, resolved.content); } async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContext) { @@ -1397,10 +1641,16 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex const record = records.find((r) => r.test_id === evalId); if (!record) return c.json({ error: 'Eval not found' }, 404); - const baseDir = path.dirname(meta.path); const trace = resolveRecordArtifactPointer(record, 'trace'); + const runPath = relativeRunPathFromManifestPath(meta.path); + const traceEntry = findPointerArtifactCatalogEntry( + buildResultArtifactCatalog(record, { runPath }), + trace, + 'trace', + runPath, + ); - if (!trace.path) { + if (!traceEntry && !trace.path) { return c.json( traceSessionArtifactResponse({ status: trace.unsupportedReason ? 'unsupported' : 'missing', @@ -1409,34 +1659,56 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex }), ); } - - const resolvedTrace = resolveReadableRunArtifactFile(baseDir, trace.path); - if (resolvedTrace.error) { + if (trace.path && !normalizeArtifactRelativePath(trace.path)) { return c.json( traceSessionArtifactResponse({ status: 'rejected', trace_path: trace.path, - message: resolvedTrace.error ?? 'Trace artifact path could not be resolved.', + message: 'Artifact path is outside the run workspace.', ...(trace.description && { pointer: trace.description }), }), 403, ); } + if (!traceEntry) { + return c.json( + traceSessionArtifactResponse({ + status: 'dangling', + ...(trace.path && { trace_path: trace.path }), + message: trace.unsupportedReason + ? trace.unsupportedReason + : `Trace artifact pointer is present, but ${trace.path} is not available in this run workspace.`, + ...(trace.description && { pointer: trace.description }), + }), + ); + } - let content: string | undefined; - if (resolvedTrace.absolutePath) { - content = readFileSync(resolvedTrace.absolutePath, 'utf8'); - } else { - content = await readSidecarArtifactText(searchDir, projectId, meta, trace); + const resolvedTrace = await readArtifactCatalogEntryText( + searchDir, + projectId, + meta, + traceEntry, + ); + if (resolvedTrace.error) { + return c.json( + traceSessionArtifactResponse({ + status: 'rejected', + trace_path: traceEntry.displayPath, + message: resolvedTrace.error, + ...(trace.description && { pointer: trace.description }), + }), + 403, + ); } + const content = resolvedTrace.content; if (content === undefined) { const refMessage = trace.ref ? ` on ${trace.ref}` : ''; return c.json( traceSessionArtifactResponse({ status: 'dangling', - trace_path: trace.path, - message: `Trace artifact pointer${refMessage} is present, but ${trace.path} is not available in this run workspace.`, + trace_path: traceEntry.displayPath, + message: `Trace artifact pointer${refMessage} is present, but ${traceEntry.displayPath} is not available in this run workspace.`, ...(trace.description && { pointer: trace.description }), }), ); @@ -1447,7 +1719,7 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex return c.json( traceSessionArtifactResponse({ status: 'invalid', - trace_path: trace.path, + trace_path: traceEntry.displayPath, message: parsed.message, ...(trace.description && { pointer: trace.description }), }), @@ -1459,14 +1731,14 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex testId: record.test_id, suite: record.suite, target: record.target, - artifactPath: trace.path, + artifactPath: traceEntry.displayPath, }); if (normalized.status === 'unsupported') { return c.json( traceSessionArtifactResponse({ status: 'unsupported', - trace_path: trace.path, + trace_path: traceEntry.displayPath, message: normalized.message, ...(trace.description && { pointer: trace.description }), }), @@ -1476,7 +1748,7 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex return c.json( traceSessionArtifactResponse({ status: 'ok', - trace_path: trace.path, + trace_path: traceEntry.displayPath, trace_session: normalized.traceSession, ...(trace.description && { pointer: trace.description }), }), @@ -1497,53 +1769,71 @@ async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext) const record = records.find((r) => r.test_id === evalId); if (!record) return c.json({ error: 'Eval not found' }, 404); - const baseDir = path.dirname(meta.path); const transcript = resolveRecordArtifactPointer(record, 'transcript'); const answer = resolveRecordArtifactPointer(record, 'answer'); + const runPath = relativeRunPathFromManifestPath(meta.path); + const catalog = buildResultArtifactCatalog(record, { runPath }); + const transcriptEntry = findPointerArtifactCatalogEntry( + catalog, + transcript, + 'transcript', + runPath, + ); + const answerEntry = findPointerArtifactCatalogEntry(catalog, answer, 'answer', runPath); - if (!transcript.path) { + if (!transcriptEntry && !transcript.path) { return c.json({ status: transcript.unsupportedReason ? 'unsupported' : 'missing', message: transcript.unsupportedReason ?? missingTranscriptMessage(), ...(transcript.description && { pointer: transcript.description }), }); } - - const resolvedTranscript = resolveReadableRunArtifactFile(baseDir, transcript.path); - if (resolvedTranscript.error) { + if (!transcriptEntry) { return c.json({ status: 'dangling', - transcript_path: transcript.path, - message: resolvedTranscript.error ?? 'Transcript artifact path could not be resolved.', + ...(transcript.path && { transcript_path: transcript.path }), + message: transcript.unsupportedReason + ? transcript.unsupportedReason + : `Transcript artifact pointer is present, but ${transcript.path} is not available in this run workspace.`, ...(transcript.description && { pointer: transcript.description }), }); } - - let content: string | undefined; - if (resolvedTranscript.absolutePath) { - content = readFileSync(resolvedTranscript.absolutePath, 'utf8'); - } else { - content = await readSidecarArtifactText(searchDir, projectId, meta, transcript); + const resolvedTranscript = await readArtifactCatalogEntryText( + searchDir, + projectId, + meta, + transcriptEntry, + ); + if (resolvedTranscript.error) { + return c.json({ + status: 'dangling', + transcript_path: transcriptEntry.displayPath, + message: resolvedTranscript.error, + ...(transcript.description && { pointer: transcript.description }), + }); } + const content = resolvedTranscript.content; if (content === undefined) { const refMessage = transcript.ref ? ` on ${transcript.ref}` : ''; return c.json({ status: 'dangling', - transcript_path: transcript.path, - message: `Transcript artifact pointer${refMessage} is present, but ${transcript.path} is not available in this run workspace.`, + transcript_path: transcriptEntry.displayPath, + message: `Transcript artifact pointer${refMessage} is present, but ${transcriptEntry.displayPath} is not available in this run workspace.`, ...(transcript.description && { pointer: transcript.description }), }); } - const answerContent = readOptionalRunArtifactText(baseDir, answer); + const answerContent = answerEntry + ? (await readArtifactCatalogEntryText(searchDir, projectId, meta, answerEntry)).content + : undefined; return c.json({ status: 'ok', - transcript_path: transcript.path, + transcript_path: transcriptEntry.displayPath, content, - language: inferLanguage(transcript.path), - ...(answer.path && { answer_path: answer.path }), + language: inferLanguage(transcriptEntry.displayPath), + ...(answerEntry && { answer_path: answerEntry.displayPath }), ...(answerContent !== undefined && { answer_content: answerContent }), ...(transcript.description && { pointer: transcript.description }), }); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 9e7be606e..9ef79def3 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -3232,6 +3232,16 @@ describe('serve app', () => { answer_path: answerArtifactPath, answer_content: 'Hello, Alice!', }); + + const filesRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/files`, + ); + expect(filesRes.status).toBe(200); + const filesData = (await filesRes.json()) as { files: Array> }; + const serializedFiles = JSON.stringify(filesData); + expect(serializedFiles).toContain(transcriptArtifactPath); + expect(serializedFiles).toContain(answerArtifactPath); + expect(serializedFiles).toContain('"storage":"local"'); }); it('loads pointer-shaped transcript metadata when it resolves to a local artifact path', async () => { @@ -3318,18 +3328,14 @@ describe('serve app', () => { toJsonl({ ...RESULT_A, experiment, - transcript_path: transcriptArtifactPath, - trace_path: traceArtifactPath, artifact_pointers: { trace: { ref: AGENTV_RESULTS_ARTIFACTS_REF, key: traceKey, - path: traceArtifactPath, }, transcript: { ref: AGENTV_RESULTS_ARTIFACTS_REF, key: transcriptKey, - path: transcriptArtifactPath, }, }, }), @@ -3396,6 +3402,17 @@ describe('serve app', () => { expect(existsSync(path.join(cloneDir, ...transcriptKey.split('/')))).toBe(false); expect(existsSync(path.join(cloneDir, ...traceKey.split('/')))).toBe(false); + const filesRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/files`, + ); + expect(filesRes.status).toBe(200); + const filesData = (await filesRes.json()) as { files: Array> }; + const serializedFiles = JSON.stringify(filesData); + expect(serializedFiles).toContain(transcriptArtifactPath); + expect(serializedFiles).toContain(traceArtifactPath); + expect(serializedFiles).toContain(AGENTV_RESULTS_ARTIFACTS_REF); + expect(serializedFiles).toContain(transcriptKey); + const transcriptRes = await app.request( `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/transcript`, ); @@ -3426,6 +3443,14 @@ describe('serve app', () => { expect(traceSessionData.trace_session.spans[0]?.name).toBe('remote root'); expect(existsSync(path.join(cloneDir, ...traceKey.split('/')))).toBe(false); + const transcriptRawRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/files/${transcriptArtifactPath}?raw=1`, + ); + expect(transcriptRawRes.status).toBe(200); + expect(transcriptRawRes.headers.get('content-type')).toContain('text/plain'); + expect(await transcriptRawRes.text()).toBe(transcriptJsonl); + expect(existsSync(path.join(cloneDir, ...transcriptKey.split('/')))).toBe(false); + const traceRes = await app.request( `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/files/${traceArtifactPath}?raw=1`, ); diff --git a/apps/dashboard/src/components/FileTree.tsx b/apps/dashboard/src/components/FileTree.tsx index 917f466b4..d664390af 100644 --- a/apps/dashboard/src/components/FileTree.tsx +++ b/apps/dashboard/src/components/FileTree.tsx @@ -11,6 +11,12 @@ export interface FileNode { name: string; path: string; type: 'file' | 'dir'; + kind?: string; + storage?: string; + ref?: string; + key?: string; + sha256?: string; + media_type?: string; children?: FileNode[]; } @@ -38,6 +44,12 @@ function getFileIcon(name: string): string { } } +function fileMetadataLabel(node: FileNode): string | undefined { + if (node.type !== 'file') return undefined; + if (node.storage === 'git' && node.ref) return node.ref; + return node.storage && node.storage !== 'local' ? node.storage : undefined; +} + function collectAllDirs(nodes: FileNode[]): string[] { const dirs: string[] = []; for (const node of nodes) { @@ -69,6 +81,7 @@ function TreeNode({ const isDir = node.type === 'dir'; const isOpen = openFolders.has(node.path); const isSelected = selectedPath === node.path; + const metadataLabel = fileMetadataLabel(node); return (
@@ -92,6 +105,20 @@ function TreeNode({ {isDir ? (isOpen ? '\u{1F4C2}' : '\u{1F4C1}') : getFileIcon(node.name)} {node.name} + {metadataLabel && ( + + {metadataLabel} + + )} {isDir && isOpen && node.children && ( diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index ff78ed288..fdad164f5 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -411,6 +411,12 @@ export interface FileNode { name: string; path: string; type: 'file' | 'dir'; + kind?: string; + storage?: string; + ref?: string; + key?: string; + sha256?: string; + media_type?: string; children?: FileNode[]; } From 85993bbfd1c24590a13738f08337263f260736e0 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 16:30:16 +1000 Subject: [PATCH 03/12] feat(config): support nested repo blocks Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apps/cli/src/commands/results/serve.ts | 2 +- .../src/content/docs/docs/tools/dashboard.mdx | 68 +- .../src/content/docs/docs/tools/results.mdx | 4 +- .../docs/docs/tools/wip-checkpoints.mdx | 2 +- .../src/evaluation/loaders/config-loader.ts | 126 +++- .../evaluation/validation/config-validator.ts | 589 +++++++++++------- packages/core/src/project-sync.ts | 4 +- packages/core/src/projects.ts | 106 ++-- .../evaluation/loaders/config-loader.test.ts | 69 ++ .../validation/config-validator.test.ts | 162 +++-- packages/core/test/project-sync.test.ts | 8 +- packages/core/test/projects.test.ts | 35 +- scripts/deploy-example-results-repos.ts | 15 +- scripts/setup-dashboard-deployment.sh | 19 +- .../references/config-schema.json | 94 +++ 15 files changed, 947 insertions(+), 356 deletions(-) diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index e69359011..ca58f6e55 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -125,7 +125,7 @@ const DIRECT_DASHBOARD_SOURCE_GUIDANCE = [ 'Dashboard reads configured project run sources only.', 'Run it from a project root, or pass --dir so Dashboard uses /.agentv/results/runs/:', ' agentv dashboard --dir ', - 'To browse external results, configure results.repo_url or results.repo_path in config YAML.', + 'To browse external results, configure results.repo.url or results.repo.path in config YAML.', 'For a one-off run bundle, use: agentv results report ', ].join('\n'); diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 63d5546c8..898ce21f5 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -209,15 +209,16 @@ folder path, and select a directory that contains `.agentv/`. Each path must contain a `.agentv/` directory. Registered projects are stored under `projects:` in `$AGENTV_HOME/config.yaml`, or `~/.agentv/config.yaml` when `AGENTV_HOME` is unset. -To register a remote repo and keep it synced automatically, add `repo_url` and `ref` to the entry in `$AGENTV_HOME/config.yaml`. `repo_url` is the Git remote URL AgentV passes to `git clone`, so it can be HTTPS or SSH: +To register a remote repo and keep it synced automatically, add a nested `repo` block to the entry in `$AGENTV_HOME/config.yaml`. `repo.url` is the Git remote URL AgentV passes to `git clone`, so it can be HTTPS or SSH. `repo.branch` is the branch or ref to check out, and `repo.path` is the local checkout path: ```yaml projects: - id: my-evals name: My Evals - repo_url: https://github.com/example/my-evals.git - path: /srv/agentv/my-evals - ref: main + repo: + url: https://github.com/example/my-evals.git + branch: main + path: /srv/agentv/my-evals ``` On each Dashboard startup, AgentV clones the repo if the path is empty (`git clone --depth 1`) or pulls the latest if a clone already exists (`git pull --ff-only`). You can also trigger a sync manually from the Dashboard UI's **Sync** button. @@ -277,44 +278,49 @@ For a registered project, put results repo settings on that project's entry in ` projects: - id: agentv name: AgentV - repo_url: https://github.com/EntityProcess/agentv.git - path: /home/entity/projects/EntityProcess/agentv - ref: main + repo: + url: https://github.com/EntityProcess/agentv.git + branch: main + path: /home/entity/projects/EntityProcess/agentv results: - repo_path: . - branch: agentv/results/v1 - remote: origin + repo: + path: . + branch: agentv/results/v1 + remote: origin sync: auto_push: false require_push: false ``` -`results.repo_path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. The name is intentionally `repo_path`, not `repo`: in AgentV, `workspace.repos[].repo` means a portable repository identity such as a clone URL or `org/name`, while `results.repo_path` means an existing local Git checkout whose object database and refs AgentV should write to. The branch defaults to `agentv/results/v1` for `repo_path` configs. AgentV creates the branch automatically on first publish and commits only `.agentv/results/**` paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. `sync.require_push: true` is for CI workflows where a push failure should fail the command after local artifacts are written. +`results.repo.path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. When `results.repo.url` is omitted, `results.repo.path` means an existing local Git checkout whose object database and refs AgentV should write to. `results.repo.remote` is the Git remote name, such as `origin`, not a URL. The branch defaults to `agentv/results/v1` for local checkout configs. AgentV creates the branch automatically on first publish and commits only `.agentv/results/**` paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. `sync.require_push: true` is for CI workflows where a push failure should fail the command after local artifacts are written. -For a separate results repository, use `repo_url` and an optional managed clone `path`: +For a separate results repository, use `results.repo.url` and an optional managed clone `results.repo.path`: ```yaml projects: - id: agentv name: AgentV - path: /home/entity/projects/EntityProcess/agentv + repo: + path: /home/entity/projects/EntityProcess/agentv results: - repo_url: git@github.com:EntityProcess/agentv-examples-eval-results.git - branch: agentv/results/v1 - path: /home/entity/projects/EntityProcess/agentv-examples-eval-results + repo: + url: git@github.com:EntityProcess/agentv-examples-eval-results.git + branch: agentv/results/v1 + path: /home/entity/projects/EntityProcess/agentv-examples-eval-results sync: auto_push: true ``` -`results.repo_url` is the Git remote URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured. `results.path` is the filesystem location of the local clone AgentV manages for that remote results repo. Use `repo_path` only when the results store is an already-existing local checkout such as `.`. +`results.repo.url` is the Git remote URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured. When `results.repo.url` is set, `results.repo.path` is the filesystem location of the local clone AgentV manages for that remote results repo. Omit `results.repo.url` only when `results.repo.path` points at an already-existing local checkout such as `.`. You can also set a top-level global fallback in the same file. This is used when the current project is not registered or its registry entry has no `results` block: ```yaml results: - repo_path: . - branch: agentv/results/v1 - remote: origin + repo: + path: . + branch: agentv/results/v1 + remote: origin sync: auto_push: false require_push: false @@ -322,10 +328,10 @@ results: Project-local `.agentv/config.yaml` is for portable eval defaults such as `execution`, `eval_patterns`, and `dashboard`. Do not put `projects` in project-local config; AgentV warns and ignores it there. `results_by_project` is deprecated; use `projects[].results` in `$AGENTV_HOME/config.yaml`. -The project `repo_url` and the `results` block sync different repositories: +The project `repo` block and the `results` block sync different repositories: -- `projects[].repo_url` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current. -- `projects[].results.repo_url` is the git-backed results store remote. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in that results repo clone. +- `projects[].repo.url` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current. +- `projects[].results.repo.url` is the git-backed results store remote. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in that results repo clone. #### Migration from the legacy project schema @@ -352,18 +358,20 @@ After: projects: - id: agentv name: AgentV - repo_url: https://github.com/EntityProcess/agentv.git - path: /home/entity/projects/EntityProcess/agentv - ref: main + repo: + url: https://github.com/EntityProcess/agentv.git + branch: main + path: /home/entity/projects/EntityProcess/agentv results: - repo_url: https://github.com/EntityProcess/agentv-eval-results.git - branch: agentv/results/v1 - path: /home/entity/projects/EntityProcess/agentv-eval-results + repo: + url: https://github.com/EntityProcess/agentv-eval-results.git + branch: agentv/results/v1 + path: /home/entity/projects/EntityProcess/agentv-eval-results sync: auto_push: true ``` -Legacy project fields (`source`, `repository`, `results.mode`, `results.repo`, `results.repository`, `results.local_path`, and `results.auto_push`) fail validation with migration guidance. +Current flat fields (`path`, `repo_url`, `ref`, `results.repo_url`, `results.repo_path`, `results.branch`, `results.remote`, and `results.path`) still load with migration warnings and are written back in nested form the next time AgentV saves the project registry. Older removed fields (`source`, `repository`, `results.mode`, `results.repo` as a string, `results.repository`, `results.local_path`, and `results.auto_push`) fail validation with migration guidance. Use project-level **Sync Project** as the results exchange workflow. It handles pulled remote runs, locally edited metadata, dirty state, and blocked conflict feedback in one project-scoped action. diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index f58dded2a..3a4025e89 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -228,7 +228,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo_path: .` with `branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. `repo_path` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.path: .` with `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. `results.repo.path` without `results.repo.url` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. -- **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.path` clone with `git` directly, then sync again. +- **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.repo.path` clone with `git` directly, then sync again. diff --git a/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx b/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx index c03ab1019..6052720ba 100644 --- a/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx +++ b/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx @@ -25,7 +25,7 @@ If no results repo is configured, or auto-push is disabled, `agentv eval` still | Local project | `.agentv/results/runs///benchmark.json` | A run-start stub with `metadata.planned_test_count` and the eval file path when known. This lets Dashboard recognize incomplete local runs as resumable. | | Local project | `.agentv/results/runs///index.jsonl` | Result rows appended as test cases finish. Rows use the normal snake_case result JSONL format. | | Results repo remote | `agentv/wip//` | A forced-updated branch containing the checkpointed run under `.agentv/results/runs//`. | -| Results repo storage branch | Configured `results.branch`; `repo_path` configs default to `agentv/results/v1` | The final published run after `agentv eval` completes and the normal auto-export succeeds. | +| Results repo storage branch | Configured `results.repo.branch`; local checkout configs default to `agentv/results/v1` | The final published run after `agentv eval` completes and the normal auto-export succeeds. | The WIP branch name is derived from the current host and the run directory basename. Non-branch-safe characters are replaced with `-`; the host component is capped at 40 characters and the run component at 60 characters. diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 0f7301826..89ae25d66 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -600,6 +600,87 @@ function isFilesystemPath(p: string): boolean { ); } +function readTrimmedString(value: unknown): string | undefined { + if (typeof value !== 'string') { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +function isGitRemoteUrl(value: string): boolean { + return /^(https?:\/\/|ssh:\/\/|git@|file:\/\/).+/.test(value); +} + +type NestedResultsRepoConfig = { + readonly repo_url?: string; + readonly repo_path?: string; + readonly branch?: string; + readonly remote?: string; + readonly path?: string; +}; + +function parseNestedResultsRepoConfig( + raw: unknown, + configPath: string, +): NestedResultsRepoConfig | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + logWarning(`Invalid results.repo in ${configPath}, expected object`); + return undefined; + } + + const repo = raw as Record; + const url = readTrimmedString(repo.url); + const repoPath = readTrimmedString(repo.path); + const branch = readTrimmedString(repo.branch); + const remote = readTrimmedString(repo.remote); + + if (repo.url !== undefined && !url) { + logWarning(`Invalid results.repo.url in ${configPath}, expected non-empty string`); + return undefined; + } + if (url && !isGitRemoteUrl(url)) { + logWarning(`Invalid results.repo.url in ${configPath}, expected Git remote URL`); + return undefined; + } + if (repo.path !== undefined && !repoPath) { + logWarning(`Invalid results.repo.path in ${configPath}, expected non-empty string`); + return undefined; + } + if (!url && !repoPath) { + logWarning(`Invalid results.repo in ${configPath}, expected url or path`); + return undefined; + } + if (url && repoPath && !isFilesystemPath(repoPath)) { + logWarning( + `Invalid results.repo.path in ${configPath}: '${repoPath}' must be an absolute or home-relative filesystem path when results.repo.url is set.`, + ); + return undefined; + } + if (repo.branch !== undefined && !branch) { + logWarning(`Invalid results.repo.branch in ${configPath}, expected non-empty string`); + return undefined; + } + if (repo.remote !== undefined && !remote) { + logWarning(`Invalid results.repo.remote in ${configPath}, expected non-empty string`); + return undefined; + } + if (remote && isGitRemoteUrl(remote)) { + logWarning( + `Invalid results.repo.remote in ${configPath}: use results.repo.url for Git remote URLs and results.repo.remote for the remote name such as origin.`, + ); + return undefined; + } + + return { + ...(url && { repo_url: url }), + ...(repoPath && !url && { repo_path: repoPath }), + ...(url && repoPath && { path: repoPath }), + ...(branch && { branch }), + ...(remote && { remote }), + }; +} + export function parseResultsConfig(raw: unknown, configPath: string): ResultsConfig | undefined { if (raw === undefined || raw === null) { return undefined; @@ -616,21 +697,48 @@ export function parseResultsConfig(raw: unknown, configPath: string): ResultsCon return undefined; } + const hasNestedRepo = obj.repo !== undefined && typeof obj.repo === 'object' && obj.repo !== null; + const nestedRepo = hasNestedRepo ? parseNestedResultsRepoConfig(obj.repo, configPath) : undefined; + if (hasNestedRepo && !nestedRepo) { + return undefined; + } + if (obj.repo !== undefined && !hasNestedRepo && typeof obj.repo !== 'string') { + logWarning(`Invalid results.repo in ${configPath}, expected string or object`); + return undefined; + } + if ( + nestedRepo && + ['repo_url', 'repo_path', 'branch', 'remote', 'path'].some((field) => obj[field] !== undefined) + ) { + logWarning( + `Invalid results in ${configPath}, do not mix nested results.repo with flat repo_url, repo_path, branch, remote, or path fields`, + ); + return undefined; + } + const legacyRepo = typeof obj.repo === 'string' ? obj.repo.trim() : ''; - const repoUrl = typeof obj.repo_url === 'string' ? obj.repo_url.trim() : ''; - const repoPath = typeof obj.repo_path === 'string' ? obj.repo_path.trim() : ''; + const repoUrl = + nestedRepo?.repo_url ?? (typeof obj.repo_url === 'string' ? obj.repo_url.trim() : ''); + const repoPath = + nestedRepo?.repo_path ?? (typeof obj.repo_path === 'string' ? obj.repo_path.trim() : ''); const repo = legacyRepo || repoUrl; if (!repo && !repoPath) { - logWarning(`Invalid results in ${configPath}, expected repo_url/repo or repo_path`); + logWarning( + `Invalid results in ${configPath}, expected nested repo.url/repo.path or compatible repo_url/repo_path`, + ); return undefined; } if (repo && repoPath) { - logWarning(`Invalid results in ${configPath}, set only one of repo_url/repo or repo_path`); + logWarning( + `Invalid results in ${configPath}, set only one of nested repo.url/repo.path or compatible repo_url/repo_path`, + ); return undefined; } let branch: string | undefined; - if (obj.branch !== undefined) { + if (nestedRepo?.branch !== undefined) { + branch = nestedRepo.branch; + } else if (obj.branch !== undefined) { if (typeof obj.branch !== 'string' || obj.branch.trim().length === 0) { logWarning(`Invalid results.branch in ${configPath}, expected non-empty string`); return undefined; @@ -639,7 +747,9 @@ export function parseResultsConfig(raw: unknown, configPath: string): ResultsCon } let remote: string | undefined; - if (obj.remote !== undefined) { + if (nestedRepo?.remote !== undefined) { + remote = nestedRepo.remote; + } else if (obj.remote !== undefined) { if (typeof obj.remote !== 'string' || obj.remote.trim().length === 0) { logWarning(`Invalid results.remote in ${configPath}, expected non-empty string`); return undefined; @@ -648,7 +758,9 @@ export function parseResultsConfig(raw: unknown, configPath: string): ResultsCon } let resultsPath: string | undefined; - if (obj.path !== undefined) { + if (nestedRepo?.path !== undefined) { + resultsPath = nestedRepo.path; + } else if (obj.path !== undefined) { if (typeof obj.path !== 'string' || obj.path.trim().length === 0) { logWarning(`Invalid results.path in ${configPath}, expected non-empty string`); return undefined; diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index 44ae5b9f1..d7f3eaba2 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -168,36 +168,141 @@ function validateProjects(errors: ValidationError[], filePath: string, projects: const projectRecord = project as Record; validateRequiredString(errors, filePath, projectRecord.id, `${location}.id`); validateRequiredString(errors, filePath, projectRecord.name, `${location}.name`); - validateRequiredString(errors, filePath, projectRecord.path, `${location}.path`); + validateProjectRepoConfig(errors, filePath, projectRecord, location); + validateProjectResultsConfig(errors, filePath, projectRecord.results, `${location}.results`); + }); +} - if (projectRecord.source !== undefined) { - errors.push({ - severity: 'error', - filePath, - location: `${location}.source`, - message: `Field '${location}.source' was removed. Move 'source.url' to '${location}.repo_url' and move 'source.ref' to '${location}.ref'. Use a Git remote URL such as https://github.com/example/repo.git or git@github.com:example/repo.git.`, - }); - } +function addWarning( + errors: ValidationError[], + filePath: string, + location: string, + message: string, +): void { + errors.push({ severity: 'warning', filePath, location, message }); +} - if (projectRecord.repository !== undefined) { - errors.push({ - severity: 'error', +function addError( + errors: ValidationError[], + filePath: string, + location: string, + message: string, +): void { + errors.push({ severity: 'error', filePath, location, message }); +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isGitRemoteUrlValue(value: string): boolean { + return /^(https?:\/\/|ssh:\/\/|git@|file:\/\/).+/.test(value.trim()); +} + +function validateProjectRepoConfig( + errors: ValidationError[], + filePath: string, + projectRecord: Record, + location: string, +): void { + if (projectRecord.source !== undefined) { + addError( + errors, + filePath, + `${location}.source`, + `Field '${location}.source' was removed. Move 'source.url' to '${location}.repo.url', move 'source.ref' to '${location}.repo.branch', and set '${location}.repo.path' to the local checkout path.`, + ); + } + + if (projectRecord.repository !== undefined) { + addError( + errors, + filePath, + `${location}.repository`, + `Field '${location}.repository' was removed. Use '${location}.repo.url' with a Git remote URL instead.`, + ); + } + + if (projectRecord.repo !== undefined) { + if (!isPlainObject(projectRecord.repo)) { + addError( + errors, filePath, - location: `${location}.repository`, - message: `Field '${location}.repository' was removed. Use '${location}.repo_url' with a Git remote URL instead.`, - }); + `${location}.repo`, + `Field '${location}.repo' must be an object with path, optional url, and optional branch.`, + ); + return; } - if (projectRecord.repo_url !== undefined) { - validateGitRemoteUrl(errors, filePath, projectRecord.repo_url, `${location}.repo_url`); + for (const flatField of ['path', 'repo_url', 'ref']) { + if (projectRecord[flatField] !== undefined) { + addError( + errors, + filePath, + `${location}.${flatField}`, + `Do not mix '${location}.${flatField}' with '${location}.repo'. Move source repo fields under '${location}.repo'.`, + ); + } } - if (projectRecord.ref !== undefined) { - validateRequiredString(errors, filePath, projectRecord.ref, `${location}.ref`); + validateRequiredString(errors, filePath, projectRecord.repo.path, `${location}.repo.path`); + if (projectRecord.repo.url !== undefined) { + validateGitRemoteUrl(errors, filePath, projectRecord.repo.url, `${location}.repo.url`); + } + if (projectRecord.repo.branch !== undefined) { + validateRequiredString( + errors, + filePath, + projectRecord.repo.branch, + `${location}.repo.branch`, + ); + } + if (projectRecord.repo.ref !== undefined) { + addError( + errors, + filePath, + `${location}.repo.ref`, + `Field '${location}.repo.ref' is not supported. Use '${location}.repo.branch'.`, + ); + } + if (projectRecord.repo.remote !== undefined) { + addError( + errors, + filePath, + `${location}.repo.remote`, + `Use '${location}.repo.url' for the source Git URL. '${location}.repo.remote' is reserved for Git remote names in results repo config.`, + ); } + return; + } - validateProjectResultsConfig(errors, filePath, projectRecord.results, `${location}.results`); - }); + validateRequiredString(errors, filePath, projectRecord.path, `${location}.path`); + addWarning( + errors, + filePath, + `${location}.path`, + `Field '${location}.path' is deprecated. Use '${location}.repo.path'. Existing flat project entries still load and are written back in nested form.`, + ); + + if (projectRecord.repo_url !== undefined) { + validateGitRemoteUrl(errors, filePath, projectRecord.repo_url, `${location}.repo_url`); + addWarning( + errors, + filePath, + `${location}.repo_url`, + `Field '${location}.repo_url' is deprecated. Use '${location}.repo.url'.`, + ); + } + + if (projectRecord.ref !== undefined) { + validateRequiredString(errors, filePath, projectRecord.ref, `${location}.ref`); + addWarning( + errors, + filePath, + `${location}.ref`, + `Field '${location}.ref' is deprecated. Use '${location}.repo.branch'.`, + ); + } } function validateRequiredString( @@ -233,7 +338,7 @@ function validateGitRemoteUrl( } const repoUrl = value.trim(); - if (!/^(https?:\/\/|ssh:\/\/|git@|file:\/\/).+/.test(repoUrl)) { + if (!isGitRemoteUrlValue(repoUrl)) { errors.push({ severity: 'error', filePath, @@ -243,109 +348,97 @@ function validateGitRemoteUrl( } } -function validateProjectResultsConfig( +function validateResultsRepoBlock( errors: ValidationError[], filePath: string, - rawResults: unknown, + rawRepo: unknown, location: string, ): void { - if (rawResults === undefined) { + if (!isPlainObject(rawRepo)) { + addError(errors, filePath, location, `Field '${location}' must be an object`); return; } - if (typeof rawResults !== 'object' || rawResults === null || Array.isArray(rawResults)) { - errors.push({ - severity: 'error', - filePath, - location, - message: `Field '${location}' must be an object`, - }); - return; - } + const repoRecord = rawRepo; + const hasUrl = repoRecord.url !== undefined; + const hasPath = repoRecord.path !== undefined; - const resultsRecord = rawResults as Record; + if (!hasUrl && !hasPath) { + addError(errors, filePath, location, `Field '${location}' must set url or path`); + } - const removedFields: Record = { - mode: `Remove '${location}.mode'; project results use '${location}.repo_url' as the Git remote URL.`, - repo: `Field '${location}.repo' was removed. Use '${location}.repo_url' with a Git remote URL instead.`, - repository: `Field '${location}.repository' was removed. Use '${location}.repo_url' with a Git remote URL instead.`, - local_path: `Field '${location}.local_path' was removed. Use '${location}.path' for the local clone path instead.`, - auto_push: `Field '${location}.auto_push' was removed. Use '${location}.sync.auto_push' instead.`, - }; + if (hasUrl) { + validateGitRemoteUrl(errors, filePath, repoRecord.url, `${location}.url`); + } - for (const [field, message] of Object.entries(removedFields)) { - if (resultsRecord[field] !== undefined) { - errors.push({ - severity: 'error', + if (hasPath) { + validateRequiredString(errors, filePath, repoRecord.path, `${location}.path`); + if ( + hasUrl && + typeof repoRecord.path === 'string' && + repoRecord.path.trim().length > 0 && + !isFilesystemPath(repoRecord.path.trim()) + ) { + addError( + errors, filePath, - location: `${location}.${field}`, - message, - }); + `${location}.path`, + `'${location}.path' must be an absolute or home-relative filesystem path when '${location}.url' is set.`, + ); } } - const hasRepoUrl = resultsRecord.repo_url !== undefined; - const hasRepoPath = resultsRecord.repo_path !== undefined; - if (hasRepoUrl && hasRepoPath) { - errors.push({ - severity: 'error', - filePath, - location, - message: `Field '${location}' must set only one of repo_url or repo_path`, - }); - } else if (hasRepoUrl) { - validateGitRemoteUrl(errors, filePath, resultsRecord.repo_url, `${location}.repo_url`); - } else if (hasRepoPath) { - validateRequiredString(errors, filePath, resultsRecord.repo_path, `${location}.repo_path`); - } else { - errors.push({ - severity: 'error', + if ( + repoRecord.branch !== undefined && + (typeof repoRecord.branch !== 'string' || repoRecord.branch.trim().length === 0) + ) { + addError( + errors, filePath, - location, - message: `Field '${location}' must set repo_url or repo_path`, - }); + `${location}.branch`, + `Field '${location}.branch' must be a non-empty string`, + ); } if ( - resultsRecord.branch !== undefined && - (typeof resultsRecord.branch !== 'string' || resultsRecord.branch.trim().length === 0) + repoRecord.remote !== undefined && + (typeof repoRecord.remote !== 'string' || repoRecord.remote.trim().length === 0) ) { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.branch`, - message: `Field '${location}.branch' must be a non-empty string`, - }); + `${location}.remote`, + `Field '${location}.remote' must be a non-empty string`, + ); } if ( - resultsRecord.remote !== undefined && - (typeof resultsRecord.remote !== 'string' || resultsRecord.remote.trim().length === 0) + typeof repoRecord.remote === 'string' && + repoRecord.remote.trim().length > 0 && + isGitRemoteUrlValue(repoRecord.remote) ) { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.remote`, - message: `Field '${location}.remote' must be a non-empty string`, - }); + `${location}.remote`, + `Field '${location}.remote' is the Git remote name (for example, origin). Use '${location}.url' for Git remote URLs.`, + ); } +} - if (resultsRecord.path !== undefined) { - if (typeof resultsRecord.path !== 'string' || resultsRecord.path.trim().length === 0) { - errors.push({ - severity: 'error', - filePath, - location: `${location}.path`, - message: `Field '${location}.path' must be a non-empty string`, - }); - } else if (!isFilesystemPath(resultsRecord.path.trim())) { - errors.push({ - severity: 'error', - filePath, - location: `${location}.path`, - message: `'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results).`, - }); - } +function validateResultsSyncAndBranchPrefix( + errors: ValidationError[], + filePath: string, + resultsRecord: Record, + location: string, +): void { + if (resultsRecord.auto_push !== undefined && typeof resultsRecord.auto_push !== 'boolean') { + addError( + errors, + filePath, + `${location}.auto_push`, + `Field '${location}.auto_push' must be a boolean`, + ); } if (resultsRecord.sync !== undefined) { @@ -354,29 +447,24 @@ function validateProjectResultsConfig( resultsRecord.sync === null || Array.isArray(resultsRecord.sync) ) { - errors.push({ - severity: 'error', - filePath, - location: `${location}.sync`, - message: `Field '${location}.sync' must be an object`, - }); + addError(errors, filePath, `${location}.sync`, `Field '${location}.sync' must be an object`); } else { const syncRecord = resultsRecord.sync as Record; if (syncRecord.auto_push !== undefined && typeof syncRecord.auto_push !== 'boolean') { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.sync.auto_push`, - message: `Field '${location}.sync.auto_push' must be a boolean`, - }); + `${location}.sync.auto_push`, + `Field '${location}.sync.auto_push' must be a boolean`, + ); } if (syncRecord.require_push !== undefined && typeof syncRecord.require_push !== 'boolean') { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.sync.require_push`, - message: `Field '${location}.sync.require_push' must be a boolean`, - }); + `${location}.sync.require_push`, + `Field '${location}.sync.require_push' must be a boolean`, + ); } } } @@ -386,62 +474,39 @@ function validateProjectResultsConfig( (typeof resultsRecord.branch_prefix !== 'string' || resultsRecord.branch_prefix.trim().length === 0) ) { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.branch_prefix`, - message: `Field '${location}.branch_prefix' must be a non-empty string`, - }); + `${location}.branch_prefix`, + `Field '${location}.branch_prefix' must be a non-empty string`, + ); } } -function validateResultsConfig( +function validateFlatResultsRepoConfig( errors: ValidationError[], filePath: string, - rawResults: unknown, + resultsRecord: Record, location: string, + options: { allowLegacyRepoString: boolean }, ): void { - if (rawResults === undefined) { - return; - } - - if (typeof rawResults !== 'object' || rawResults === null || Array.isArray(rawResults)) { - errors.push({ - severity: 'error', - filePath, - location, - message: `Field '${location}' must be an object`, - }); - return; - } - - const resultsRecord = rawResults as Record; - if (resultsRecord.mode !== undefined && resultsRecord.mode !== 'github') { - errors.push({ - severity: 'error', - filePath, - location: `${location}.mode`, - message: `Field '${location}.mode' must be 'github'`, - }); - } - const hasLegacyRepo = resultsRecord.repo !== undefined; + const hasLegacyRepo = typeof resultsRecord.repo === 'string'; const hasRepoUrl = resultsRecord.repo_url !== undefined; const hasRepoPath = resultsRecord.repo_path !== undefined; - const sourceCount = [hasLegacyRepo, hasRepoUrl, hasRepoPath].filter(Boolean).length; + const sourceCount = [ + options.allowLegacyRepoString && hasLegacyRepo, + hasRepoUrl, + hasRepoPath, + ].filter(Boolean).length; if (sourceCount === 0) { - errors.push({ - severity: 'error', - filePath, - location, - message: `Field '${location}' must set repo_url/repo or repo_path`, - }); + addError(errors, filePath, location, `Field '${location}' must set repo.url or repo.path`); } else if (sourceCount > 1) { - errors.push({ - severity: 'error', + addError( + errors, filePath, location, - message: `Field '${location}' must set only one of repo_url/repo or repo_path`, - }); + `Field '${location}' must set only one results repo source. Use '${location}.repo.url' for a managed clone or '${location}.repo.path' for an existing local checkout.`, + ); } else if (hasLegacyRepo) { validateRequiredString(errors, filePath, resultsRecord.repo, `${location}.repo`); } else if (hasRepoUrl) { @@ -454,101 +519,183 @@ function validateResultsConfig( resultsRecord.branch !== undefined && (typeof resultsRecord.branch !== 'string' || resultsRecord.branch.trim().length === 0) ) { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.branch`, - message: `Field '${location}.branch' must be a non-empty string`, - }); + `${location}.branch`, + `Field '${location}.branch' must be a non-empty string`, + ); } if ( resultsRecord.remote !== undefined && (typeof resultsRecord.remote !== 'string' || resultsRecord.remote.trim().length === 0) ) { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.remote`, - message: `Field '${location}.remote' must be a non-empty string`, - }); + `${location}.remote`, + `Field '${location}.remote' must be a non-empty string`, + ); } if (resultsRecord.path !== undefined) { if (typeof resultsRecord.path !== 'string' || resultsRecord.path.trim().length === 0) { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.path`, - message: `Field '${location}.path' must be a non-empty string`, - }); + `${location}.path`, + `Field '${location}.path' must be a non-empty string`, + ); } else { const p = resultsRecord.path.trim(); if (!isFilesystemPath(p)) { - errors.push({ - severity: 'error', + addError( + errors, filePath, - location: `${location}.path`, - message: `'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results). Found: '${p}'. Remove 'path' to use the default.`, - }); + `${location}.path`, + `'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results). Found: '${p}'. Remove 'path' to use the default.`, + ); } } } +} - if (resultsRecord.auto_push !== undefined && typeof resultsRecord.auto_push !== 'boolean') { - errors.push({ - severity: 'error', - filePath, - location: `${location}.auto_push`, - message: `Field '${location}.auto_push' must be a boolean`, - }); +function warnFlatResultsMigration( + errors: ValidationError[], + filePath: string, + resultsRecord: Record, + location: string, +): void { + const migrations: Record = { + repo: `${location}.repo.url`, + repo_url: `${location}.repo.url`, + repo_path: `${location}.repo.path`, + branch: `${location}.repo.branch`, + remote: `${location}.repo.remote`, + path: `${location}.repo.path`, + auto_push: `${location}.sync.auto_push`, + mode: '(remove this field)', + }; + + for (const [field, replacement] of Object.entries(migrations)) { + if (resultsRecord[field] !== undefined) { + addWarning( + errors, + filePath, + `${location}.${field}`, + `Field '${location}.${field}' is deprecated. Use '${replacement}' in the nested results repo schema.`, + ); + } } +} - if (resultsRecord.sync !== undefined) { - if ( - typeof resultsRecord.sync !== 'object' || - resultsRecord.sync === null || - Array.isArray(resultsRecord.sync) - ) { - errors.push({ - severity: 'error', +function validateResultsConfigBody( + errors: ValidationError[], + filePath: string, + rawResults: unknown, + location: string, + options: { allowLegacyRepoString: boolean; projectScoped: boolean }, +): void { + if (rawResults === undefined) { + return; + } + + if (!isPlainObject(rawResults)) { + addError(errors, filePath, location, `Field '${location}' must be an object`); + return; + } + + const resultsRecord = rawResults; + if (resultsRecord.mode !== undefined) { + if (options.projectScoped) { + addError( + errors, filePath, - location: `${location}.sync`, - message: `Field '${location}.sync' must be an object`, - }); - } else { - const syncRecord = resultsRecord.sync as Record; - if (syncRecord.auto_push !== undefined && typeof syncRecord.auto_push !== 'boolean') { - errors.push({ - severity: 'error', - filePath, - location: `${location}.sync.auto_push`, - message: `Field '${location}.sync.auto_push' must be a boolean`, - }); - } - if (syncRecord.require_push !== undefined && typeof syncRecord.require_push !== 'boolean') { - errors.push({ - severity: 'error', - filePath, - location: `${location}.sync.require_push`, - message: `Field '${location}.sync.require_push' must be a boolean`, - }); - } + `${location}.mode`, + `Remove '${location}.mode'; project results use '${location}.repo.url' or '${location}.repo.path'.`, + ); + } else if (resultsRecord.mode !== 'github') { + addError(errors, filePath, `${location}.mode`, `Field '${location}.mode' must be 'github'`); } } - if ( - resultsRecord.branch_prefix !== undefined && - (typeof resultsRecord.branch_prefix !== 'string' || - resultsRecord.branch_prefix.trim().length === 0) - ) { - errors.push({ - severity: 'error', + for (const [field, message] of Object.entries({ + repository: `Field '${location}.repository' was removed. Use '${location}.repo.url' with a Git remote URL instead.`, + local_path: `Field '${location}.local_path' was removed. Use '${location}.repo.path' for the local clone path instead.`, + })) { + if (resultsRecord[field] !== undefined) { + addError(errors, filePath, `${location}.${field}`, message); + } + } + + if (options.projectScoped && resultsRecord.auto_push !== undefined) { + addError( + errors, filePath, - location: `${location}.branch_prefix`, - message: `Field '${location}.branch_prefix' must be a non-empty string`, + `${location}.auto_push`, + `Field '${location}.auto_push' was removed. Use '${location}.sync.auto_push' instead.`, + ); + } + + const hasNestedRepo = isPlainObject(resultsRecord.repo); + if (resultsRecord.repo !== undefined && !hasNestedRepo) { + if (typeof resultsRecord.repo === 'string' && options.allowLegacyRepoString) { + // Handled by the flat compatibility branch below. + } else { + addError( + errors, + filePath, + `${location}.repo`, + `Field '${location}.repo' must be an object. Use '${location}.repo.url' for a managed results clone or '${location}.repo.path' for an existing local checkout.`, + ); + } + } + + if (hasNestedRepo) { + for (const flatField of ['repo_url', 'repo_path', 'branch', 'remote', 'path']) { + if (resultsRecord[flatField] !== undefined) { + addError( + errors, + filePath, + `${location}.${flatField}`, + `Do not mix '${location}.${flatField}' with '${location}.repo'. Move results repo fields under '${location}.repo'.`, + ); + } + } + validateResultsRepoBlock(errors, filePath, resultsRecord.repo, `${location}.repo`); + } else { + validateFlatResultsRepoConfig(errors, filePath, resultsRecord, location, { + allowLegacyRepoString: options.allowLegacyRepoString, }); + warnFlatResultsMigration(errors, filePath, resultsRecord, location); } + + validateResultsSyncAndBranchPrefix(errors, filePath, resultsRecord, location); +} + +function validateProjectResultsConfig( + errors: ValidationError[], + filePath: string, + rawResults: unknown, + location: string, +): void { + validateResultsConfigBody(errors, filePath, rawResults, location, { + allowLegacyRepoString: false, + projectScoped: true, + }); +} + +function validateResultsConfig( + errors: ValidationError[], + filePath: string, + rawResults: unknown, + location: string, +): void { + validateResultsConfigBody(errors, filePath, rawResults, location, { + allowLegacyRepoString: true, + projectScoped: false, + }); } function isFilesystemPath(p: string): boolean { diff --git a/packages/core/src/project-sync.ts b/packages/core/src/project-sync.ts index 59fb675a5..03521cda3 100644 --- a/packages/core/src/project-sync.ts +++ b/packages/core/src/project-sync.ts @@ -26,10 +26,10 @@ import type { ProjectEntry } from './projects.js'; */ export async function syncProject(entry: ProjectEntry): Promise { if (!entry.repoUrl) { - throw new Error(`Project '${entry.id}' has no repo_url defined`); + throw new Error(`Project '${entry.id}' has no repo.url defined`); } if (!entry.ref) { - throw new Error(`Project '${entry.id}' has no ref defined`); + throw new Error(`Project '${entry.id}' has no repo.branch defined`); } const dest = entry.path; diff --git a/packages/core/src/projects.ts b/packages/core/src/projects.ts index 46988a6a3..ad7827307 100644 --- a/packages/core/src/projects.ts +++ b/packages/core/src/projects.ts @@ -16,14 +16,15 @@ * projects: * - id: my-app * name: My App - * repo_url: https://github.com/example/my-app.git - * path: /home/user/projects/my-app - * ref: main + * repo: + * url: https://github.com/example/my-app.git + * branch: main + * path: /home/user/projects/my-app * results: - * repo_path: . - * branch: agentv/results/v1 - * remote: origin - * path: /srv/agentv/results/my-app + * repo: + * path: . + * branch: agentv/results/v1 + * remote: origin * sync: * auto_push: true * require_push: false @@ -106,6 +107,7 @@ interface ProjectResultsSyncYaml { } interface ProjectResultsYaml { + repo?: ProjectResultsRepoYaml; repo_url?: string; repo_path?: string; branch?: string; @@ -115,58 +117,81 @@ interface ProjectResultsYaml { branch_prefix?: string; } +interface ProjectRepoYaml { + url?: string; + branch?: string; + path?: string; +} + +interface ProjectResultsRepoYaml { + url?: string; + path?: string; + branch?: string; + remote?: string; +} + interface ProjectEntryYaml { id: string; name: string; + repo?: ProjectRepoYaml; repo_url?: string; - path: string; + path?: string; ref?: string; added_at: string; last_opened_at: string; results?: ProjectResultsYaml; } +function readTrimmedString(value: unknown): string | undefined { + if (typeof value !== 'string') return undefined; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + function fromYaml(raw: unknown): ProjectEntry | null { if (!raw || typeof raw !== 'object') return null; const e = raw as Partial; - if (typeof e.id !== 'string' || typeof e.name !== 'string' || typeof e.path !== 'string') { + const repo = e.repo && typeof e.repo === 'object' ? e.repo : undefined; + const sourcePath = readTrimmedString(repo?.path) ?? readTrimmedString(e.path); + if (typeof e.id !== 'string' || typeof e.name !== 'string' || !sourcePath) { return null; } const entry: ProjectEntry = { id: e.id, name: e.name, - path: e.path, + path: sourcePath, addedAt: typeof e.added_at === 'string' ? e.added_at : '', lastOpenedAt: typeof e.last_opened_at === 'string' ? e.last_opened_at : '', }; - if (typeof e.repo_url === 'string' && e.repo_url.trim().length > 0) { - entry.repoUrl = e.repo_url.trim(); + const repoUrl = readTrimmedString(repo?.url) ?? readTrimmedString(e.repo_url); + if (repoUrl) { + entry.repoUrl = repoUrl; } - if (typeof e.ref === 'string' && e.ref.trim().length > 0) { - entry.ref = e.ref.trim(); + const branch = readTrimmedString(repo?.branch) ?? readTrimmedString(e.ref); + if (branch) { + entry.ref = branch; } if (e.results && typeof e.results === 'object') { const r = e.results as Partial; - const repoUrl = - typeof r.repo_url === 'string' && r.repo_url.trim().length > 0 - ? r.repo_url.trim() - : undefined; - const repoPath = - typeof r.repo_path === 'string' && r.repo_path.trim().length > 0 - ? r.repo_path.trim() - : undefined; + const resultsRepo = + r.repo && typeof r.repo === 'object' && !Array.isArray(r.repo) ? r.repo : undefined; + const repoUrl = readTrimmedString(resultsRepo?.url) ?? readTrimmedString(r.repo_url); + const repoPath = resultsRepo?.url + ? undefined + : (readTrimmedString(resultsRepo?.path) ?? readTrimmedString(r.repo_path)); + const clonePath = resultsRepo?.url + ? readTrimmedString(resultsRepo.path) + : readTrimmedString(r.path); + const resultsBranch = readTrimmedString(resultsRepo?.branch) ?? readTrimmedString(r.branch); + const resultsRemote = readTrimmedString(resultsRepo?.remote) ?? readTrimmedString(r.remote); if (repoUrl || repoPath) { const sync = r.sync && typeof r.sync === 'object' ? r.sync : undefined; entry.results = { ...(repoUrl ? { repoUrl } : {}), ...(repoPath ? { repoPath } : {}), - ...(typeof r.branch === 'string' && r.branch.trim().length > 0 - ? { branch: r.branch.trim() } - : {}), - ...(typeof r.remote === 'string' && r.remote.trim().length > 0 - ? { remote: r.remote.trim() } - : {}), - ...(typeof r.path === 'string' && r.path.trim().length > 0 ? { path: r.path.trim() } : {}), + ...(resultsBranch ? { branch: resultsBranch } : {}), + ...(resultsRemote ? { remote: resultsRemote } : {}), + ...(clonePath ? { path: clonePath } : {}), ...(sync && (typeof sync.auto_push === 'boolean' || typeof sync.require_push === 'boolean') ? { sync: { @@ -190,19 +215,28 @@ function toYaml(entry: ProjectEntry): ProjectEntryYaml { const yaml: ProjectEntryYaml = { id: entry.id, name: entry.name, - ...(entry.repoUrl !== undefined && { repo_url: entry.repoUrl }), - path: entry.path, - ...(entry.ref !== undefined && { ref: entry.ref }), + repo: { + ...(entry.repoUrl !== undefined && { url: entry.repoUrl }), + ...(entry.ref !== undefined && { branch: entry.ref }), + path: entry.path, + }, added_at: entry.addedAt, last_opened_at: entry.lastOpenedAt, }; if (entry.results) { - yaml.results = { - ...(entry.results.repoUrl !== undefined && { repo_url: entry.results.repoUrl }), - ...(entry.results.repoPath !== undefined && { repo_path: entry.results.repoPath }), + const resultsRepo: ProjectResultsRepoYaml = { + ...(entry.results.repoUrl !== undefined && { url: entry.results.repoUrl }), ...(entry.results.branch !== undefined && { branch: entry.results.branch }), ...(entry.results.remote !== undefined && { remote: entry.results.remote }), - ...(entry.results.path !== undefined && { path: entry.results.path }), + ...(entry.results.repoUrl && + entry.results.path !== undefined && { + path: entry.results.path, + }), + ...(entry.results.repoUrl === undefined && + entry.results.repoPath !== undefined && { path: entry.results.repoPath }), + }; + yaml.results = { + repo: resultsRepo, ...((entry.results.sync?.autoPush !== undefined || entry.results.sync?.requirePush !== undefined) && { sync: { diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index c09117671..f802589a7 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -327,6 +327,75 @@ describe('parseResultsConfig', () => { }); }); + it('parses nested repo config for a managed results clone', () => { + const result = parseResultsConfig( + { + repo: { + url: 'https://github.com/example/results.git', + branch: 'agentv/results/v1', + remote: 'origin', + path: '~/data/agentv-results', + }, + sync: { + auto_push: true, + }, + }, + '/tmp/.agentv/config.yaml', + ); + + expect(result).toEqual({ + mode: 'github', + repo: 'https://github.com/example/results.git', + repo_url: 'https://github.com/example/results.git', + branch: 'agentv/results/v1', + remote: 'origin', + path: '~/data/agentv-results', + sync: { + auto_push: true, + }, + }); + }); + + it('parses nested repo config for an existing local results checkout', () => { + const result = parseResultsConfig( + { + repo: { + path: '.', + branch: 'agentv/results/v1', + remote: 'origin', + }, + sync: { + require_push: true, + }, + }, + '/tmp/.agentv/config.yaml', + ); + + expect(result).toEqual({ + mode: 'github', + repo_path: '.', + branch: 'agentv/results/v1', + remote: 'origin', + sync: { + require_push: true, + }, + }); + }); + + it('returns undefined when nested repo config is mixed with flat repo fields', () => { + const result = parseResultsConfig( + { + repo: { + url: 'https://github.com/example/results.git', + }, + branch: 'agentv/results/v1', + }, + '/tmp/.agentv/config.yaml', + ); + + expect(result).toBeUndefined(); + }); + it('returns undefined when mode is not github', () => { const result = parseResultsConfig( { diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index a81636e82..be818b12f 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -51,10 +51,12 @@ describe('validateConfigFile', () => { await writeFile( filePath, `results: - mode: github - repo: EntityProcess/agentv-evals - branch: agentv-results - auto_push: true + repo: + url: https://github.com/EntityProcess/agentv-evals.git + branch: agentv-results + path: ~/data/agentv-results + sync: + auto_push: true branch_prefix: eval-results `, ); @@ -87,13 +89,15 @@ describe('validateConfigFile', () => { `projects: - id: agentv name: AgentV - repo_url: https://github.com/EntityProcess/agentv.git - path: /srv/agentv - ref: main + repo: + url: https://github.com/EntityProcess/agentv.git + branch: main + path: /srv/agentv results: - repo_url: git@github.com:EntityProcess/agentv-results.git - branch: agentv-results - path: /srv/agentv-results + repo: + url: git@github.com:EntityProcess/agentv-results.git + branch: agentv-results + path: /srv/agentv-results sync: auto_push: true branch_prefix: eval-results @@ -113,11 +117,13 @@ describe('validateConfigFile', () => { `projects: - id: agentv name: AgentV - path: /srv/agentv + repo: + path: /srv/agentv results: - repo_path: . - branch: agentv/results/v1 - remote: origin + repo: + path: . + branch: agentv/results/v1 + remote: origin sync: auto_push: false require_push: true @@ -130,6 +136,43 @@ describe('validateConfigFile', () => { expect(result.errors).toHaveLength(0); }); + it('keeps flat project repo fields compatible with migration warnings', async () => { + const filePath = path.join(tempDir, 'global-config-flat-project.yaml'); + await writeFile( + filePath, + `projects: + - id: agentv + name: AgentV + repo_url: https://github.com/EntityProcess/agentv.git + path: /srv/agentv + ref: main + results: + repo_url: git@github.com:EntityProcess/agentv-results.git + branch: agentv-results + path: /srv/agentv-results + sync: + auto_push: true +`, + ); + + const result = await validateConfigFile(filePath, { scope: 'global' }); + + expect(result.valid).toBe(true); + expect(result.errors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ severity: 'warning', location: 'projects[0].repo_url' }), + expect.objectContaining({ severity: 'warning', location: 'projects[0].path' }), + expect.objectContaining({ severity: 'warning', location: 'projects[0].ref' }), + expect.objectContaining({ + severity: 'warning', + location: 'projects[0].results.repo_url', + }), + expect.objectContaining({ severity: 'warning', location: 'projects[0].results.branch' }), + expect.objectContaining({ severity: 'warning', location: 'projects[0].results.path' }), + ]), + ); + }); + it('infers AGENTV_HOME config.yaml as global even when the home dir is named .agentv', async () => { const fakeHome = path.join(tempDir, 'fake-user-home'); const homeConfigDir = path.join(fakeHome, '.agentv'); @@ -140,7 +183,8 @@ describe('validateConfigFile', () => { `projects: - id: agentv name: AgentV - path: /srv/agentv + repo: + path: /srv/agentv `, ); @@ -186,14 +230,16 @@ describe('validateConfigFile', () => { `projects: - id: "" name: 42 - repo_url: EntityProcess/agentv - path: - ref: "" - results: - repo_url: EntityProcess/results + repo: + url: EntityProcess/agentv + path: branch: "" - remote: "" - path: repo/subdir + results: + repo: + url: EntityProcess/results + branch: "" + remote: "" + path: repo/subdir sync: auto_push: yes branch_prefix: "" @@ -208,13 +254,19 @@ describe('validateConfigFile', () => { expect.arrayContaining([ expect.objectContaining({ severity: 'error', location: 'projects[0].id' }), expect.objectContaining({ severity: 'error', location: 'projects[0].name' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].repo_url' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].path' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].ref' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].results.repo_url' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].results.branch' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].results.remote' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].results.path' }), + expect.objectContaining({ severity: 'error', location: 'projects[0].repo.url' }), + expect.objectContaining({ severity: 'error', location: 'projects[0].repo.path' }), + expect.objectContaining({ severity: 'error', location: 'projects[0].repo.branch' }), + expect.objectContaining({ severity: 'error', location: 'projects[0].results.repo.url' }), + expect.objectContaining({ + severity: 'error', + location: 'projects[0].results.repo.branch', + }), + expect.objectContaining({ + severity: 'error', + location: 'projects[0].results.repo.remote', + }), + expect.objectContaining({ severity: 'error', location: 'projects[0].results.repo.path' }), expect.objectContaining({ severity: 'error', location: 'projects[0].results.sync.auto_push', @@ -233,7 +285,7 @@ describe('validateConfigFile', () => { field: 'repository', yaml: 'repository: example/repo', location: 'projects[0].repository', - migration: 'repo_url', + migration: 'repo.url', }, { field: 'source', @@ -254,24 +306,25 @@ describe('validateConfigFile', () => { { field: 'results.repo', yaml: `results: - repo_url: https://github.com/example/results.git repo: example/legacy-results`, location: 'projects[0].results.repo', - migration: 'Use', + migration: 'repo.url', }, { field: 'results.repository', yaml: `results: - repo_url: https://github.com/example/results.git + repo: + url: https://github.com/example/results.git repository: example/results`, location: 'projects[0].results.repository', - migration: 'repo_url', + migration: 'repo.url', }, { field: 'results.local_path', yaml: `results: - repo_url: https://github.com/example/results.git + repo: + url: https://github.com/example/results.git local_path: /srv/results`, location: 'projects[0].results.local_path', migration: 'path', @@ -279,7 +332,8 @@ describe('validateConfigFile', () => { { field: 'results.auto_push', yaml: `results: - repo_url: https://github.com/example/results.git + repo: + url: https://github.com/example/results.git auto_push: true`, location: 'projects[0].results.auto_push', migration: 'sync.auto_push', @@ -291,9 +345,10 @@ describe('validateConfigFile', () => { `projects: - id: legacy name: Legacy - repo_url: https://github.com/example/repo.git - path: /srv/legacy - ref: main + repo: + url: https://github.com/example/repo.git + path: /srv/legacy + branch: main ${legacy.yaml} `, ); @@ -354,7 +409,9 @@ describe('validateConfigFile', () => { await writeFile( filePath, `results: - repo: EntityProcess/agentv-evals + repo: + path: . + branch: agentv/results/v1 `, ); @@ -364,6 +421,31 @@ describe('validateConfigFile', () => { expect(result.errors).toHaveLength(0); }); + it('keeps flat top-level results compatible with migration warnings', async () => { + const filePath = path.join(tempDir, 'config-results-flat-compatible.yaml'); + await writeFile( + filePath, + `results: + repo_url: https://github.com/EntityProcess/agentv-evals.git + branch: agentv-results + path: ~/data/agentv-results + sync: + auto_push: true +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ severity: 'warning', location: 'results.repo_url' }), + expect.objectContaining({ severity: 'warning', location: 'results.branch' }), + expect.objectContaining({ severity: 'warning', location: 'results.path' }), + ]), + ); + }); + it('errors on old-style subdirectory path', async () => { const filePath = path.join(tempDir, 'config-results-old-path.yaml'); await writeFile( diff --git a/packages/core/test/project-sync.test.ts b/packages/core/test/project-sync.test.ts index fa92d116f..f758f67cd 100644 --- a/packages/core/test/project-sync.test.ts +++ b/packages/core/test/project-sync.test.ts @@ -30,9 +30,9 @@ describe('syncProject', () => { mock.restore(); }); - it('throws when entry has no repo_url', async () => { + it('throws when entry has no repo.url', async () => { const entry = makeEntry({ path: tmpDir }); - await expect(syncProject(entry)).rejects.toThrow(/no repo_url defined/); + await expect(syncProject(entry)).rejects.toThrow(/no repo\.url defined/); }); it('runs git clone when .git does not exist', async () => { @@ -82,13 +82,13 @@ describe('syncProjects', () => { mock.restore(); }); - it('skips entries with no repo_url', async () => { + it('skips entries with no repo.url', async () => { const spy = spyOn(childProcess, 'execFileSync').mockReturnValue(Buffer.from('')); await syncProjects([makeEntry()]); expect(spy).not.toHaveBeenCalled(); }); - it('syncs entries that have a repo_url', async () => { + it('syncs entries that have a repo.url', async () => { const spy = spyOn(childProcess, 'execFileSync').mockReturnValue(Buffer.from('')); const entries = [makeEntry({ repoUrl: 'git@github.com:example/repo.git', ref: 'main' })]; await syncProjects(entries); diff --git a/packages/core/test/projects.test.ts b/packages/core/test/projects.test.ts index 65a0f8800..4b13d16a8 100644 --- a/packages/core/test/projects.test.ts +++ b/packages/core/test/projects.test.ts @@ -133,6 +133,34 @@ describe('projects registry', () => { expect(entry.ref).toBe('main'); }); + it('round-trips nested source repo fields through YAML', () => { + const registryPath = getProjectsRegistryPath(); + mkdirSync(path.dirname(registryPath), { recursive: true }); + writeFileSync( + registryPath, + `projects: + - id: nested-source + name: Nested Source + repo: + url: git@github.com:example/repo.git + branch: main + path: /srv/agentv/repo + added_at: "2026-01-01T00:00:00Z" + last_opened_at: "2026-01-01T00:00:00Z" +`, + 'utf-8', + ); + + const registry = loadProjectRegistry(); + expect(registry.projects).toHaveLength(1); + const entry = registry.projects[0]; + expect(entry).toMatchObject({ + repoUrl: 'git@github.com:example/repo.git', + ref: 'main', + path: '/srv/agentv/repo', + }); + }); + it('round-trips project results config through YAML', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); @@ -166,10 +194,13 @@ describe('projects registry', () => { saveProjectRegistry(registry); const yamlOnDisk = readFileSync(registryPath, 'utf-8'); + expect(yamlOnDisk).toContain('repo:'); + expect(yamlOnDisk).toContain('url: https://github.com/EntityProcess/results-project-runs.git'); expect(yamlOnDisk).toContain('branch: agentv-results'); expect(yamlOnDisk).toContain('path: /srv/agentv/results/results-project'); expect(yamlOnDisk).toContain('auto_push: true'); expect(yamlOnDisk).toContain('branch_prefix: eval-results'); + expect(yamlOnDisk).not.toContain('repo_url:'); expect(yamlOnDisk).not.toContain('localPath:'); expect(yamlOnDisk).not.toContain('local_path:'); expect(yamlOnDisk).not.toContain('autoPush:'); @@ -208,11 +239,13 @@ describe('projects registry', () => { saveProjectRegistry(registry); const yamlOnDisk = readFileSync(registryPath, 'utf-8'); - expect(yamlOnDisk).toContain('repo_path: .'); + expect(yamlOnDisk).toContain('repo:'); + expect(yamlOnDisk).toContain('path: .'); expect(yamlOnDisk).toContain('branch: agentv/results/v1'); expect(yamlOnDisk).toContain('remote: origin'); expect(yamlOnDisk).toContain('auto_push: false'); expect(yamlOnDisk).toContain('require_push: true'); + expect(yamlOnDisk).not.toContain('repo_path:'); expect(yamlOnDisk).not.toContain('repoPath:'); expect(yamlOnDisk).not.toContain('requirePush:'); }); diff --git a/scripts/deploy-example-results-repos.ts b/scripts/deploy-example-results-repos.ts index 2eec8a02f..51cef2f08 100644 --- a/scripts/deploy-example-results-repos.ts +++ b/scripts/deploy-example-results-repos.ts @@ -132,6 +132,10 @@ function repoSlug(repo: string): string { .toLowerCase(); } +function repoUrl(repo: string): string { + return `https://github.com/${repo}.git`; +} + function assertRepoName(value: string, field: string): void { if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(value)) { throw new Error(`${field} must be GitHub owner/name, got: ${value}`); @@ -416,10 +420,13 @@ This repo is configured to push evaluation results to \`${params.resultsRepo}\`. const configPath = path.join(params.repoDir, '.agentv', 'config.yaml'); const config = readYamlObject(configPath); config.results = { - mode: 'github', - repo: params.resultsRepo, - path: params.resultsPath, - auto_push: params.autoPush, + repo: { + url: repoUrl(params.resultsRepo), + path: params.resultsPath, + }, + sync: { + auto_push: params.autoPush, + }, ...(params.branchPrefix ? { branch_prefix: params.branchPrefix } : {}), }; diff --git a/scripts/setup-dashboard-deployment.sh b/scripts/setup-dashboard-deployment.sh index 9392cc64b..53dec801a 100755 --- a/scripts/setup-dashboard-deployment.sh +++ b/scripts/setup-dashboard-deployment.sh @@ -129,9 +129,10 @@ eval_patterns: - "examples/**/dataset*.yaml" results: - repo_url: $results_url - branch: agentv/results/v1 - path: /data/results/agentv-evalresults + repo: + url: $results_url + branch: agentv/results/v1 + path: /data/results/agentv-evalresults sync: auto_push: false require_push: false @@ -162,11 +163,15 @@ const existingEntry = projects.find((project) => { const nextEntry = { id: "agentv-examples", name: "AgentV Examples", - path: "/data/projects/agentv-examples", + repo: { + path: "/data/projects/agentv-examples", + }, results: { - repo_url: resultsRepo, - branch: "agentv/results/v1", - path: "/data/results/agentv-evalresults", + repo: { + url: resultsRepo, + branch: "agentv/results/v1", + path: "/data/results/agentv-evalresults", + }, sync: { auto_push: false, require_push: false, diff --git a/skills-data/agentv-eval-writer/references/config-schema.json b/skills-data/agentv-eval-writer/references/config-schema.json index f95da9c49..fd016c8a8 100644 --- a/skills-data/agentv-eval-writer/references/config-schema.json +++ b/skills-data/agentv-eval-writer/references/config-schema.json @@ -45,6 +45,16 @@ }, "additionalProperties": false }, + "results": { + "$ref": "#/definitions/resultsConfig" + }, + "projects": { + "type": "array", + "description": "Global Dashboard project registry entries. Only valid in $AGENTV_HOME/config.yaml.", + "items": { + "$ref": "#/definitions/projectEntry" + } + }, "hooks": { "type": "object", "description": "Lifecycle hooks that run at specific points during agentv execution.", @@ -58,6 +68,90 @@ "additionalProperties": false } }, + "definitions": { + "repoBlock": { + "type": "object", + "description": "Nested repository block. Use url for Git URLs, path for local checkout or clone paths, branch for the checked-out/storage branch, and remote for a Git remote name such as origin.", + "properties": { + "url": { + "type": "string", + "description": "Git remote URL used for cloning or pushing.", + "examples": ["https://github.com/example/repo.git", "git@github.com:example/repo.git"] + }, + "path": { + "type": "string", + "description": "Local checkout path. For results repos, this is an existing local repo when url is omitted, or the managed clone path when url is set.", + "examples": ["/srv/agentv/my-evals", ".", "~/data/agentv-results"] + }, + "branch": { + "type": "string", + "description": "Source checkout branch or results storage branch.", + "examples": ["main", "agentv/results/v1"] + }, + "remote": { + "type": "string", + "description": "Git remote name for an existing local checkout, such as origin.", + "examples": ["origin"] + } + }, + "additionalProperties": false + }, + "resultsConfig": { + "type": "object", + "description": "Git-backed results repo configuration. Canonical form uses a nested repo block.", + "properties": { + "repo": { + "$ref": "#/definitions/repoBlock" + }, + "sync": { + "type": "object", + "properties": { + "auto_push": { + "type": "boolean", + "description": "Push result commits best-effort after each completed run." + }, + "require_push": { + "type": "boolean", + "description": "Fail the command if a configured push fails after writing local artifacts." + } + }, + "additionalProperties": false + }, + "branch_prefix": { + "type": "string", + "description": "Prefix for temporary result branch names; this is not the storage branch." + } + }, + "required": ["repo"], + "additionalProperties": false + }, + "projectEntry": { + "type": "object", + "description": "Dashboard project registry entry.", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "repo": { + "$ref": "#/definitions/repoBlock" + }, + "results": { + "$ref": "#/definitions/resultsConfig" + }, + "added_at": { + "type": "string" + }, + "last_opened_at": { + "type": "string" + } + }, + "required": ["id", "name", "repo"], + "additionalProperties": false + } + }, "required": ["$schema"], "additionalProperties": false } From 238d957064deeb54f47d63e875247fb905474644 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 17:48:03 +1000 Subject: [PATCH 04/12] fix(results): block conflicting metadata sync --- packages/core/src/evaluation/results-repo.ts | 66 +++++++++++++++++++ .../core/test/evaluation/results-repo.test.ts | 50 ++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index 295a8cb0c..eeaac8b26 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -868,6 +868,44 @@ async function fastForwardStorageBranchRef( }); } +async function getDirtyPathsChangedUpstream( + repoDir: string, + normalized: StorageBranchResultsConfig, + dirtyPaths: readonly string[], + upstream: string, +): Promise { + const safeDirtyPaths = [...new Set(dirtyPaths.filter(isSafeResultsRepoPath))].sort(); + if (safeDirtyPaths.length === 0) { + return []; + } + + const localRef = `refs/heads/${normalized.branch}`; + const localExists = await gitRefExists(repoDir, localRef); + let changedPaths: string[]; + if (localExists) { + const { stdout } = await runGit(['diff', '--name-only', `${localRef}..${upstream}`], { + cwd: repoDir, + check: false, + }); + changedPaths = stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .filter(isSafeResultsRepoPath) + .sort(); + } else { + changedPaths = []; + for (const dirtyPath of safeDirtyPaths) { + if ((await readGitText(repoDir, upstream, dirtyPath)) !== undefined) { + changedPaths.push(dirtyPath); + } + } + } + + const dirtySet = new Set(safeDirtyPaths); + return [...new Set(changedPaths.filter((changedPath) => dirtySet.has(changedPath)))].sort(); +} + async function hasInProgressGitConflict(repoDir: string): Promise { const markers = ['MERGE_HEAD', 'CHERRY_PICK_HEAD', 'REVERT_HEAD', 'REBASE_HEAD']; for (const marker of markers) { @@ -1282,6 +1320,34 @@ export async function syncResultsRepoForProject(config: ResultsConfig): Promise< } if ((inspection.behind ?? 0) > 0) { + if (inspection.upstream && normalized.branch) { + const changedDirtyPaths = await getDirtyPathsChangedUpstream( + repoDir, + { ...normalized, branch: normalized.branch }, + inspection.dirtyPaths, + inspection.upstream, + ); + if (changedDirtyPaths.length > 0) { + const conflictInspection: ResultsRepoGitInspection = { + ...inspection, + syncStatus: 'conflicted', + conflictedPaths: [ + ...new Set([...inspection.conflictedPaths, ...changedDirtyPaths]), + ].sort(), + }; + const status = withGitInspection( + getResultsRepoStatus(normalized), + conflictInspection, + ); + const reason = `Results repo local metadata changes conflict with upstream changes: ${changedDirtyPaths.join(', ')}`; + updateStatusFile(normalized, { last_error: reason }); + return withBlockedStatus(status, reason, { + pullPerformed, + pushPerformed, + commitCreated, + }); + } + } const blockedStatus = await fastForwardBehindStorageBranch(); if (blockedStatus) { return blockedStatus; diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index 70e958fe0..05e0b0c71 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -855,6 +855,56 @@ describe('results repo write path', () => { expect(git('git branch --show-current', projectDir)).toBe('main'); }, 20000); + it('blocks repo_path metadata sync when upstream changed the same dirty path', async () => { + const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); + const storageBranch = initializeRemoteStorageBranch(seedDir, DEFAULT_RESULTS_BRANCH); + const projectDir = path.join(rootDir, 'source-project-repo-path-dirty-ff-conflict'); + git(`git clone --quiet "${remoteDir}" "${projectDir}"`, rootDir); + git('git config user.email "test@example.com"', projectDir); + git('git config user.name "Test User"', projectDir); + git( + `git fetch --quiet origin refs/heads/${storageBranch}:refs/heads/${storageBranch}`, + projectDir, + ); + + const metadataPath = 'metadata/runs/shared/2026-06-22T03-00-00-000Z/tags.json'; + git(`git switch --quiet ${storageBranch}`, seedDir); + const remoteTagPath = path.join(seedDir, ...metadataPath.split('/')); + mkdirSync(path.dirname(remoteTagPath), { recursive: true }); + writeFileSync(remoteTagPath, `${JSON.stringify({ tags: ['remote'] }, null, 2)}\n`); + git('git add metadata && git commit --quiet -m "remote shared tag metadata"', seedDir); + git(`git push --quiet origin HEAD:${storageBranch}`, seedDir); + + const localTagPath = path.join(projectDir, ...metadataPath.split('/')); + mkdirSync(path.dirname(localTagPath), { recursive: true }); + writeFileSync(localTagPath, `${JSON.stringify({ tags: ['local'] }, null, 2)}\n`); + + const status = await syncResultsRepoForProject({ + repo_path: projectDir, + branch: storageBranch, + remote: 'origin', + sync: { auto_push: true }, + }); + + expect(status).toMatchObject({ + sync_status: 'conflicted', + pull_performed: false, + push_performed: false, + commit_created: false, + blocked: true, + branch: storageBranch, + upstream: `origin/${storageBranch}`, + dirty_paths: [metadataPath], + conflicted_paths: [metadataPath], + }); + expect(status.block_reason).toContain(metadataPath); + expect( + git(`git --git-dir "${remoteDir}" show ${storageBranch}:${metadataPath}`, rootDir), + ).toContain('"remote"'); + expect(readFileSync(localTagPath, 'utf8')).toContain('"local"'); + expect(git('git branch --show-current', projectDir)).toBe('main'); + }, 20000); + it('reports repo_path metadata push rejection without dropping the local commit', async () => { const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); const storageBranch = initializeRemoteStorageBranch(seedDir, DEFAULT_RESULTS_BRANCH); From a860e58946529a2cdb2b4898f6b8c7907ca2dec0 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 18:03:56 +1000 Subject: [PATCH 05/12] fix(results): preserve storage ref on tag no-op --- .../src/commands/results/remote-metadata.ts | 2 +- .../commands/results/remote-metadata.test.ts | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/apps/cli/src/commands/results/remote-metadata.ts b/apps/cli/src/commands/results/remote-metadata.ts index e41287ffb..fa46d41d8 100644 --- a/apps/cli/src/commands/results/remote-metadata.ts +++ b/apps/cli/src/commands/results/remote-metadata.ts @@ -288,7 +288,7 @@ export function writeRemoteRunTags( context.baseOverlayTags === undefined ) { rmSync(context.paths.overlayTagsPath, { force: true }); - return readRemoteRunTags(repoDir, manifestPath); + return readRemoteRunTags(repoDir, manifestPath, comparisonRef); } const operation = createRunTagsSetOperation({ diff --git a/apps/cli/test/commands/results/remote-metadata.test.ts b/apps/cli/test/commands/results/remote-metadata.test.ts index 44fe86dbc..5fe26b51f 100644 --- a/apps/cli/test/commands/results/remote-metadata.test.ts +++ b/apps/cli/test/commands/results/remote-metadata.test.ts @@ -115,6 +115,37 @@ describe('remote metadata tags', () => { expect(reloaded.oplogWatermark.ref).toBe(RUN_OPLOG_REF); }); + it('preserves the storage ref when unchanged remote artifact tags clear a local overlay', () => { + git('git init --quiet', repoDir); + git('git config user.email "test@example.com"', repoDir); + git('git config user.name "Test User"', repoDir); + writeFileSync(path.join(repoDir, 'README.md'), '# source branch\n'); + git('git add README.md', repoDir); + git('git commit --quiet -m "seed source branch"', repoDir); + git('git branch -M main', repoDir); + + git('git checkout --quiet -b agentv/results/v1', repoDir); + const runDir = path.join(repoDir, 'runs', 'default', RUN_TIMESTAMP); + mkdirSync(runDir, { recursive: true }); + writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id":"alpha","score":1}\n'); + writeFileSync( + path.join(runDir, 'tags.json'), + `${JSON.stringify({ tags: ['remote'], updated_at: '2026-06-06T09:00:00.000Z' }, null, 2)}\n`, + ); + git('git add runs', repoDir); + git('git commit --quiet -m "seed result artifact"', repoDir); + git('git checkout --quiet main', repoDir); + + const manifestPath = path.join(runDir, 'index.jsonl'); + const state = writeRemoteRunTags(repoDir, manifestPath, ['remote'], 'agentv/results/v1'); + + expect(state.tags).toEqual(['remote']); + expect(state.remoteTags).toEqual(['remote']); + expect(state.pendingTags).toBeUndefined(); + expect(state.dirty).toBe(false); + expect(existsSync(state.metadataPath)).toBe(false); + }); + it('persists clearing remote tags as an empty pending overlay', () => { const manifestPath = seedRepo(repoDir); From 5d3f1e317b0d22755f9816df8c8d3ceb01916edd Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 19:29:18 +1000 Subject: [PATCH 06/12] fix(results): support portable results repo URLs --- .../src/content/docs/docs/tools/dashboard.mdx | 8 +-- .../src/content/docs/docs/tools/results.mdx | 2 +- .../src/evaluation/loaders/config-loader.ts | 8 +-- packages/core/src/evaluation/results-repo.ts | 62 +++++++++++++--- .../evaluation/validation/config-validator.ts | 15 +--- packages/core/src/projects.ts | 2 +- .../evaluation/loaders/config-loader.test.ts | 27 +++++++ .../core/test/evaluation/results-repo.test.ts | 72 ++++++++++++++++++- .../validation/config-validator.test.ts | 6 +- packages/core/test/projects.test.ts | 18 +++-- 10 files changed, 170 insertions(+), 50 deletions(-) diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 898ce21f5..8b9089626 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -284,15 +284,15 @@ projects: path: /home/entity/projects/EntityProcess/agentv results: repo: + url: https://github.com/EntityProcess/agentv.git path: . branch: agentv/results/v1 - remote: origin sync: auto_push: false require_push: false ``` -`results.repo.path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. When `results.repo.url` is omitted, `results.repo.path` means an existing local Git checkout whose object database and refs AgentV should write to. `results.repo.remote` is the Git remote name, such as `origin`, not a URL. The branch defaults to `agentv/results/v1` for local checkout configs. AgentV creates the branch automatically on first publish and commits only `.agentv/results/**` paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. `sync.require_push: true` is for CI workflows where a push failure should fail the command after local artifacts are written. +`results.repo.url` is the Git remote URL AgentV fetches and pushes. `results.repo.path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. AgentV manages the local Git remote alias for that URL, so the normal config does not need `results.repo.remote`. Use `results.repo.remote` only as an advanced compatibility override when an existing checkout must use a specific local Git remote name. When `results.repo.url` is omitted, `results.repo.path` means an existing local Git checkout whose object database and refs AgentV should write to, and the branch defaults to `agentv/results/v1`. AgentV creates the branch automatically on first publish and commits only AgentV result paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. `sync.require_push: true` is for CI workflows where a push failure should fail the command after local artifacts are written. For a separate results repository, use `results.repo.url` and an optional managed clone `results.repo.path`: @@ -318,9 +318,9 @@ You can also set a top-level global fallback in the same file. This is used when ```yaml results: repo: + url: https://github.com/EntityProcess/agentv.git path: . branch: agentv/results/v1 - remote: origin sync: auto_push: false require_push: false @@ -331,7 +331,7 @@ Project-local `.agentv/config.yaml` is for portable eval defaults such as `execu The project `repo` block and the `results` block sync different repositories: - `projects[].repo.url` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current. -- `projects[].results.repo.url` is the git-backed results store remote. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in that results repo clone. +- `projects[].results.repo.url` is the git-backed results store remote. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in the local checkout at `projects[].results.repo.path`. #### Migration from the legacy project schema diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index 3a4025e89..9e9ffff2d 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -228,7 +228,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.path: .` with `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. `results.repo.path` without `results.repo.url` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.url` with `repo.path: .` and `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo without requiring a machine-local Git remote name. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. `results.repo.path` without `results.repo.url` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. `results.repo.remote` is only an advanced local remote-name override. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. - **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.repo.path` clone with `git` directly, then sync again. diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 89ae25d66..700f06215 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -651,12 +651,6 @@ function parseNestedResultsRepoConfig( logWarning(`Invalid results.repo in ${configPath}, expected url or path`); return undefined; } - if (url && repoPath && !isFilesystemPath(repoPath)) { - logWarning( - `Invalid results.repo.path in ${configPath}: '${repoPath}' must be an absolute or home-relative filesystem path when results.repo.url is set.`, - ); - return undefined; - } if (repo.branch !== undefined && !branch) { logWarning(`Invalid results.repo.branch in ${configPath}, expected non-empty string`); return undefined; @@ -667,7 +661,7 @@ function parseNestedResultsRepoConfig( } if (remote && isGitRemoteUrl(remote)) { logWarning( - `Invalid results.repo.remote in ${configPath}: use results.repo.url for Git remote URLs and results.repo.remote for the remote name such as origin.`, + `Invalid results.repo.remote in ${configPath}: use results.repo.url for Git remote URLs and results.repo.remote only for an optional local Git remote name override.`, ); return undefined; } diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index eeaac8b26..3a6681bb3 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -51,6 +51,7 @@ const GIT_ENV_INHERIT_ALLOWLIST = new Set([ 'GIT_USERNAME', ]); export const DEFAULT_RESULTS_BRANCH = AGENTV_RESULTS_PRIMARY_REF; +const MANAGED_RESULTS_REMOTE = 'agentv-results'; const GIT_EMPTY_TREE = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'; // The results branch is a self-rooted orphan whose first commit is a fixed, // byte-identical empty-tree genesis. Pinning the message, identity, and dates @@ -118,6 +119,8 @@ export interface NormalizedResultsConfig { readonly auto_push: boolean; readonly require_push: boolean; readonly branch_prefix: string; + /** @internal Runtime mode; not part of YAML wire format. */ + readonly storageBranchWorktree: boolean; } type StorageBranchResultsConfig = NormalizedResultsConfig & { readonly branch: string }; @@ -176,22 +179,30 @@ function expandHome(p: string): string { return p; } +function resolveLocalPath(p: string, baseDir: string): string { + const expanded = expandHome(p); + return path.isAbsolute(expanded) ? expanded : path.resolve(baseDir, expanded); +} + export function normalizeResultsConfig( config: ResultsConfig, options?: { baseDir?: string }, ): NormalizedResultsConfig { + const baseDir = options?.baseDir ?? process.cwd(); const repoUrl = (config.repo_url ?? config.repo)?.trim(); const repoPath = config.repo_path?.trim(); + const explicitClonePath = config.path?.trim(); const repo = repoUrl ?? repoPath ?? ''; const branch = config.branch?.trim() || (repoPath ? DEFAULT_RESULTS_BRANCH : undefined); - const remote = config.remote?.trim() || 'origin'; + const useStorageBranchWorktree = Boolean(repoPath || (repoUrl && explicitClonePath && branch)); + const remote = + config.remote?.trim() || + (repoUrl && useStorageBranchWorktree ? MANAGED_RESULTS_REMOTE : 'origin'); const autoPush = config.sync?.auto_push ?? config.auto_push === true; const requirePush = config.sync?.require_push === true; - const resolvedRepoPath = repoPath - ? path.resolve(options?.baseDir ?? process.cwd(), expandHome(repoPath)) - : undefined; - const resolvedPath = config.path - ? expandHome(config.path.trim()) + const resolvedRepoPath = repoPath ? resolveLocalPath(repoPath, baseDir) : undefined; + const resolvedPath = explicitClonePath + ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ? path.join(getAgentvDataDir(), 'results', sanitizeRepoSlug(repoUrl)) : (resolvedRepoPath ?? path.join(getAgentvDataDir(), 'results', sanitizeRepoSlug(repo))); @@ -206,6 +217,7 @@ export function normalizeResultsConfig( auto_push: autoPush, require_push: requirePush, branch_prefix: config.branch_prefix?.trim() || 'eval-results', + storageBranchWorktree: useStorageBranchWorktree, }; } @@ -225,6 +237,10 @@ export function getResultsRepoLocalPaths(repo: string): ResultsRepoLocalPaths { }; } +function usesStorageBranchWorktree(config: NormalizedResultsConfig): boolean { + return config.storageBranchWorktree === true || Boolean(config.repo_path); +} + function readPersistedStatus(statusFile: string): PersistedStatus { if (!existsSync(statusFile)) { return {}; @@ -537,6 +553,29 @@ async function resolveGitTopLevel(repoDir: string): Promise { return stdout.trim() || repoDir; } +async function ensureResultsRepoRemote( + repoDir: string, + config: NormalizedResultsConfig, +): Promise { + if (!config.repo_url) { + return; + } + + const remoteUrl = resolveResultsRepoUrl(config.repo_url); + const { stdout } = await runGit(['remote', 'get-url', config.remote], { + cwd: repoDir, + check: false, + }); + const existingUrl = stdout.trim(); + if (!existingUrl) { + await runGit(['remote', 'add', config.remote, remoteUrl], { cwd: repoDir }); + return; + } + if (existingUrl !== remoteUrl) { + await runGit(['remote', 'set-url', config.remote, remoteUrl], { cwd: repoDir }); + } +} + function updateStatusFile( config: ResultsConfig | NormalizedResultsConfig, patch: PersistedStatus, @@ -577,6 +616,7 @@ export async function ensureResultsRepoClone(config: ResultsConfig): Promise undefined, ); @@ -1186,7 +1228,7 @@ export async function syncResultsRepo(config: ResultsConfig): Promise 0 && - !isFilesystemPath(repoRecord.path.trim()) - ) { - addError( - errors, - filePath, - `${location}.path`, - `'${location}.path' must be an absolute or home-relative filesystem path when '${location}.url' is set.`, - ); - } } if ( @@ -421,7 +408,7 @@ function validateResultsRepoBlock( errors, filePath, `${location}.remote`, - `Field '${location}.remote' is the Git remote name (for example, origin). Use '${location}.url' for Git remote URLs.`, + `Field '${location}.remote' is an optional local Git remote name override. Use '${location}.url' for Git remote URLs.`, ); } } diff --git a/packages/core/src/projects.ts b/packages/core/src/projects.ts index ad7827307..ddc59dc06 100644 --- a/packages/core/src/projects.ts +++ b/packages/core/src/projects.ts @@ -22,9 +22,9 @@ * path: /home/user/projects/my-app * results: * repo: + * url: https://github.com/example/my-app.git * path: . * branch: agentv/results/v1 - * remote: origin * sync: * auto_push: true * require_push: false diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index f802589a7..68d10621c 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -356,6 +356,33 @@ describe('parseResultsConfig', () => { }); }); + it('parses nested repo config for a URL-backed source storage branch', () => { + const result = parseResultsConfig( + { + repo: { + url: 'https://github.com/example/source.git', + path: '.', + branch: 'agentv/results/v1', + }, + sync: { + auto_push: true, + }, + }, + '/tmp/.agentv/config.yaml', + ); + + expect(result).toEqual({ + mode: 'github', + repo: 'https://github.com/example/source.git', + repo_url: 'https://github.com/example/source.git', + path: '.', + branch: 'agentv/results/v1', + sync: { + auto_push: true, + }, + }); + }); + it('parses nested repo config for an existing local results checkout', () => { const result = parseResultsConfig( { diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index 05e0b0c71..a1e0e5cd1 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -494,6 +494,27 @@ describe('results repo write path', () => { expect(normalized.branch).toBe('agentv/results/v1'); expect(normalized.repo_path).toBe('/tmp/source-project'); expect(normalized.auto_push).toBe(false); + expect(normalized.remote).toBe('origin'); + }); + + it('normalizes URL-backed source storage branch config without requiring a local remote name', () => { + const normalized = normalizeResultsConfig( + { + repo_url: 'https://github.com/example/source.git', + path: '.', + branch: DEFAULT_RESULTS_BRANCH, + sync: { auto_push: true }, + }, + { baseDir: '/tmp/source-project' }, + ); + + expect(normalized.repo).toBe('https://github.com/example/source.git'); + expect(normalized.repo_url).toBe('https://github.com/example/source.git'); + expect(normalized.repo_path).toBeUndefined(); + expect(normalized.path).toBe('/tmp/source-project'); + expect(normalized.branch).toBe(DEFAULT_RESULTS_BRANCH); + expect(normalized.remote).toBe('agentv-results'); + expect(normalized.auto_push).toBe(true); }); it('publishes current-repo results to an auto-created branch without switching source checkout', async () => { @@ -688,6 +709,50 @@ describe('results repo write path', () => { ); }, 20000); + it('publishes URL-backed source results through a managed remote alias', async () => { + const { remoteDir } = initializeRemoteRepo(rootDir); + const projectDir = path.join(rootDir, 'source-project-url-path'); + mkdirSync(projectDir, { recursive: true }); + git('git init --initial-branch=main --quiet', projectDir); + git('git config user.email "test@example.com"', projectDir); + git('git config user.name "Test User"', projectDir); + writeFileSync(path.join(projectDir, 'README.md'), '# source project\n'); + git('git add README.md && git commit --quiet -m "seed source"', projectDir); + + const runTimestamp = '2026-06-22T04-00-00-000Z'; + const runDir = path.join( + projectDir, + '.agentv', + 'results', + 'runs', + 'url-backed-source', + runTimestamp, + ); + writeRunArtifacts(runDir, 'url-backed-source', '2026-06-22T04:00:00.000Z'); + + const published = await directPushResults({ + config: { + repo_url: `file://${remoteDir}`, + path: projectDir, + branch: DEFAULT_RESULTS_BRANCH, + sync: { auto_push: true }, + }, + sourceDir: runDir, + destinationPath: path.join('url-backed-source', runTimestamp), + commitMessage: 'feat(results): url-backed-source - 1/1 PASS (1.000)', + }); + + expect(published).toBe(true); + expect(git('git remote get-url agentv-results', projectDir)).toBe(`file://${remoteDir}`); + expect(git('git branch --show-current', projectDir)).toBe('main'); + const remoteFiles = git( + `git --git-dir "${remoteDir}" ls-tree -r --name-only ${DEFAULT_RESULTS_BRANCH}`, + rootDir, + ); + expect(remoteFiles).toContain(`runs/url-backed-source/${runTimestamp}/benchmark.json`); + expect(remoteFiles).not.toContain('README.md'); + }, 20000); + it('commits repo_path metadata overlays to the configured storage branch during sync', async () => { const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); const storageBranch = initializeRemoteStorageBranch(seedDir, DEFAULT_RESULTS_BRANCH); @@ -1395,11 +1460,12 @@ describe('results repo write path', () => { commit_created: false, blocked: false, branch: storageBranch, - upstream: `origin/${storageBranch}`, + upstream: `agentv-results/${storageBranch}`, }); - expect(readFileSync(path.join(cloneDir, 'REMOTE_BRANCH.md'), 'utf8')).toBe( - 'branch remote update\n', + expect(git(`git show ${storageBranch}:REMOTE_BRANCH.md`, cloneDir)).toBe( + 'branch remote update', ); + expect(git('git branch --show-current', cloneDir)).toBe('main'); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).not.toContain( 'REMOTE_BRANCH.md', ); diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index be818b12f..098d5d210 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -110,7 +110,7 @@ describe('validateConfigFile', () => { expect(result.errors).toHaveLength(0); }); - it('accepts branch-backed repo_path results in global project config', async () => { + it('accepts URL-backed source storage branch results in global project config', async () => { const filePath = path.join(tempDir, 'global-config-repo-path.yaml'); await writeFile( filePath, @@ -118,12 +118,13 @@ describe('validateConfigFile', () => { - id: agentv name: AgentV repo: + url: https://github.com/EntityProcess/agentv.git path: /srv/agentv results: repo: + url: https://github.com/EntityProcess/agentv.git path: . branch: agentv/results/v1 - remote: origin sync: auto_push: false require_push: true @@ -266,7 +267,6 @@ describe('validateConfigFile', () => { severity: 'error', location: 'projects[0].results.repo.remote', }), - expect.objectContaining({ severity: 'error', location: 'projects[0].results.repo.path' }), expect.objectContaining({ severity: 'error', location: 'projects[0].results.sync.auto_push', diff --git a/packages/core/test/projects.test.ts b/packages/core/test/projects.test.ts index 4b13d16a8..e1eca3008 100644 --- a/packages/core/test/projects.test.ts +++ b/packages/core/test/projects.test.ts @@ -215,11 +215,14 @@ describe('projects registry', () => { `projects: - id: branch-results name: Branch Results - path: /srv/agentv/repo + repo: + url: git@github.com:example/source.git + path: /srv/agentv/repo results: - repo_path: . - branch: agentv/results/v1 - remote: origin + repo: + url: git@github.com:example/source.git + path: . + branch: agentv/results/v1 sync: auto_push: false require_push: true @@ -231,18 +234,19 @@ describe('projects registry', () => { const registry = loadProjectRegistry(); expect(registry.projects[0].results).toEqual({ - repoPath: '.', + repoUrl: 'git@github.com:example/source.git', + path: '.', branch: 'agentv/results/v1', - remote: 'origin', sync: { autoPush: false, requirePush: true }, }); saveProjectRegistry(registry); const yamlOnDisk = readFileSync(registryPath, 'utf-8'); expect(yamlOnDisk).toContain('repo:'); + expect(yamlOnDisk).toContain('url: git@github.com:example/source.git'); expect(yamlOnDisk).toContain('path: .'); expect(yamlOnDisk).toContain('branch: agentv/results/v1'); - expect(yamlOnDisk).toContain('remote: origin'); + expect(yamlOnDisk).not.toContain('remote: origin'); expect(yamlOnDisk).toContain('auto_push: false'); expect(yamlOnDisk).toContain('require_push: true'); expect(yamlOnDisk).not.toContain('repo_path:'); From 445c0405b13ec0554e828145c4aebdd30a18619f Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 19:53:40 +1000 Subject: [PATCH 07/12] fix(dashboard): read portable results branch refs --- apps/cli/src/commands/results/remote.ts | 7 ++++- apps/cli/test/commands/results/serve.test.ts | 18 +++++++----- packages/core/src/evaluation/results-repo.ts | 31 ++++++++++++++------ 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/apps/cli/src/commands/results/remote.ts b/apps/cli/src/commands/results/remote.ts index bba92030a..5670a6aaa 100644 --- a/apps/cli/src/commands/results/remote.ts +++ b/apps/cli/src/commands/results/remote.ts @@ -43,7 +43,12 @@ const gitRunsCache = new Map; expiresAt: const GIT_RUNS_CACHE_TTL_MS = 60_000; function getResultsStorageRef(config: NormalizedResultsConfig): string | undefined { - return config.branch; + if (!config.branch) { + return undefined; + } + return config.storageBranchWorktree + ? `refs/remotes/${config.remote}/${config.branch}` + : config.branch; } function cachedListGitRuns(repoDir: string, ref?: string) { diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 9ef79def3..fa152dad8 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -1542,8 +1542,13 @@ describe('serve app', () => { run_count: 0, }); expect(statusData.last_error).toBeUndefined(); - expect(git('git branch --show-current', cloneDir)).toBe('agentv-results'); - expect(git('git ls-tree -r --name-only agentv-results', cloneDir)).toBe(''); + expect(git('git branch --show-current', cloneDir)).toBe('main'); + expect( + git( + 'git show-ref --verify --quiet refs/remotes/agentv-results/agentv-results && echo present || true', + cloneDir, + ), + ).toBe(''); const listRes = await app.request('/api/runs'); expect(listRes.status).toBe(200); @@ -2333,7 +2338,8 @@ describe('serve app', () => { process.env.AGENTV_HOME = homeDir; try { - const { remoteDir, cloneDir } = initializeRemoteRepo(tempDir); + const { cloneDir } = initializeRemoteRepo(tempDir); + const missingRemoteUrl = `file://${path.join(tempDir, 'missing-results-remote.git')}`; const projectDir = path.join(tempDir, 'source-project-sync-offline'); mkdirSync(path.join(projectDir, '.agentv'), { recursive: true }); mkdirSync(homeDir, { recursive: true }); @@ -2344,7 +2350,7 @@ describe('serve app', () => { name: 'Project Sync Offline', path: projectDir, results: { - repoUrl: `file://${remoteDir}`, + repoUrl: missingRemoteUrl, path: cloneDir, sync: { autoPush: true }, }, @@ -2359,8 +2365,6 @@ describe('serve app', () => { '2026-03-26T12-30-00-000Z', RESULT_A, ); - git('git remote set-url origin "file:///tmp/agentv-missing-results-remote.git"', cloneDir); - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); const res = await app.request('/api/projects/project-sync-offline/remote/sync', { method: 'POST', @@ -3382,7 +3386,7 @@ describe('serve app', () => { `, ); - const artifactRemoteRef = `refs/remotes/origin/${AGENTV_RESULTS_ARTIFACTS_REF}`; + const artifactRemoteRef = `refs/remotes/agentv-results/${AGENTV_RESULTS_ARTIFACTS_REF}`; const artifactRefLookup = () => git( `git -C "${cloneDir}" show-ref --verify --quiet ${artifactRemoteRef} && echo present || true`, diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index 3a6681bb3..97bd28746 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -262,19 +262,20 @@ async function runCommand( executable: string, args: readonly string[], options?: { cwd?: string; check?: boolean; env?: NodeJS.ProcessEnv }, -): Promise<{ stdout: string; stderr: string }> { +): Promise<{ stdout: string; stderr: string; exitCode: number }> { try { const { stdout, stderr } = await execFileAsync(executable, [...args], { cwd: options?.cwd, env: options?.env ?? process.env, }); - return { stdout, stderr }; + return { stdout, stderr, exitCode: 0 }; } catch (error) { if (options?.check === false && error && typeof error === 'object') { - const execError = error as { stdout?: string; stderr?: string }; + const execError = error as { code?: unknown; stdout?: string; stderr?: string }; return { stdout: execError.stdout ?? '', stderr: execError.stderr ?? '', + exitCode: typeof execError.code === 'number' ? execError.code : 1, }; } throw withFriendlyGitHubAuthError(error); @@ -295,14 +296,14 @@ function getGitEnv(): NodeJS.ProcessEnv { async function runGit( args: readonly string[], options?: { cwd?: string; check?: boolean; env?: NodeJS.ProcessEnv }, -): Promise<{ stdout: string; stderr: string }> { +): Promise<{ stdout: string; stderr: string; exitCode: number }> { return runCommand('git', args, { ...options, env: { ...getGitEnv(), ...options?.env } }); } async function runGh( args: readonly string[], options?: { cwd?: string }, -): Promise<{ stdout: string; stderr: string }> { +): Promise<{ stdout: string; stderr: string; exitCode: number }> { return runCommand('gh', args, options); } @@ -412,15 +413,27 @@ async function fetchResultsRepo( await runGit(['fetch', remote, '--prune'], { cwd: repoDir }); } +function isMissingRemoteBranchFetch(stderr: string): boolean { + const lower = stderr.toLowerCase(); + return lower.includes("couldn't find remote ref") || lower.includes('could not find remote ref'); +} + async function fetchResultsBranchRef( repoDir: string, remote: string, branch: string, ): Promise { - await runGit(['fetch', remote, `+refs/heads/${branch}:refs/remotes/${remote}/${branch}`], { - cwd: repoDir, - check: false, - }); + const { exitCode, stderr } = await runGit( + ['fetch', remote, `+refs/heads/${branch}:refs/remotes/${remote}/${branch}`], + { + cwd: repoDir, + check: false, + }, + ); + const fetchError = stderr.trim(); + if (exitCode !== 0 && !isMissingRemoteBranchFetch(fetchError)) { + throw new Error(fetchError); + } } async function fetchResultsArtifactRef( From 133c2fcad5f446ecd24c01caabfbd1c3bfdbe4a5 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 20:04:25 +1000 Subject: [PATCH 08/12] fix(results): reject control chars in artifact keys --- packages/core/src/evaluation/results-repo.ts | 6 +++++- .../core/test/evaluation/results-repo.test.ts | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index 97bd28746..d98ffe452 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -1858,7 +1858,11 @@ function normalizeDestinationPath(destinationPath: string): string { if ( segments.length === 0 || normalized.startsWith('/') || - segments.some((segment) => segment === '..') + segments.some((segment) => segment === '..') || + normalized.split('').some((char) => { + const code = char.charCodeAt(0); + return code <= 31 || code === 127; + }) ) { throw new Error(`Invalid results destination path: ${destinationPath}`); } diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index a1e0e5cd1..b25d4bd13 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -27,6 +27,7 @@ import { materializeGitRun, normalizeResultsConfig, pushWipCheckpoint, + readGitResultArtifact, setupWipWorktree, syncResultsRepo, syncResultsRepoForProject, @@ -442,6 +443,20 @@ describe('listGitRuns', () => { ); }); + it('rejects artifact keys with control characters before git batch reads', async () => { + writeFileSync(path.join(repoDir, 'public.txt'), 'public'); + writeFileSync(path.join(repoDir, 'secret.txt'), 'secret'); + git('git add public.txt secret.txt && git commit --quiet -m "seed files"', repoDir); + + await expect( + readGitResultArtifact({ + repoDir, + ref: 'HEAD', + key: 'public.txt\nHEAD:secret.txt', + }), + ).rejects.toThrow('Invalid results destination path'); + }); + it('lists and materializes runs from a non-default ref', async () => { writeFileSync(path.join(repoDir, 'README.md'), '# test\n'); git('git add README.md && git commit -m "initial"', repoDir); From 5694ffcb8df7cb1a09f8ed15d5522d8109055391 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 21:57:32 +1000 Subject: [PATCH 09/12] feat(results): use remote URLs in results repo config --- apps/cli/src/commands/eval/commands/run.ts | 2 +- apps/cli/src/commands/results/serve.ts | 2 +- .../src/content/docs/docs/tools/dashboard.mdx | 16 ++--- .../src/content/docs/docs/tools/results.mdx | 2 +- .../src/evaluation/loaders/config-loader.ts | 30 +++++---- packages/core/src/evaluation/results-repo.ts | 5 +- .../evaluation/validation/config-validator.ts | 65 +++++++++---------- packages/core/src/projects.ts | 18 ++--- .../evaluation/loaders/config-loader.test.ts | 8 +-- .../validation/config-validator.test.ts | 20 +++--- packages/core/test/projects.test.ts | 9 +-- scripts/setup-dashboard-deployment.sh | 4 +- .../references/config-schema.json | 11 ++-- 13 files changed, 95 insertions(+), 97 deletions(-) diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index f06cb8f05..5b74c3884 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -79,7 +79,7 @@ export const evalRunCommand = command({ resultsRemote: option({ type: optional(string), long: 'results-remote', - description: 'Git remote name for results push/fetch (default: origin)', + description: 'Advanced compatibility override: local Git remote name for results push/fetch', }), resultsPush: flag({ long: 'results-push', diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index ca58f6e55..06114a4f4 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -125,7 +125,7 @@ const DIRECT_DASHBOARD_SOURCE_GUIDANCE = [ 'Dashboard reads configured project run sources only.', 'Run it from a project root, or pass --dir so Dashboard uses /.agentv/results/runs/:', ' agentv dashboard --dir ', - 'To browse external results, configure results.repo.url or results.repo.path in config YAML.', + 'To browse external results, configure results.repo.remote or results.repo.path in config YAML.', 'For a one-off run bundle, use: agentv results report ', ].join('\n'); diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 8b9089626..9fedb9ae1 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -284,7 +284,7 @@ projects: path: /home/entity/projects/EntityProcess/agentv results: repo: - url: https://github.com/EntityProcess/agentv.git + remote: https://github.com/EntityProcess/agentv.git path: . branch: agentv/results/v1 sync: @@ -292,9 +292,9 @@ projects: require_push: false ``` -`results.repo.url` is the Git remote URL AgentV fetches and pushes. `results.repo.path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. AgentV manages the local Git remote alias for that URL, so the normal config does not need `results.repo.remote`. Use `results.repo.remote` only as an advanced compatibility override when an existing checkout must use a specific local Git remote name. When `results.repo.url` is omitted, `results.repo.path` means an existing local Git checkout whose object database and refs AgentV should write to, and the branch defaults to `agentv/results/v1`. AgentV creates the branch automatically on first publish and commits only AgentV result paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. `sync.require_push: true` is for CI workflows where a push failure should fail the command after local artifacts are written. +`results.repo.remote` is the Git remote URL AgentV fetches and pushes. `results.repo.path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. AgentV manages the local Git remote alias for that URL, so the normal config stays portable across machines. When `results.repo.remote` is omitted, `results.repo.path` means an existing local Git checkout whose object database and refs AgentV should write to, and the branch defaults to `agentv/results/v1`. AgentV creates the branch automatically on first publish and commits only AgentV result paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. `sync.require_push: true` is for CI workflows where a push failure should fail the command after local artifacts are written. -For a separate results repository, use `results.repo.url` and an optional managed clone `results.repo.path`: +For a separate results repository, use `results.repo.remote` and an optional managed clone `results.repo.path`: ```yaml projects: @@ -304,21 +304,21 @@ projects: path: /home/entity/projects/EntityProcess/agentv results: repo: - url: git@github.com:EntityProcess/agentv-examples-eval-results.git + remote: git@github.com:EntityProcess/agentv-examples-eval-results.git branch: agentv/results/v1 path: /home/entity/projects/EntityProcess/agentv-examples-eval-results sync: auto_push: true ``` -`results.repo.url` is the Git remote URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured. When `results.repo.url` is set, `results.repo.path` is the filesystem location of the local clone AgentV manages for that remote results repo. Omit `results.repo.url` only when `results.repo.path` points at an already-existing local checkout such as `.`. +`results.repo.remote` is the Git remote URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured. When `results.repo.remote` is set, `results.repo.path` is the filesystem location of the local clone AgentV manages for that remote results repo. Omit `results.repo.remote` only when `results.repo.path` points at an already-existing local checkout such as `.`. You can also set a top-level global fallback in the same file. This is used when the current project is not registered or its registry entry has no `results` block: ```yaml results: repo: - url: https://github.com/EntityProcess/agentv.git + remote: https://github.com/EntityProcess/agentv.git path: . branch: agentv/results/v1 sync: @@ -331,7 +331,7 @@ Project-local `.agentv/config.yaml` is for portable eval defaults such as `execu The project `repo` block and the `results` block sync different repositories: - `projects[].repo.url` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current. -- `projects[].results.repo.url` is the git-backed results store remote. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in the local checkout at `projects[].results.repo.path`. +- `projects[].results.repo.remote` is the git-backed results store remote URL. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in the local checkout at `projects[].results.repo.path`. #### Migration from the legacy project schema @@ -364,7 +364,7 @@ projects: path: /home/entity/projects/EntityProcess/agentv results: repo: - url: https://github.com/EntityProcess/agentv-eval-results.git + remote: https://github.com/EntityProcess/agentv-eval-results.git branch: agentv/results/v1 path: /home/entity/projects/EntityProcess/agentv-eval-results sync: diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index 9e9ffff2d..6d49909c4 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -228,7 +228,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.url` with `repo.path: .` and `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo without requiring a machine-local Git remote name. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. `results.repo.path` without `results.repo.url` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. `results.repo.remote` is only an advanced local remote-name override. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.remote` with `repo.path: .` and `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo without requiring a machine-local Git remote name. AgentV reserves `agentv/results/v1` for primary results, `agentv/artifacts/v1` for heavy artifact payloads, and `agentv/oplog/v1` for mutable run/result operations. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `benchmark.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. `results.repo.path` without `results.repo.remote` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. AgentV manages any local Git remote alias internally. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. - **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.repo.path` clone with `git` directly, then sync again. diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 700f06215..6030be217 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -634,6 +634,7 @@ function parseNestedResultsRepoConfig( const repoPath = readTrimmedString(repo.path); const branch = readTrimmedString(repo.branch); const remote = readTrimmedString(repo.remote); + const remoteUrl = remote ?? url; if (repo.url !== undefined && !url) { logWarning(`Invalid results.repo.url in ${configPath}, expected non-empty string`); @@ -647,10 +648,6 @@ function parseNestedResultsRepoConfig( logWarning(`Invalid results.repo.path in ${configPath}, expected non-empty string`); return undefined; } - if (!url && !repoPath) { - logWarning(`Invalid results.repo in ${configPath}, expected url or path`); - return undefined; - } if (repo.branch !== undefined && !branch) { logWarning(`Invalid results.repo.branch in ${configPath}, expected non-empty string`); return undefined; @@ -659,19 +656,24 @@ function parseNestedResultsRepoConfig( logWarning(`Invalid results.repo.remote in ${configPath}, expected non-empty string`); return undefined; } - if (remote && isGitRemoteUrl(remote)) { - logWarning( - `Invalid results.repo.remote in ${configPath}: use results.repo.url for Git remote URLs and results.repo.remote only for an optional local Git remote name override.`, - ); + if (remote && !isGitRemoteUrl(remote)) { + logWarning(`Invalid results.repo.remote in ${configPath}, expected Git remote URL`); + return undefined; + } + if (remote && url) { + logWarning(`Invalid results.repo in ${configPath}, set only one of remote or url`); + return undefined; + } + if (!remoteUrl && !repoPath) { + logWarning(`Invalid results.repo in ${configPath}, expected remote or path`); return undefined; } return { - ...(url && { repo_url: url }), - ...(repoPath && !url && { repo_path: repoPath }), - ...(url && repoPath && { path: repoPath }), + ...(remoteUrl && { repo_url: remoteUrl }), + ...(repoPath && !remoteUrl && { repo_path: repoPath }), + ...(remoteUrl && repoPath && { path: repoPath }), ...(branch && { branch }), - ...(remote && { remote }), }; } @@ -718,13 +720,13 @@ export function parseResultsConfig(raw: unknown, configPath: string): ResultsCon const repo = legacyRepo || repoUrl; if (!repo && !repoPath) { logWarning( - `Invalid results in ${configPath}, expected nested repo.url/repo.path or compatible repo_url/repo_path`, + `Invalid results in ${configPath}, expected nested repo.remote/repo.path or compatible repo_url/repo_path`, ); return undefined; } if (repo && repoPath) { logWarning( - `Invalid results in ${configPath}, set only one of nested repo.url/repo.path or compatible repo_url/repo_path`, + `Invalid results in ${configPath}, set only one of nested repo.remote/repo.path or compatible repo_url/repo_path`, ); return undefined; } diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index d98ffe452..e29c8ef1e 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -2250,7 +2250,6 @@ async function commitStorageBranchWorktreePaths(params: { return false; } - await ensureResultsRepoCommitIdentity(params.repoDir); const commitArgs = [ 'commit-tree', newTree, @@ -2258,7 +2257,9 @@ async function commitStorageBranchWorktreePaths(params: { '-m', params.commitMessage, ]; - const { stdout: commitStdout } = await runGit(commitArgs, { cwd: params.repoDir }); + const { stdout: commitStdout } = await runGitWithFallbackCommitIdentity(commitArgs, { + cwd: params.repoDir, + }); const commitSha = commitStdout.trim(); await runGit( [ diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index 4f7615e6b..19dcee188 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -270,7 +270,7 @@ function validateProjectRepoConfig( errors, filePath, `${location}.repo.remote`, - `Use '${location}.repo.url' for the source Git URL. '${location}.repo.remote' is reserved for Git remote names in results repo config.`, + `Use '${location}.repo.url' for the source Git URL. '${location}.repo.remote' is only valid inside results repo config.`, ); } return; @@ -361,14 +361,34 @@ function validateResultsRepoBlock( const repoRecord = rawRepo; const hasUrl = repoRecord.url !== undefined; + const hasRemote = repoRecord.remote !== undefined; const hasPath = repoRecord.path !== undefined; - if (!hasUrl && !hasPath) { - addError(errors, filePath, location, `Field '${location}' must set url or path`); + if (!hasRemote && !hasUrl && !hasPath) { + addError(errors, filePath, location, `Field '${location}' must set remote or path`); + } + + if (hasRemote && hasUrl) { + addError( + errors, + filePath, + location, + `Field '${location}' must set only one remote endpoint. Use '${location}.remote'.`, + ); + } + + if (hasRemote) { + validateGitRemoteUrl(errors, filePath, repoRecord.remote, `${location}.remote`); } if (hasUrl) { validateGitRemoteUrl(errors, filePath, repoRecord.url, `${location}.url`); + addWarning( + errors, + filePath, + `${location}.url`, + `Field '${location}.url' is accepted for compatibility. Use '${location}.remote' for the Git remote URL.`, + ); } if (hasPath) { @@ -386,31 +406,6 @@ function validateResultsRepoBlock( `Field '${location}.branch' must be a non-empty string`, ); } - - if ( - repoRecord.remote !== undefined && - (typeof repoRecord.remote !== 'string' || repoRecord.remote.trim().length === 0) - ) { - addError( - errors, - filePath, - `${location}.remote`, - `Field '${location}.remote' must be a non-empty string`, - ); - } - - if ( - typeof repoRecord.remote === 'string' && - repoRecord.remote.trim().length > 0 && - isGitRemoteUrlValue(repoRecord.remote) - ) { - addError( - errors, - filePath, - `${location}.remote`, - `Field '${location}.remote' is an optional local Git remote name override. Use '${location}.url' for Git remote URLs.`, - ); - } } function validateResultsSyncAndBranchPrefix( @@ -486,13 +481,13 @@ function validateFlatResultsRepoConfig( hasRepoPath, ].filter(Boolean).length; if (sourceCount === 0) { - addError(errors, filePath, location, `Field '${location}' must set repo.url or repo.path`); + addError(errors, filePath, location, `Field '${location}' must set repo.remote or repo.path`); } else if (sourceCount > 1) { addError( errors, filePath, location, - `Field '${location}' must set only one results repo source. Use '${location}.repo.url' for a managed clone or '${location}.repo.path' for an existing local checkout.`, + `Field '${location}' must set only one results repo source. Use '${location}.repo.remote' for a managed clone or '${location}.repo.path' for an existing local checkout.`, ); } else if (hasLegacyRepo) { validateRequiredString(errors, filePath, resultsRecord.repo, `${location}.repo`); @@ -555,8 +550,8 @@ function warnFlatResultsMigration( location: string, ): void { const migrations: Record = { - repo: `${location}.repo.url`, - repo_url: `${location}.repo.url`, + repo: `${location}.repo.remote`, + repo_url: `${location}.repo.remote`, repo_path: `${location}.repo.path`, branch: `${location}.repo.branch`, remote: `${location}.repo.remote`, @@ -600,7 +595,7 @@ function validateResultsConfigBody( errors, filePath, `${location}.mode`, - `Remove '${location}.mode'; project results use '${location}.repo.url' or '${location}.repo.path'.`, + `Remove '${location}.mode'; project results use '${location}.repo.remote' or '${location}.repo.path'.`, ); } else if (resultsRecord.mode !== 'github') { addError(errors, filePath, `${location}.mode`, `Field '${location}.mode' must be 'github'`); @@ -608,7 +603,7 @@ function validateResultsConfigBody( } for (const [field, message] of Object.entries({ - repository: `Field '${location}.repository' was removed. Use '${location}.repo.url' with a Git remote URL instead.`, + repository: `Field '${location}.repository' was removed. Use '${location}.repo.remote' with a Git remote URL instead.`, local_path: `Field '${location}.local_path' was removed. Use '${location}.repo.path' for the local clone path instead.`, })) { if (resultsRecord[field] !== undefined) { @@ -634,7 +629,7 @@ function validateResultsConfigBody( errors, filePath, `${location}.repo`, - `Field '${location}.repo' must be an object. Use '${location}.repo.url' for a managed results clone or '${location}.repo.path' for an existing local checkout.`, + `Field '${location}.repo' must be an object. Use '${location}.repo.remote' for a Git remote URL or '${location}.repo.path' for an existing local checkout.`, ); } } diff --git a/packages/core/src/projects.ts b/packages/core/src/projects.ts index ddc59dc06..7e1078fa4 100644 --- a/packages/core/src/projects.ts +++ b/packages/core/src/projects.ts @@ -22,7 +22,7 @@ * path: /home/user/projects/my-app * results: * repo: - * url: https://github.com/example/my-app.git + * remote: https://github.com/example/my-app.git * path: . * branch: agentv/results/v1 * sync: @@ -175,15 +175,18 @@ function fromYaml(raw: unknown): ProjectEntry | null { const r = e.results as Partial; const resultsRepo = r.repo && typeof r.repo === 'object' && !Array.isArray(r.repo) ? r.repo : undefined; - const repoUrl = readTrimmedString(resultsRepo?.url) ?? readTrimmedString(r.repo_url); - const repoPath = resultsRepo?.url + const repoUrl = + readTrimmedString(resultsRepo?.remote) ?? + readTrimmedString(resultsRepo?.url) ?? + readTrimmedString(r.repo_url); + const repoPath = repoUrl ? undefined : (readTrimmedString(resultsRepo?.path) ?? readTrimmedString(r.repo_path)); - const clonePath = resultsRepo?.url - ? readTrimmedString(resultsRepo.path) + const clonePath = repoUrl + ? (readTrimmedString(resultsRepo?.path) ?? readTrimmedString(r.path)) : readTrimmedString(r.path); const resultsBranch = readTrimmedString(resultsRepo?.branch) ?? readTrimmedString(r.branch); - const resultsRemote = readTrimmedString(resultsRepo?.remote) ?? readTrimmedString(r.remote); + const resultsRemote = resultsRepo ? undefined : readTrimmedString(r.remote); if (repoUrl || repoPath) { const sync = r.sync && typeof r.sync === 'object' ? r.sync : undefined; entry.results = { @@ -225,9 +228,8 @@ function toYaml(entry: ProjectEntry): ProjectEntryYaml { }; if (entry.results) { const resultsRepo: ProjectResultsRepoYaml = { - ...(entry.results.repoUrl !== undefined && { url: entry.results.repoUrl }), + ...(entry.results.repoUrl !== undefined && { remote: entry.results.repoUrl }), ...(entry.results.branch !== undefined && { branch: entry.results.branch }), - ...(entry.results.remote !== undefined && { remote: entry.results.remote }), ...(entry.results.repoUrl && entry.results.path !== undefined && { path: entry.results.path, diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 68d10621c..edec63f52 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -331,9 +331,8 @@ describe('parseResultsConfig', () => { const result = parseResultsConfig( { repo: { - url: 'https://github.com/example/results.git', + remote: 'https://github.com/example/results.git', branch: 'agentv/results/v1', - remote: 'origin', path: '~/data/agentv-results', }, sync: { @@ -348,7 +347,6 @@ describe('parseResultsConfig', () => { repo: 'https://github.com/example/results.git', repo_url: 'https://github.com/example/results.git', branch: 'agentv/results/v1', - remote: 'origin', path: '~/data/agentv-results', sync: { auto_push: true, @@ -360,7 +358,7 @@ describe('parseResultsConfig', () => { const result = parseResultsConfig( { repo: { - url: 'https://github.com/example/source.git', + remote: 'https://github.com/example/source.git', path: '.', branch: 'agentv/results/v1', }, @@ -389,7 +387,6 @@ describe('parseResultsConfig', () => { repo: { path: '.', branch: 'agentv/results/v1', - remote: 'origin', }, sync: { require_push: true, @@ -402,7 +399,6 @@ describe('parseResultsConfig', () => { mode: 'github', repo_path: '.', branch: 'agentv/results/v1', - remote: 'origin', sync: { require_push: true, }, diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index 098d5d210..f11d1e856 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -52,7 +52,7 @@ describe('validateConfigFile', () => { filePath, `results: repo: - url: https://github.com/EntityProcess/agentv-evals.git + remote: https://github.com/EntityProcess/agentv-evals.git branch: agentv-results path: ~/data/agentv-results sync: @@ -95,7 +95,7 @@ describe('validateConfigFile', () => { path: /srv/agentv results: repo: - url: git@github.com:EntityProcess/agentv-results.git + remote: git@github.com:EntityProcess/agentv-results.git branch: agentv-results path: /srv/agentv-results sync: @@ -122,7 +122,7 @@ describe('validateConfigFile', () => { path: /srv/agentv results: repo: - url: https://github.com/EntityProcess/agentv.git + remote: https://github.com/EntityProcess/agentv.git path: . branch: agentv/results/v1 sync: @@ -237,9 +237,8 @@ describe('validateConfigFile', () => { branch: "" results: repo: - url: EntityProcess/results + remote: EntityProcess/results branch: "" - remote: "" path: repo/subdir sync: auto_push: yes @@ -258,7 +257,6 @@ describe('validateConfigFile', () => { expect.objectContaining({ severity: 'error', location: 'projects[0].repo.url' }), expect.objectContaining({ severity: 'error', location: 'projects[0].repo.path' }), expect.objectContaining({ severity: 'error', location: 'projects[0].repo.branch' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].results.repo.url' }), expect.objectContaining({ severity: 'error', location: 'projects[0].results.repo.branch', @@ -308,23 +306,23 @@ describe('validateConfigFile', () => { yaml: `results: repo: example/legacy-results`, location: 'projects[0].results.repo', - migration: 'repo.url', + migration: 'repo.remote', }, { field: 'results.repository', yaml: `results: repo: - url: https://github.com/example/results.git + remote: https://github.com/example/results.git repository: example/results`, location: 'projects[0].results.repository', - migration: 'repo.url', + migration: 'repo.remote', }, { field: 'results.local_path', yaml: `results: repo: - url: https://github.com/example/results.git + remote: https://github.com/example/results.git local_path: /srv/results`, location: 'projects[0].results.local_path', migration: 'path', @@ -333,7 +331,7 @@ describe('validateConfigFile', () => { field: 'results.auto_push', yaml: `results: repo: - url: https://github.com/example/results.git + remote: https://github.com/example/results.git auto_push: true`, location: 'projects[0].results.auto_push', migration: 'sync.auto_push', diff --git a/packages/core/test/projects.test.ts b/packages/core/test/projects.test.ts index e1eca3008..8c0683bc5 100644 --- a/packages/core/test/projects.test.ts +++ b/packages/core/test/projects.test.ts @@ -195,7 +195,9 @@ describe('projects registry', () => { saveProjectRegistry(registry); const yamlOnDisk = readFileSync(registryPath, 'utf-8'); expect(yamlOnDisk).toContain('repo:'); - expect(yamlOnDisk).toContain('url: https://github.com/EntityProcess/results-project-runs.git'); + expect(yamlOnDisk).toContain( + 'remote: https://github.com/EntityProcess/results-project-runs.git', + ); expect(yamlOnDisk).toContain('branch: agentv-results'); expect(yamlOnDisk).toContain('path: /srv/agentv/results/results-project'); expect(yamlOnDisk).toContain('auto_push: true'); @@ -220,7 +222,7 @@ describe('projects registry', () => { path: /srv/agentv/repo results: repo: - url: git@github.com:example/source.git + remote: git@github.com:example/source.git path: . branch: agentv/results/v1 sync: @@ -243,10 +245,9 @@ describe('projects registry', () => { saveProjectRegistry(registry); const yamlOnDisk = readFileSync(registryPath, 'utf-8'); expect(yamlOnDisk).toContain('repo:'); - expect(yamlOnDisk).toContain('url: git@github.com:example/source.git'); + expect(yamlOnDisk).toContain('remote: git@github.com:example/source.git'); expect(yamlOnDisk).toContain('path: .'); expect(yamlOnDisk).toContain('branch: agentv/results/v1'); - expect(yamlOnDisk).not.toContain('remote: origin'); expect(yamlOnDisk).toContain('auto_push: false'); expect(yamlOnDisk).toContain('require_push: true'); expect(yamlOnDisk).not.toContain('repo_path:'); diff --git a/scripts/setup-dashboard-deployment.sh b/scripts/setup-dashboard-deployment.sh index 53dec801a..9d2c104bc 100755 --- a/scripts/setup-dashboard-deployment.sh +++ b/scripts/setup-dashboard-deployment.sh @@ -130,7 +130,7 @@ eval_patterns: results: repo: - url: $results_url + remote: $results_url branch: agentv/results/v1 path: /data/results/agentv-evalresults sync: @@ -168,7 +168,7 @@ const nextEntry = { }, results: { repo: { - url: resultsRepo, + remote: resultsRepo, branch: "agentv/results/v1", path: "/data/results/agentv-evalresults", }, diff --git a/skills-data/agentv-eval-writer/references/config-schema.json b/skills-data/agentv-eval-writer/references/config-schema.json index fd016c8a8..02f8b4143 100644 --- a/skills-data/agentv-eval-writer/references/config-schema.json +++ b/skills-data/agentv-eval-writer/references/config-schema.json @@ -71,11 +71,11 @@ "definitions": { "repoBlock": { "type": "object", - "description": "Nested repository block. Use url for Git URLs, path for local checkout or clone paths, branch for the checked-out/storage branch, and remote for a Git remote name such as origin.", + "description": "Nested repository block. Source project repos use url for Git URLs. Results repos use remote for the Git endpoint URL. Use path for local checkout or clone paths and branch for the checked-out/storage branch.", "properties": { "url": { "type": "string", - "description": "Git remote URL used for cloning or pushing.", + "description": "Source Git remote URL. For results repos, accepted as a compatibility alias for remote.", "examples": ["https://github.com/example/repo.git", "git@github.com:example/repo.git"] }, "path": { @@ -90,8 +90,11 @@ }, "remote": { "type": "string", - "description": "Git remote name for an existing local checkout, such as origin.", - "examples": ["origin"] + "description": "Results Git remote endpoint URL used for fetching and pushing.", + "examples": [ + "https://github.com/example/results.git", + "git@github.com:example/results.git" + ] } }, "additionalProperties": false From 8095da0e1f1b022657546fd967bec16321f41e44 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 21:57:37 +1000 Subject: [PATCH 10/12] docs: capture portable config endpoint guidance --- ...ortable-config-endpoints-by-user-intent.md | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 docs/learnings/best-practices/name-portable-config-endpoints-by-user-intent.md diff --git a/docs/learnings/best-practices/name-portable-config-endpoints-by-user-intent.md b/docs/learnings/best-practices/name-portable-config-endpoints-by-user-intent.md new file mode 100644 index 000000000..40719bceb --- /dev/null +++ b/docs/learnings/best-practices/name-portable-config-endpoints-by-user-intent.md @@ -0,0 +1,91 @@ +--- +title: "Name portable config endpoints by user intent" +module: configuration contracts +date: 2026-06-22 +problem_type: best_practice +component: tooling +severity: medium +tags: + - configuration + - portability + - git-remotes + - results + - dashboard +applies_when: + - Designing persisted YAML or JSON config that will be shared across machines + - Adding Git-backed storage, sync, or artifact configuration + - Choosing between exposing an endpoint URL and exposing a local runtime alias +--- + +# Name portable config endpoints by user intent + +## Context + +While adding Git-backed dashboard result remotes, the initial nested results repository config used `results.repo.url` for the Git endpoint URL and left `remote` available for the local Git remote-name implementation detail. That split was technically precise but did not match how users talk about portable Git sync configuration: in related tools such as Beads, `remote` names the shared endpoint URL, not a machine-local alias. + +For a checked-in AgentV project config, requiring a local Git remote alias would make the file depend on per-machine setup. The portable contract only needs the endpoint URL, a local path, and the branch/ref that stores artifacts. + +## Guidance + +For persisted project and eval configuration, prefer field names that describe the user-facing contract and remain portable across machines. Keep local runtime mechanics out of normal config. + +For AgentV results repositories, use `results.repo.remote` for the Git endpoint URL: + +```yaml +projects: + - id: academy-evals + repo: + url: https://github.com/WiseTechAcademy/Evals.git + branch: main + results: + repo: + remote: https://github.com/WiseTechAcademy/EvalResults.git + path: .agentv/remotes/academy-results + branch: agentv/results/v1 +``` + +Internally, the runtime can still create or use a Git remote name such as `origin`; that alias is an implementation detail. Expose it only through advanced compatibility surfaces when existing callers need it, and label it as local-machine configuration. + +Keep established neighboring contracts intact unless they are part of the same user-facing block. Source project repositories already use `projects[].repo.url`, so changing results repository config to `results.repo.remote` does not require renaming source project `repo.url`. + +## Why This Matters + +Portable config is commonly checked into a repository, copied into Docker development environments, or generated by setup scripts. A field that accepts a machine-local alias forces every machine and container to create the same Git remote name before the config works. + +Using `remote` for the endpoint URL in the results repository block matches the user's model of "where this syncs", aligns with adjacent tools, and keeps Docker/latest-clone local development from depending on mutable host Git state. + +## When to Apply + +- A config file describes Git sync, artifact publishing, or result storage that should work from a fresh clone or container. +- A field could mean either a remote endpoint URL or a local Git remote-name alias. +- A setup script is expected to generate config that a developer can run with `bun run ...` from a latest checkout. + +## Examples + +**Prefer portable endpoint configuration:** + +```yaml +results: + repo: + remote: https://github.com/EntityProcess/agentv-private.git + path: .agentv/remotes/agentv-private + branch: agentv/results/v1 +``` + +**Avoid making a checked-in config depend on a local alias:** + +```yaml +results: + repo: + remote: origin + path: .agentv/remotes/agentv-private + branch: agentv/results/v1 +``` + +If a local alias override must exist for backward compatibility, keep it on an explicitly advanced or legacy option, not on the ordinary project config contract. + +## Related + +- `packages/core/src/evaluation/loaders/config-loader.ts` parses nested results repository config and normalizes the endpoint URL into internal runtime fields. +- `packages/core/src/projects.ts` serializes project config and keeps `results.repo.remote` as the portable Git endpoint. +- `apps/web/src/content/docs/docs/tools/dashboard.mdx` documents the dashboard remote-results setup path. From 7a1d77dfc98282f14fc71747ca65520645f34cb7 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 22:16:42 +1000 Subject: [PATCH 11/12] fix(results): preserve legacy remote aliases --- .../evaluation/validation/config-validator.ts | 10 ++++- packages/core/src/projects.ts | 45 +++++++++++++------ packages/core/test/projects.test.ts | 34 ++++++++++++++ scripts/deploy-example-results-repos.ts | 2 +- .../references/config-schema.json | 37 ++++++++++----- 5 files changed, 102 insertions(+), 26 deletions(-) diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index 19dcee188..76b0f85d9 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -554,7 +554,6 @@ function warnFlatResultsMigration( repo_url: `${location}.repo.remote`, repo_path: `${location}.repo.path`, branch: `${location}.repo.branch`, - remote: `${location}.repo.remote`, path: `${location}.repo.path`, auto_push: `${location}.sync.auto_push`, mode: '(remove this field)', @@ -570,6 +569,15 @@ function warnFlatResultsMigration( ); } } + + if (resultsRecord.remote !== undefined) { + addWarning( + errors, + filePath, + `${location}.remote`, + `Field '${location}.remote' is a legacy local Git remote-name override. Prefer omitting it; nested '${location}.repo.remote' is the portable Git remote URL and AgentV manages local aliases automatically.`, + ); + } } function validateResultsConfigBody( diff --git a/packages/core/src/projects.ts b/packages/core/src/projects.ts index 7e1078fa4..505c0fb02 100644 --- a/packages/core/src/projects.ts +++ b/packages/core/src/projects.ts @@ -227,6 +227,35 @@ function toYaml(entry: ProjectEntry): ProjectEntryYaml { last_opened_at: entry.lastOpenedAt, }; if (entry.results) { + const resultsSync = + entry.results.sync?.autoPush !== undefined || entry.results.sync?.requirePush !== undefined + ? { + sync: { + ...(entry.results.sync?.autoPush !== undefined && { + auto_push: entry.results.sync.autoPush, + }), + ...(entry.results.sync?.requirePush !== undefined && { + require_push: entry.results.sync.requirePush, + }), + }, + } + : {}; + const branchPrefix = + entry.results.branchPrefix !== undefined ? { branch_prefix: entry.results.branchPrefix } : {}; + + if (entry.results.remote !== undefined) { + yaml.results = { + ...(entry.results.repoUrl !== undefined && { repo_url: entry.results.repoUrl }), + ...(entry.results.repoPath !== undefined && { repo_path: entry.results.repoPath }), + ...(entry.results.branch !== undefined && { branch: entry.results.branch }), + remote: entry.results.remote, + ...(entry.results.path !== undefined && { path: entry.results.path }), + ...resultsSync, + ...branchPrefix, + }; + return yaml; + } + const resultsRepo: ProjectResultsRepoYaml = { ...(entry.results.repoUrl !== undefined && { remote: entry.results.repoUrl }), ...(entry.results.branch !== undefined && { branch: entry.results.branch }), @@ -239,20 +268,8 @@ function toYaml(entry: ProjectEntry): ProjectEntryYaml { }; yaml.results = { repo: resultsRepo, - ...((entry.results.sync?.autoPush !== undefined || - entry.results.sync?.requirePush !== undefined) && { - sync: { - ...(entry.results.sync?.autoPush !== undefined && { - auto_push: entry.results.sync.autoPush, - }), - ...(entry.results.sync?.requirePush !== undefined && { - require_push: entry.results.sync.requirePush, - }), - }, - }), - ...(entry.results.branchPrefix !== undefined && { - branch_prefix: entry.results.branchPrefix, - }), + ...resultsSync, + ...branchPrefix, }; } return yaml; diff --git a/packages/core/test/projects.test.ts b/packages/core/test/projects.test.ts index 8c0683bc5..1e3e2b335 100644 --- a/packages/core/test/projects.test.ts +++ b/packages/core/test/projects.test.ts @@ -255,6 +255,40 @@ describe('projects registry', () => { expect(yamlOnDisk).not.toContain('requirePush:'); }); + it('preserves legacy flat results remote aliases through YAML', () => { + const registryPath = getProjectsRegistryPath(); + mkdirSync(path.dirname(registryPath), { recursive: true }); + writeFileSync( + registryPath, + `projects: + - id: legacy-results-remote + name: Legacy Results Remote + path: /srv/agentv/repo + results: + repo_path: . + branch: agentv/results/v1 + remote: upstream + added_at: "2026-01-01T00:00:00Z" + last_opened_at: "2026-01-01T00:00:00Z" +`, + 'utf-8', + ); + + const registry = loadProjectRegistry(); + expect(registry.projects[0].results).toEqual({ + repoPath: '.', + branch: 'agentv/results/v1', + remote: 'upstream', + }); + + saveProjectRegistry(registry); + const yamlOnDisk = readFileSync(registryPath, 'utf-8'); + expect(yamlOnDisk).toContain('repo_path: .'); + expect(yamlOnDisk).toContain('branch: agentv/results/v1'); + expect(yamlOnDisk).toContain('remote: upstream'); + expect(yamlOnDisk).not.toContain('results:\n repo:\n'); + }); + it('preserves unrelated global config keys when saving projects', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); diff --git a/scripts/deploy-example-results-repos.ts b/scripts/deploy-example-results-repos.ts index 51cef2f08..220103181 100644 --- a/scripts/deploy-example-results-repos.ts +++ b/scripts/deploy-example-results-repos.ts @@ -421,7 +421,7 @@ This repo is configured to push evaluation results to \`${params.resultsRepo}\`. const config = readYamlObject(configPath); config.results = { repo: { - url: repoUrl(params.resultsRepo), + remote: repoUrl(params.resultsRepo), path: params.resultsPath, }, sync: { diff --git a/skills-data/agentv-eval-writer/references/config-schema.json b/skills-data/agentv-eval-writer/references/config-schema.json index 02f8b4143..babedf934 100644 --- a/skills-data/agentv-eval-writer/references/config-schema.json +++ b/skills-data/agentv-eval-writer/references/config-schema.json @@ -69,25 +69,32 @@ } }, "definitions": { - "repoBlock": { + "sourceRepoBlock": { "type": "object", - "description": "Nested repository block. Source project repos use url for Git URLs. Results repos use remote for the Git endpoint URL. Use path for local checkout or clone paths and branch for the checked-out/storage branch.", + "description": "Source project repository block. Use url for the Git endpoint, path for the local checkout path, and branch for the checkout branch.", "properties": { "url": { "type": "string", - "description": "Source Git remote URL. For results repos, accepted as a compatibility alias for remote.", + "description": "Source Git remote URL.", "examples": ["https://github.com/example/repo.git", "git@github.com:example/repo.git"] }, "path": { "type": "string", - "description": "Local checkout path. For results repos, this is an existing local repo when url is omitted, or the managed clone path when url is set.", - "examples": ["/srv/agentv/my-evals", ".", "~/data/agentv-results"] + "description": "Local source checkout path.", + "examples": ["/srv/agentv/my-evals"] }, "branch": { "type": "string", - "description": "Source checkout branch or results storage branch.", - "examples": ["main", "agentv/results/v1"] - }, + "description": "Source checkout branch.", + "examples": ["main"] + } + }, + "additionalProperties": false + }, + "resultsRepoBlock": { + "type": "object", + "description": "Results repository block. Use remote for the Git endpoint URL, path for an existing local results checkout or managed clone path, and branch for the storage branch.", + "properties": { "remote": { "type": "string", "description": "Results Git remote endpoint URL used for fetching and pushing.", @@ -95,6 +102,16 @@ "https://github.com/example/results.git", "git@github.com:example/results.git" ] + }, + "path": { + "type": "string", + "description": "Existing local results checkout path when remote is omitted, or managed results clone path when remote is set.", + "examples": [".", "/srv/agentv/results", "~/data/agentv-results"] + }, + "branch": { + "type": "string", + "description": "Results storage branch.", + "examples": ["agentv/results/v1"] } }, "additionalProperties": false @@ -104,7 +121,7 @@ "description": "Git-backed results repo configuration. Canonical form uses a nested repo block.", "properties": { "repo": { - "$ref": "#/definitions/repoBlock" + "$ref": "#/definitions/resultsRepoBlock" }, "sync": { "type": "object", @@ -139,7 +156,7 @@ "type": "string" }, "repo": { - "$ref": "#/definitions/repoBlock" + "$ref": "#/definitions/sourceRepoBlock" }, "results": { "$ref": "#/definitions/resultsConfig" From e8b1ebd7ce5637e5e579505d215af7b447ff5716 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 22 Jun 2026 22:36:53 +1000 Subject: [PATCH 12/12] test(results): use canonical results repo config fixtures --- .../results/remote-auto-export.test.ts | 9 +- apps/cli/test/commands/results/serve.test.ts | 225 ++++++------------ 2 files changed, 79 insertions(+), 155 deletions(-) diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index d1244b75b..8ec7568a2 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -50,10 +50,11 @@ function writeProjectConfig( writeFileSync( path.join(projectDir, '.agentv', 'config.yaml'), `results: - mode: github - repo: ${JSON.stringify(params.repo)} -${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''} path: ${JSON.stringify(params.path)} - auto_push: ${params.autoPush} + repo: + remote: ${JSON.stringify(params.repo)} +${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''} path: ${JSON.stringify(params.path)} + sync: + auto_push: ${params.autoPush} `, ); } diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index fa152dad8..b9a69b1fb 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -170,6 +170,47 @@ function git(command: string, cwd: string): string { return execSync(command, { cwd, encoding: 'utf8', env: cleanGitEnv() }).trim(); } +function writeResultsConfig( + projectDir: string, + params: { + readonly remote: string; + readonly path?: string; + readonly branch?: string; + readonly autoPush?: boolean; + }, +): void { + mkdirSync(path.join(projectDir, '.agentv'), { recursive: true }); + writeFileSync( + path.join(projectDir, '.agentv', 'config.yaml'), + `results: + repo: + remote: ${JSON.stringify(params.remote)} +${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''}${params.path ? ` path: ${JSON.stringify(params.path)}\n` : ''}${params.autoPush !== undefined ? ` sync:\n auto_push: ${params.autoPush}\n` : ''}`, + ); +} + +function writeLegacyShorthandResultsConfig(projectDir: string, repo: string): void { + mkdirSync(path.join(projectDir, '.agentv'), { recursive: true }); + writeFileSync( + path.join(projectDir, '.agentv', 'config.yaml'), + `results: + mode: github + repo: ${repo} +`, + ); +} + +function writeLegacyGlobalShorthandResultsConfig(homeDir: string, repo: string): void { + mkdirSync(homeDir, { recursive: true }); + writeFileSync( + path.join(homeDir, 'config.yaml'), + `results: + mode: github + repo: ${repo} +`, + ); +} + function initializeRemoteRepo(rootDir: string): { remoteDir: string; cloneDir: string; @@ -1363,14 +1404,7 @@ describe('serve app', () => { process.env.AGENTV_HOME = path.join(tempDir, 'agentv-home'); try { - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: EntityProcess/agentv-evals -`, - ); + writeLegacyShorthandResultsConfig(tempDir, 'EntityProcess/agentv-evals'); const remoteRunDir = path.join( process.env.AGENTV_HOME, @@ -1418,15 +1452,7 @@ describe('serve app', () => { RESULT_A, ); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - path: ${cloneDir} -`, - ); + writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: cloneDir }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); @@ -1478,16 +1504,11 @@ describe('serve app', () => { storageBranch, ); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - branch: ${storageBranch} - path: ${cloneDir} -`, - ); + writeResultsConfig(tempDir, { + remote: `file://${remoteDir}`, + branch: storageBranch, + path: cloneDir, + }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); @@ -1513,16 +1534,11 @@ describe('serve app', () => { RESULT_A, ); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - branch: agentv-results - path: ${cloneDir} -`, - ); + writeResultsConfig(tempDir, { + remote: `file://${remoteDir}`, + branch: 'agentv-results', + path: cloneDir, + }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); @@ -1571,15 +1587,7 @@ describe('serve app', () => { ); writeLocalRunArtifact(tempDir, 'green-uat', '2026-03-26T10-30-00-000Z', RESULT_A); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - path: ${cloneDir} -`, - ); + writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: cloneDir }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); @@ -1607,15 +1615,7 @@ describe('serve app', () => { const runId = writeRemoteRunArtifact(cloneDir, experiment, timestamp, RESULT_A); writeLocalRunArtifact(tempDir, experiment, timestamp, RESULT_A); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - path: ${cloneDir} -`, - ); + writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: cloneDir }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); const putRes = await app.request(`/api/runs/${encodeURIComponent(runId)}/tags`, { @@ -1699,15 +1699,7 @@ describe('serve app', () => { failingResult, ]); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - path: ${cloneDir} -`, - ); + writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: cloneDir }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); @@ -1747,15 +1739,7 @@ describe('serve app', () => { RESULT_A, ); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - path: ${cloneDir} -`, - ); + writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: cloneDir }); const runManifestPath = path.join( cloneDir, @@ -1798,15 +1782,7 @@ describe('serve app', () => { RESULT_A, ); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - path: ${cloneDir} -`, - ); + writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: cloneDir }); const filename = `remote::${runId}`; const app = createApp([], tempDir, tempDir, undefined, { studioDir }); @@ -1898,15 +1874,10 @@ describe('serve app', () => { mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${path.join(tempDir, 'missing.git')} - path: ${plainResultsDir} -`, - ); + writeResultsConfig(tempDir, { + remote: `file://${path.join(tempDir, 'missing.git')}`, + path: plainResultsDir, + }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); const res = await app.request( @@ -1935,15 +1906,7 @@ describe('serve app', () => { RESULT_A, ); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - path: ${missingCloneDir} -`, - ); + writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: missingCloneDir }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); const detailRes = await app.request(`/api/runs/${encodeURIComponent(runId)}`); @@ -1966,15 +1929,7 @@ describe('serve app', () => { RESULT_A, ); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: file://${remoteDir} - path: ${missingCloneDir} -`, - ); + writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: missingCloneDir }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); const previewRes = await app.request(`/api/runs/${encodeURIComponent(runId)}/publish`); @@ -2092,14 +2047,7 @@ describe('serve app', () => { process.env.AGENTV_HOME = path.join(tempDir, 'agentv-home-status'); try { - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: EntityProcess/agentv-evals -`, - ); + writeLegacyShorthandResultsConfig(tempDir, 'EntityProcess/agentv-evals'); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); const res = await app.request('/api/remote/status'); @@ -2139,13 +2087,7 @@ describe('serve app', () => { path.join(projectDir, '.agentv', 'config.yaml'), 'execution:\n verbose: true\n', ); - writeFileSync( - path.join(homeDir, 'config.yaml'), - `results: - mode: github - repo: EntityProcess/fallback-results -`, - ); + writeLegacyGlobalShorthandResultsConfig(homeDir, 'EntityProcess/fallback-results'); saveProjectRegistry({ projects: [ { @@ -2718,14 +2660,7 @@ describe('serve app', () => { process.env.AGENTV_HOME = path.join(tempDir, 'agentv-home'); try { const local = seedRun('2026-06-01T10-00-00-000Z', [RESULT_A]); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: EntityProcess/agentv-evals -`, - ); + writeLegacyShorthandResultsConfig(tempDir, 'EntityProcess/agentv-evals'); const remoteRunDir = path.join( process.env.AGENTV_HOME, 'results', @@ -2846,14 +2781,7 @@ describe('serve app', () => { const previousHome = process.env.AGENTV_HOME; process.env.AGENTV_HOME = path.join(tempDir, 'agentv-home'); try { - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: EntityProcess/agentv-evals -`, - ); + writeLegacyShorthandResultsConfig(tempDir, 'EntityProcess/agentv-evals'); const remoteRunDir = path.join( process.env.AGENTV_HOME, 'results', @@ -3374,17 +3302,12 @@ describe('serve app', () => { git(`git push --quiet origin HEAD:${AGENTV_RESULTS_ARTIFACTS_REF}`, seedDir); git('git switch --quiet main', seedDir); - mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); - writeFileSync( - path.join(tempDir, '.agentv', 'config.yaml'), - `results: - mode: github - repo: ${JSON.stringify(`file://${remoteDir}`)} - branch: ${JSON.stringify(resultsBranch)} - path: ${JSON.stringify(cloneDir)} - auto_push: false -`, - ); + writeResultsConfig(tempDir, { + remote: `file://${remoteDir}`, + branch: resultsBranch, + path: cloneDir, + autoPush: false, + }); const artifactRemoteRef = `refs/remotes/agentv-results/${AGENTV_RESULTS_ARTIFACTS_REF}`; const artifactRefLookup = () =>