From 58ba943212c97d39688252770d1f0eb96175e85c Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 19 Jun 2026 04:34:36 +0200 Subject: [PATCH] feat(cli): bundle eval suites for portable reruns --- apps/cli/src/commands/eval/commands/bundle.ts | 246 +++++++ apps/cli/src/commands/eval/index.ts | 2 + apps/cli/src/commands/eval/task-bundle.ts | 656 +++++++++++++++++- apps/cli/src/index.ts | 2 +- apps/cli/test/commands/eval/bundle.test.ts | 213 ++++++ apps/cli/test/unit/preprocess-argv.test.ts | 5 + 6 files changed, 1105 insertions(+), 19 deletions(-) create mode 100644 apps/cli/src/commands/eval/commands/bundle.ts create mode 100644 apps/cli/test/commands/eval/bundle.test.ts diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts new file mode 100644 index 000000000..c0f544784 --- /dev/null +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -0,0 +1,246 @@ +import path from 'node:path'; +import { + type EvalTargetRef, + type TargetDefinition, + loadTestSuite, + readTargetDefinitions, +} from '@agentv/core'; +import { array, command, multioption, option, optional, positional, string } from 'cmd-ts'; + +import { discoverTargetsFile } from '../../../utils/targets.js'; +import { loadEnvFromHierarchy } from '../env.js'; +import { findRepoRoot, resolveEvalPaths } from '../shared.js'; +import { readTestSuiteTarget } from '../targets.js'; +import { type TaskBundleTargetSelection, materializeEvalBundle } from '../task-bundle.js'; + +function unique(values: readonly string[]): readonly string[] { + const result: string[] = []; + const seen = new Set(); + for (const value of values) { + const trimmed = value.trim(); + if (!trimmed || seen.has(trimmed)) { + continue; + } + seen.add(trimmed); + result.push(trimmed); + } + return result; +} + +function targetReferenceNames(target: TargetDefinition): readonly string[] { + const references: string[] = []; + for (const key of ['use_target', 'grader_target', 'judge_target'] as const) { + const value = target[key]; + if (typeof value === 'string' && value.trim().length > 0 && !value.includes('${{')) { + references.push(value.trim()); + } + } + + const fallbackTargets = target.fallback_targets; + if (Array.isArray(fallbackTargets)) { + for (const value of fallbackTargets) { + if (typeof value === 'string' && value.trim().length > 0 && !value.includes('${{')) { + references.push(value.trim()); + } + } + } + + return references; +} + +function ensureTargetGraph( + targetName: string, + definitions: readonly TargetDefinition[], + targetsFilePath: string, +): void { + const byName = new Map(definitions.map((definition) => [definition.name, definition])); + const seen = new Set(); + + function visit(name: string, requestedBy?: string): void { + if (seen.has(name)) { + return; + } + const definition = byName.get(name); + if (!definition) { + const available = definitions + .map((entry) => entry.name) + .sort() + .join(', '); + const owner = requestedBy ? ` referenced by target '${requestedBy}'` : ''; + throw new Error( + `Target '${name}'${owner} not found in ${targetsFilePath}. Available targets: ${available}`, + ); + } + seen.add(name); + for (const referenceName of targetReferenceNames(definition)) { + visit(referenceName, name); + } + } + + visit(targetName); +} + +function definitionsWithEvalTargetRefs( + definitions: readonly TargetDefinition[], + targetRefs: readonly EvalTargetRef[] | undefined, +): readonly TargetDefinition[] { + if (!targetRefs) { + return definitions; + } + + const result = [...definitions]; + for (const ref of targetRefs) { + if (ref.use_target && !result.some((definition) => definition.name === ref.name)) { + result.push({ name: ref.name, use_target: ref.use_target } as TargetDefinition); + } + } + return result; +} + +function buildBundleExecution(options: { + readonly targetNames: readonly string[]; + readonly targetRefs?: readonly EvalTargetRef[]; + readonly workers?: number; + readonly cache?: boolean; + readonly cachePath?: string; + readonly budgetUsd?: number; + readonly threshold?: number; +}): Record { + const targetRefsByName = new Map((options.targetRefs ?? []).map((ref) => [ref.name, ref])); + const serializeTargetRef = (name: string) => { + const ref = targetRefsByName.get(name); + return ref?.hooks || ref?.use_target ? ref : name; + }; + const singleTargetRef = options.targetNames[0] + ? targetRefsByName.get(options.targetNames[0]) + : undefined; + const execution: Record = + options.targetNames.length === 1 && !singleTargetRef?.hooks && !singleTargetRef?.use_target + ? { target: options.targetNames[0] } + : { + targets: options.targetNames.map((name) => serializeTargetRef(name)), + }; + + if (options.workers !== undefined) { + execution.workers = options.workers; + } + if (options.cache !== undefined) { + execution.cache = options.cache; + } + if (options.cachePath !== undefined) { + execution.cache_path = options.cachePath; + } + if (options.budgetUsd !== undefined) { + execution.budget_usd = options.budgetUsd; + } + if (options.threshold !== undefined) { + execution.threshold = options.threshold; + } + return execution; +} + +export const evalBundleCommand = command({ + name: 'bundle', + description: 'Create a portable self-contained directory for an eval suite', + args: { + evalPath: positional({ + type: string, + displayName: 'eval', + description: 'Path or glob resolving to one eval file', + }), + out: option({ + type: string, + long: 'out', + description: 'Portable bundle output directory', + }), + target: multioption({ + type: array(string), + long: 'target', + description: 'Target name to bundle (repeatable). Defaults to eval target(s) or default.', + }), + targets: option({ + type: optional(string), + long: 'targets', + description: 'Path to targets.yaml (overrides discovery)', + }), + }, + handler: async (args) => { + const cwd = process.cwd(); + const repoRoot = await findRepoRoot(cwd); + const resolvedPaths = await resolveEvalPaths([args.evalPath], cwd); + if (resolvedPaths.length !== 1) { + throw new Error( + `agentv eval bundle requires exactly one eval file, but matched ${resolvedPaths.length}: ${resolvedPaths.join(', ')}`, + ); + } + + const evalFilePath = resolvedPaths[0]; + await loadEnvFromHierarchy({ testFilePath: evalFilePath, repoRoot, verbose: false }); + const suite = await loadTestSuite(evalFilePath, repoRoot); + if (suite.providerFactory) { + throw new Error( + 'TypeScript evals with task() provider functions cannot be bundled into portable YAML yet.', + ); + } + + let definitions: readonly TargetDefinition[]; + let targetNames: readonly string[]; + if (suite.inlineTarget) { + definitions = [suite.inlineTarget]; + targetNames = unique(args.target.length > 0 ? args.target : [suite.inlineTarget.name]); + } else { + const targetsFilePath = await discoverTargetsFile({ + explicitPath: args.targets, + testFilePath: evalFilePath, + repoRoot, + cwd, + }); + definitions = definitionsWithEvalTargetRefs( + await readTargetDefinitions(targetsFilePath), + suite.targetRefs, + ); + const suiteTarget = await readTestSuiteTarget(evalFilePath); + targetNames = unique( + args.target.length > 0 ? args.target : (suite.targets ?? [suiteTarget ?? 'default']), + ); + for (const targetName of targetNames) { + ensureTargetGraph(targetName, definitions, targetsFilePath); + } + } + + const targetSelections: TaskBundleTargetSelection[] = targetNames.map((targetName) => ({ + evalFileAbsolutePath: evalFilePath, + targetName, + definitions, + })); + + const paths = await materializeEvalBundle({ + evalFilePath, + tests: suite.tests, + targetSelections, + outputDir: args.out, + cwd, + repoRoot, + execution: buildBundleExecution({ + targetNames, + targetRefs: suite.targetRefs, + workers: suite.workers, + cache: suite.cacheConfig?.enabled, + cachePath: suite.cacheConfig?.cachePath, + budgetUsd: suite.budgetUsd, + threshold: suite.threshold, + }), + }); + + console.log(`Bundle written to: ${paths.bundleDir}`); + console.log( + ` Eval: ${path.relative(paths.bundleDir, paths.evalPath).split(path.sep).join('/')}`, + ); + console.log( + ` Targets: ${path.relative(paths.bundleDir, paths.targetsPath).split(path.sep).join('/')}`, + ); + console.log( + ` Manifest: ${path.relative(paths.bundleDir, paths.manifestPath).split(path.sep).join('/')}`, + ); + }, +}); diff --git a/apps/cli/src/commands/eval/index.ts b/apps/cli/src/commands/eval/index.ts index 0cad65e54..0ec0e9412 100644 --- a/apps/cli/src/commands/eval/index.ts +++ b/apps/cli/src/commands/eval/index.ts @@ -2,6 +2,7 @@ import { subcommands } from 'cmd-ts'; import { evalAggregateCommand } from './commands/aggregate.js'; import { evalAssertCommand } from './commands/assert.js'; +import { evalBundleCommand } from './commands/bundle.js'; import { evalRunCommand } from './commands/run.js'; export const evalCommand = subcommands({ @@ -12,5 +13,6 @@ export const evalCommand = subcommands({ run: evalRunCommand, assert: evalAssertCommand, aggregate: evalAggregateCommand, + bundle: evalBundleCommand, }, }); diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 27ab2ac2c..a716eeda7 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -5,7 +5,10 @@ import path from 'node:path'; import { type EvalSourceReference, type EvalTest, + type JsonObject, type TargetDefinition, + type TestMessage, + type WorkspaceConfig, parseYamlValue, } from '@agentv/core'; import { stringify as stringifyYaml } from 'yaml'; @@ -17,6 +20,11 @@ const TASK_EVAL_FILENAME = 'EVAL.yaml'; const TASK_TARGETS_FILENAME = 'targets.yaml'; const TASK_FILES_DIRNAME = 'files'; const TASK_GRADERS_DIRNAME = 'graders'; +const BUNDLE_EVALS_DIRNAME = 'evals'; +const BUNDLE_MANIFEST_FILENAME = 'agentv_bundle.json'; +const BUNDLE_TARGETS_FILENAME = 'targets.yaml'; +const BUNDLE_WORKSPACES_DIRNAME = 'workspaces'; +const BUNDLE_SCRIPTS_DIRNAME = 'scripts'; const REDACTED_SOURCE_VALUE = '[redacted]'; const SECRET_KEY_PATTERN = /(?:api[_-]?key|authorization|bearer|credential|password|private[_-]?key|secret|token)/i; @@ -61,12 +69,51 @@ export interface MaterializedTaskBundlePaths { readonly gradersPath?: string; } +export interface MaterializeEvalBundleOptions { + readonly evalFilePath: string; + readonly tests: readonly EvalTest[]; + readonly targetSelections: readonly TaskBundleTargetSelection[]; + readonly outputDir: string; + readonly cwd?: string; + readonly repoRoot?: string; + readonly execution?: Record; + readonly now?: () => Date; +} + +export interface MaterializedEvalBundlePaths { + readonly bundleDir: string; + readonly evalsDir: string; + readonly evalPath: string; + readonly targetsPath: string; + readonly manifestPath: string; + readonly filesPath?: string; + readonly gradersPath?: string; + readonly workspacesPath?: string; + readonly scriptsPath?: string; +} + +type BundleReferenceKind = + | EvalSourceReference['kind'] + | 'expected_output_file' + | 'workspace_template' + | 'workspace_hook_command'; + +interface BundleSourceReference extends Omit { + readonly kind: BundleReferenceKind; + readonly location?: string; +} + interface CopiedReference { - readonly reference: EvalSourceReference; + readonly reference: BundleSourceReference; readonly localPath: string; readonly destinationPath: string; } +interface BundleReferenceFailure { + readonly reference: BundleSourceReference; + readonly reason: string; +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } @@ -98,7 +145,7 @@ function hashedExternalPath(resolvedPath: string): string { } function relativeReferencePath( - reference: EvalSourceReference, + reference: BundleSourceReference, options: Pick, ): string | undefined { if (reference.resolvedPath) { @@ -111,8 +158,19 @@ function relativeReferencePath( return safeRelativePath(reference.displayPath); } -function referenceBucket(reference: EvalSourceReference): 'files' | 'graders' { - return reference.kind === 'input_file' ? TASK_FILES_DIRNAME : TASK_GRADERS_DIRNAME; +function referenceBucket( + reference: BundleSourceReference, +): 'files' | 'graders' | 'workspaces' | 'scripts' { + if (reference.kind === 'input_file' || reference.kind === 'expected_output_file') { + return TASK_FILES_DIRNAME; + } + if (reference.kind === 'workspace_template') { + return BUNDLE_WORKSPACES_DIRNAME; + } + if (reference.kind === 'workspace_hook_command') { + return BUNDLE_SCRIPTS_DIRNAME; + } + return TASK_GRADERS_DIRNAME; } function isLikelyBinary(buffer: Buffer): boolean { @@ -219,25 +277,35 @@ async function copyDirectory(sourcePath: string, destinationPath: string): Promi return copiedAny; } -function shouldCopyDirectory(reference: EvalSourceReference): boolean { +function shouldCopyDirectory(reference: BundleSourceReference): boolean { if (reference.kind !== 'code_grader_cwd') { return true; } return !path.isAbsolute(reference.displayPath); } -async function copyReference( - reference: EvalSourceReference, +async function copyReferenceWithFailure( + reference: BundleSourceReference, taskDir: string, options: Pick, -): Promise { +): Promise<{ copied?: CopiedReference; failure?: BundleReferenceFailure }> { if (!reference.resolvedPath) { - return undefined; + return { + failure: { + reference, + reason: `${reference.location ?? reference.kind} has no resolved path: ${reference.displayPath}`, + }, + }; } const relPath = relativeReferencePath(reference, options); if (!relPath) { - return undefined; + return { + failure: { + reference, + reason: `${reference.location ?? reference.kind} could not be assigned a portable path: ${reference.displayPath}`, + }, + }; } const bucket = referenceBucket(reference); @@ -246,29 +314,62 @@ async function copyReference( const sourcePath = path.resolve(reference.resolvedPath); const sourceStat = await stat(sourcePath).catch(() => undefined); if (!sourceStat) { - return undefined; + return { + failure: { + reference, + reason: `${reference.location ?? reference.kind} not found: ${reference.displayPath} (resolved to ${sourcePath})`, + }, + }; } if (sourceStat.isDirectory()) { if (!shouldCopyDirectory(reference)) { - return undefined; + return { + failure: { + reference, + reason: `${reference.location ?? reference.kind} uses an absolute directory that is not safe to copy automatically: ${reference.displayPath}`, + }, + }; } if (!(await copyDirectory(sourcePath, destinationPath))) { - return undefined; + return { + failure: { + reference, + reason: `${reference.location ?? reference.kind} directory contained no bundleable files: ${reference.displayPath}`, + }, + }; } } else if (sourceStat.isFile()) { if (!(await copyFileRedactingText(sourcePath, destinationPath))) { - return undefined; + return { + failure: { + reference, + reason: `${reference.location ?? reference.kind} could not be copied: ${reference.displayPath}`, + }, + }; } } else { - return undefined; + return { + failure: { + reference, + reason: `${reference.location ?? reference.kind} is not a regular file or directory: ${reference.displayPath}`, + }, + }; } - return { reference, localPath: localPath.split(path.sep).join('/'), destinationPath }; + return { copied: { reference, localPath: localPath.split(path.sep).join('/'), destinationPath } }; +} + +async function copyReference( + reference: BundleSourceReference, + taskDir: string, + options: Pick, +): Promise { + return (await copyReferenceWithFailure(reference, taskDir, options)).copied; } async function copyReferences( - references: readonly EvalSourceReference[], + references: readonly BundleSourceReference[], taskDir: string, options: Pick, ): Promise { @@ -285,6 +386,42 @@ async function copyReferences( return copied; } +async function copyReferencesStrict( + references: readonly BundleSourceReference[], + taskDir: string, + options: Pick, +): Promise<{ + readonly copied: readonly CopiedReference[]; + readonly failures: readonly BundleReferenceFailure[]; +}> { + const copied: CopiedReference[] = []; + const failures: BundleReferenceFailure[] = []; + const seenDestinations = new Set(); + const seenFailures = new Set(); + + for (const reference of references) { + const result = await copyReferenceWithFailure(reference, taskDir, options); + if (result.copied) { + if (seenDestinations.has(result.copied.destinationPath)) { + continue; + } + seenDestinations.add(result.copied.destinationPath); + copied.push(result.copied); + continue; + } + + if (result.failure) { + const key = `${result.failure.reference.kind}:${result.failure.reference.displayPath}:${result.failure.reason}`; + if (!seenFailures.has(key)) { + seenFailures.add(key); + failures.push(result.failure); + } + } + } + + return { copied, failures }; +} + function addRewrite(rewrites: Map, from: string | undefined, to: string): void { if (!from || from.trim().length === 0) { return; @@ -414,10 +551,400 @@ async function writeYamlFile(filePath: string, value: unknown): Promise { await writeFile(filePath, `${yaml}\n`, 'utf8'); } -function hasCopiedBucket(copied: readonly CopiedReference[], bucket: 'files' | 'graders'): boolean { +function hasCopiedBucket( + copied: readonly CopiedReference[], + bucket: 'files' | 'graders' | 'workspaces' | 'scripts', +): boolean { return copied.some((entry) => entry.localPath.startsWith(`${bucket}/`)); } +function bundledEvalFileName(evalFilePath: string): string { + const basename = path.basename(evalFilePath); + const withoutKnownExtension = basename + .replace(/\.eval\.[cm]?ts$/i, '') + .replace(/\.eval\.ya?ml$/i, '') + .replace(/\.[cm]?ts$/i, '') + .replace(/\.ya?ml$/i, '') + .replace(/\.jsonl?$/i, ''); + const safeName = withoutKnownExtension + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + return `${safeName || 'bundle'}.eval.yaml`; +} + +function uniqueTargetDefinitions( + selections: readonly TaskBundleTargetSelection[], +): readonly TargetDefinition[] { + const selected: TargetDefinition[] = []; + const seen = new Set(); + + for (const selection of selections) { + for (const definition of selectTargetDefinitions(selection.targetName, selection.definitions)) { + if (seen.has(definition.name)) { + continue; + } + seen.add(definition.name); + selected.push(definition); + } + } + + return selected; +} + +function uniqueTargetNames(selections: readonly TaskBundleTargetSelection[]): readonly string[] { + const names: string[] = []; + const seen = new Set(); + for (const selection of selections) { + if (seen.has(selection.targetName)) { + continue; + } + seen.add(selection.targetName); + names.push(selection.targetName); + } + return names; +} + +function rewritePathString(value: string, rewrites: ReadonlyMap): string { + const direct = rewrites.get(value); + if (direct) { + return direct.startsWith('file://') ? direct.slice('file://'.length) : direct; + } + const fileUrl = rewrites.get(`file://${value}`); + if (fileUrl) { + return fileUrl.startsWith('file://') ? fileUrl.slice('file://'.length) : fileUrl; + } + return value; +} + +function serializeContentValue(value: unknown, rewrites: ReadonlyMap): unknown { + if (typeof value === 'string') { + return value; + } + + if (Array.isArray(value)) { + return value.map((item) => serializeContentItem(item, rewrites)); + } + + if (isRecord(value)) { + return rewritePathsDeep(value, rewrites); + } + + return value; +} + +function serializeContentItem(value: unknown, rewrites: ReadonlyMap): unknown { + if (typeof value === 'string') { + return value; + } + if (!isRecord(value)) { + return value; + } + + const type = value.type; + if (type === 'text') { + const text = typeof value.value === 'string' ? value.value : value.text; + return { type: 'text', value: typeof text === 'string' ? text : '' }; + } + + if (type === 'file') { + const source = + typeof value.value === 'string' + ? value.value + : typeof value.path === 'string' + ? value.path + : typeof value.resolvedPath === 'string' + ? value.resolvedPath + : undefined; + return source + ? { type: 'file', value: rewritePathString(source, rewrites) } + : { type: 'file', value: '' }; + } + + if (type === 'image') { + const source = + typeof value.value === 'string' + ? value.value + : typeof value.path === 'string' + ? value.path + : typeof value.resolvedPath === 'string' + ? value.resolvedPath + : undefined; + if (source) { + return { type: 'image', value: rewritePathString(source, rewrites) }; + } + } + + return rewritePathsDeep(value, rewrites); +} + +function serializeMessage( + message: TestMessage, + rewrites: ReadonlyMap, +): Record { + return { + role: message.role, + content: serializeContentValue(message.content, rewrites), + }; +} + +function serializeExpectedMessage( + message: JsonObject, + rewrites: ReadonlyMap, +): Record { + const serialized: Record = {}; + for (const [key, value] of Object.entries(message)) { + serialized[key] = key === 'content' ? serializeContentValue(value, rewrites) : value; + } + return rewritePathsDeep(serialized, rewrites) as Record; +} + +function serializeWorkspace( + workspace: WorkspaceConfig, + rewrites: ReadonlyMap, +): Record { + const { + workspaceFileDir: _workspaceFileDir, + path: _path, + mode, + ...portableWorkspace + } = workspace; + const withoutStaticMode = + mode === 'static' ? portableWorkspace : { ...portableWorkspace, ...(mode ? { mode } : {}) }; + return rewritePathsDeep(withoutStaticMode, rewrites) as Record; +} + +function buildPortableEvalCase( + test: EvalTest, + rewrites: ReadonlyMap, +): Record { + const testCase = buildEvalCase(test, rewrites); + testCase.id = test.id; + testCase.input = test.input.map((message) => serializeMessage(message, rewrites)); + + if (test.criteria.trim().length > 0) { + testCase.criteria = test.criteria; + } + if (test.expected_output.length > 0) { + testCase.expected_output = test.expected_output.map((message) => + serializeExpectedMessage(message, rewrites), + ); + } + if (test.workspace) { + testCase.workspace = serializeWorkspace(test.workspace, rewrites); + } + if (test.metadata && Object.keys(test.metadata).length > 0) { + testCase.metadata = rewritePathsDeep(test.metadata, rewrites); + } + if (test.conversation_id) { + testCase.conversation_id = test.conversation_id; + } + if (test.targets && test.targets.length > 0) { + const existingExecution = isRecord(testCase.execution) ? testCase.execution : {}; + testCase.execution = { ...existingExecution, targets: test.targets }; + } + if (test.threshold !== undefined) { + const existingExecution = isRecord(testCase.execution) ? testCase.execution : {}; + testCase.execution = { ...existingExecution, threshold: test.threshold }; + } + if (test.mode) { + testCase.mode = test.mode; + } + if (test.turns && test.turns.length > 0) { + testCase.turns = rewritePathsDeep(test.turns, rewrites); + } + if (test.aggregation) { + testCase.aggregation = test.aggregation; + } + if (test.on_turn_failure) { + testCase.on_turn_failure = test.on_turn_failure; + } + if (test.window_size !== undefined) { + testCase.window_size = test.window_size; + } + if (test.depends_on && test.depends_on.length > 0) { + testCase.depends_on = test.depends_on; + } + if (test.on_dependency_failure) { + testCase.on_dependency_failure = test.on_dependency_failure; + } + + return testCase; +} + +function isLikelyCommandPath(value: string): boolean { + if (value.trim().length === 0 || value.startsWith('-') || value.includes('${{')) { + return false; + } + return ( + value.startsWith('.') || + value.includes('/') || + value.includes('\\') || + path.extname(value).length > 0 + ); +} + +async function maybeWorkspaceHookCommandReference(options: { + readonly arg: string; + readonly baseDir: string; + readonly testId: string; + readonly hookName: string; +}): Promise { + if (!isLikelyCommandPath(options.arg)) { + return undefined; + } + const resolvedPath = path.isAbsolute(options.arg) + ? path.resolve(options.arg) + : path.resolve(options.baseDir, options.arg); + const sourceStat = await stat(resolvedPath).catch(() => undefined); + if (!sourceStat?.isFile()) { + return undefined; + } + return { + kind: 'workspace_hook_command', + displayPath: options.arg, + resolvedPath, + location: `workspace.hooks.${options.hookName}.command for test "${options.testId}"`, + }; +} + +async function collectWorkspaceReferences( + tests: readonly EvalTest[], + evalFileDir: string, +): Promise<{ + readonly references: readonly BundleSourceReference[]; + readonly errors: readonly string[]; +}> { + const references: BundleSourceReference[] = []; + const errors: string[] = []; + + for (const test of tests) { + const workspace = test.workspace; + if (!workspace) { + continue; + } + + if (workspace.path || workspace.mode === 'static') { + errors.push( + `workspace.path for test "${test.id}" cannot be bundled because it points at an existing static workspace. Use workspace.template, workspace.repos, or workspace.hooks for portable bundles.`, + ); + } + + if (workspace.template) { + references.push({ + kind: 'workspace_template', + displayPath: workspace.template, + resolvedPath: workspace.template, + location: `workspace.template for test "${test.id}"`, + }); + } + + const hooks = workspace.hooks; + if (!hooks) { + continue; + } + + for (const hookName of ['before_all', 'before_each', 'after_each', 'after_all'] as const) { + const hook = hooks[hookName]; + const command = hook?.command ?? hook?.script; + if (!command || command.length === 0) { + continue; + } + if (hook?.cwd) { + errors.push( + `workspace.hooks.${hookName}.cwd for test "${test.id}" cannot be bundled safely yet: ${hook.cwd}`, + ); + continue; + } + + const baseDir = workspace.workspaceFileDir ?? evalFileDir; + for (const arg of command) { + const reference = await maybeWorkspaceHookCommandReference({ + arg, + baseDir, + testId: test.id, + hookName, + }); + if (reference) { + references.push(reference); + } + } + } + } + + return { references, errors }; +} + +function collectExpectedOutputReferences( + tests: readonly EvalTest[], +): readonly BundleSourceReference[] { + const references: BundleSourceReference[] = []; + for (const test of tests) { + for (const message of test.expected_output) { + const content = message.content; + if (!Array.isArray(content)) { + continue; + } + for (const segment of content) { + if (!isRecord(segment) || segment.type !== 'file') { + continue; + } + const resolvedPath = + typeof segment.resolvedPath === 'string' ? path.resolve(segment.resolvedPath) : undefined; + const displayPath = + typeof segment.path === 'string' + ? segment.path + : typeof segment.value === 'string' + ? segment.value + : resolvedPath; + if (!displayPath) { + continue; + } + references.push({ + kind: 'expected_output_file', + displayPath, + ...(resolvedPath ? { resolvedPath } : {}), + location: `expected_output file for test "${test.id}"`, + }); + } + } + } + return references; +} + +function bundleManifest(options: { + readonly outputDir: string; + readonly evalFilePath: string; + readonly evalPath: string; + readonly targetsPath: string; + readonly copiedReferences: readonly CopiedReference[]; + readonly tests: readonly EvalTest[]; + readonly targetNames: readonly string[]; + readonly createdAt: string; +}): Record { + const relative = (filePath: string) => + path.relative(options.outputDir, filePath).split(path.sep).join('/'); + return { + schema_version: 1, + created_at: options.createdAt, + source_eval: options.evalFilePath, + eval_path: relative(options.evalPath), + targets_path: relative(options.targetsPath), + test_count: options.tests.length, + targets: options.targetNames, + ...(hasCopiedBucket(options.copiedReferences, 'files') ? { files_path: 'evals/files' } : {}), + ...(hasCopiedBucket(options.copiedReferences, 'graders') + ? { graders_path: 'evals/graders' } + : {}), + ...(hasCopiedBucket(options.copiedReferences, 'workspaces') + ? { workspaces_path: 'evals/workspaces' } + : {}), + ...(hasCopiedBucket(options.copiedReferences, 'scripts') + ? { scripts_path: 'evals/scripts' } + : {}), + }; +} + /** * Materialize the native AgentV task source for one completed result row. * @@ -464,3 +991,96 @@ export async function materializeTaskBundle( : {}), }; } + +/** + * Materialize a whole eval suite as a portable directory. + * + * This reuses the same source snapshots, dependency copying, path rewriting, + * target slicing, and secret redaction used by per-result task bundles. The + * output eval is intentionally explicit: inherited suite defaults are written + * onto each bundled test case so the bundle can run without the source tree. + */ +export async function materializeEvalBundle( + options: MaterializeEvalBundleOptions, +): Promise { + if (options.tests.length === 0) { + throw new Error('Cannot bundle eval with no runnable tests.'); + } + if (options.targetSelections.length === 0) { + throw new Error('Cannot bundle eval without a selected target.'); + } + + const missingSource = options.tests.find((test) => !test.source); + if (missingSource) { + throw new Error( + `Cannot bundle test "${missingSource.id}" because it has no source metadata. YAML evals are supported; programmatic evals with inline functions are not portable yet.`, + ); + } + + const outputDir = path.resolve(options.outputDir); + const evalsDir = path.join(outputDir, BUNDLE_EVALS_DIRNAME); + await mkdir(evalsDir, { recursive: true }); + + const evalFileDir = path.dirname(path.resolve(options.evalFilePath)); + const workspaceReferences = await collectWorkspaceReferences(options.tests, evalFileDir); + const references: BundleSourceReference[] = [ + ...options.tests.flatMap((test) => test.source?.references ?? []), + ...collectExpectedOutputReferences(options.tests), + ...workspaceReferences.references, + ]; + + const { copied, failures } = await copyReferencesStrict(references, evalsDir, options); + const errors = [...workspaceReferences.errors, ...failures.map((failure) => failure.reason)]; + if (errors.length > 0) { + throw new Error(`Cannot bundle eval:\n${errors.map((error) => `- ${error}`).join('\n')}`); + } + + const rewrites = buildPathRewrites(copied); + const targetNames = uniqueTargetNames(options.targetSelections); + const evalPath = path.join(evalsDir, bundledEvalFileName(options.evalFilePath)); + const targetsPath = path.join(outputDir, BUNDLE_TARGETS_FILENAME); + const manifestPath = path.join(outputDir, BUNDLE_MANIFEST_FILENAME); + const execution = + options.execution ?? + (targetNames.length === 1 ? { target: targetNames[0] } : { targets: targetNames }); + + await writeYamlFile(evalPath, { + execution, + tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)), + }); + await writeYamlFile(targetsPath, { + targets: uniqueTargetDefinitions(options.targetSelections), + }); + + const manifest = bundleManifest({ + outputDir, + evalFilePath: path.resolve(options.evalFilePath), + evalPath, + targetsPath, + copiedReferences: copied, + tests: options.tests, + targetNames, + createdAt: (options.now ?? (() => new Date()))().toISOString(), + }); + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); + + return { + bundleDir: outputDir, + evalsDir, + evalPath, + targetsPath, + manifestPath, + ...(hasCopiedBucket(copied, 'files') + ? { filesPath: path.join(evalsDir, TASK_FILES_DIRNAME) } + : {}), + ...(hasCopiedBucket(copied, 'graders') + ? { gradersPath: path.join(evalsDir, TASK_GRADERS_DIRNAME) } + : {}), + ...(hasCopiedBucket(copied, 'workspaces') + ? { workspacesPath: path.join(evalsDir, BUNDLE_WORKSPACES_DIRNAME) } + : {}), + ...(hasCopiedBucket(copied, 'scripts') + ? { scriptsPath: path.join(evalsDir, BUNDLE_SCRIPTS_DIRNAME) } + : {}), + }; +} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index e653dabac..48e4124c8 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -61,7 +61,7 @@ export const app = subcommands({ * Known eval subcommand names — used to decide whether to inject the * implicit `run` subcommand for backward-compatible `agentv eval `. */ -const EVAL_SUBCOMMANDS = new Set(['run', 'assert', 'aggregate']); +const EVAL_SUBCOMMANDS = new Set(['run', 'assert', 'aggregate', 'bundle']); /** * Top-level CLI command names (excluding `eval` itself). diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts new file mode 100644 index 000000000..c205e8f41 --- /dev/null +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -0,0 +1,213 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseYamlValue } from '@agentv/core'; +import { execa } from 'execa'; +import { assertCoreBuild } from '../../setup-core-build.js'; + +assertCoreBuild(); + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const projectRoot = path.resolve(__dirname, '../../../../..'); +const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); + +async function runCli(cwd: string, args: readonly string[]) { + return execa('bun', ['--no-env-file', CLI_ENTRY, ...args], { + cwd, + env: { ...process.env, CI: 'true' }, + reject: false, + }); +} + +async function expectFileExists(filePath: string): Promise { + await access(filePath); +} + +describe('agentv eval bundle', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), 'agentv-eval-bundle-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + async function createPortableSourceFixture(): Promise<{ + sourceDir: string; + bundleDir: string; + evalPath: string; + sourceEvalBefore: string; + }> { + const sourceDir = path.join(tempDir, 'source'); + const bundleDir = path.join(tempDir, 'bundle'); + await mkdir(path.join(sourceDir, '.agentv'), { recursive: true }); + await mkdir(path.join(sourceDir, 'evals'), { recursive: true }); + await mkdir(path.join(sourceDir, 'data'), { recursive: true }); + await mkdir(path.join(sourceDir, 'scripts'), { recursive: true }); + await mkdir(path.join(sourceDir, 'workspace-template'), { recursive: true }); + + await writeFile( + path.join(sourceDir, '.agentv', 'targets.yaml'), + `targets: + - name: inherited + provider: mock + response: '{"answer":"Mock provider response from inherited target"}' + fallback_targets: [backup] + - name: backup + provider: mock + response: '{"answer":"Backup mock response"}' +`, + 'utf8', + ); + await writeFile(path.join(sourceDir, 'data', 'input.txt'), 'portable fixture input\n', 'utf8'); + await writeFile( + path.join(sourceDir, 'data', 'cases.yaml'), + `- id: case-alpha + input: + - role: user + content: + - type: file + value: ../data/input.txt + - type: text + value: Answer using the fixture. + assertions: + - type: contains + value: Mock +`, + 'utf8', + ); + await writeFile(path.join(sourceDir, 'workspace-template', 'marker.txt'), 'template\n', 'utf8'); + await writeFile( + path.join(sourceDir, 'scripts', 'setup.ts'), + `const payload = JSON.parse(await Bun.stdin.text()); +await Bun.write(\`\${payload.workspace_path}/hook-ran.txt\`, 'ok\\n'); +`, + 'utf8', + ); + + const evalPath = path.join(sourceDir, 'evals', 'demo.eval.yaml'); + const sourceEvalBefore = `name: portable-demo +execution: + target: inherited +workspace: + template: ../workspace-template + hooks: + before_each: + command: ["bun", "../scripts/setup.ts"] +tests: ../data/cases.yaml +`; + await writeFile(evalPath, sourceEvalBefore, 'utf8'); + + return { sourceDir, bundleDir, evalPath, sourceEvalBefore }; + } + + it('bundles inherited targets, relative data, workspace templates, and scripts into a runnable directory', async () => { + const { sourceDir, bundleDir, evalPath, sourceEvalBefore } = + await createPortableSourceFixture(); + + const bundle = await runCli(sourceDir, [ + 'eval', + 'bundle', + 'evals/demo.eval.yaml', + '--out', + bundleDir, + ]); + + expect(bundle.exitCode).toBe(0); + expect(bundle.stdout).toContain(`Bundle written to: ${bundleDir}`); + expect(await readFile(evalPath, 'utf8')).toBe(sourceEvalBefore); + + const manifest = JSON.parse( + await readFile(path.join(bundleDir, 'agentv_bundle.json'), 'utf8'), + ) as Record; + expect(manifest.schema_version).toBe(1); + expect(manifest.eval_path).toBe('evals/demo.eval.yaml'); + expect(manifest.targets_path).toBe('targets.yaml'); + expect(manifest.test_count).toBe(1); + expect(manifest).not.toHaveProperty('schemaVersion'); + + await expectFileExists(path.join(bundleDir, 'targets.yaml')); + await expectFileExists(path.join(bundleDir, 'evals', 'demo.eval.yaml')); + await expectFileExists(path.join(bundleDir, 'evals', 'files', 'data', 'input.txt')); + await expectFileExists( + path.join(bundleDir, 'evals', 'workspaces', 'workspace-template', 'marker.txt'), + ); + await expectFileExists(path.join(bundleDir, 'evals', 'scripts', 'scripts', 'setup.ts')); + + const bundledEvalText = await readFile(path.join(bundleDir, 'evals', 'demo.eval.yaml'), 'utf8'); + expect(bundledEvalText).not.toContain(sourceDir); + const bundledEval = parseYamlValue(bundledEvalText) as Record; + expect(bundledEval.execution).toEqual({ target: 'inherited' }); + const [testCase] = bundledEval.tests as Record[]; + expect(testCase.id).toBe('case-alpha'); + expect(testCase.workspace).toMatchObject({ + template: 'workspaces/workspace-template', + hooks: { before_each: { command: ['bun', 'scripts/scripts/setup.ts'] } }, + }); + const input = testCase.input as Array<{ content: Array> }>; + expect(input[0]?.content[0]).toEqual({ type: 'file', value: 'files/data/input.txt' }); + + const bundledTargets = await readFile(path.join(bundleDir, 'targets.yaml'), 'utf8'); + expect(bundledTargets).toContain('name: inherited'); + expect(bundledTargets).toContain('name: backup'); + + await rm(sourceDir, { recursive: true, force: true }); + const run = await runCli(bundleDir, [ + 'eval', + 'evals/demo.eval.yaml', + '--output', + path.join(bundleDir, 'run'), + ]); + + expect(run.exitCode).toBe(0); + expect(run.stdout).toContain('RESULT: PASS'); + await expectFileExists(path.join(bundleDir, 'run', 'index.jsonl')); + }, 60_000); + + it('reports unbundleable workspace references with their eval location', async () => { + const sourceDir = path.join(tempDir, 'missing-source'); + const bundleDir = path.join(tempDir, 'missing-bundle'); + await mkdir(path.join(sourceDir, '.agentv'), { recursive: true }); + await mkdir(path.join(sourceDir, 'evals'), { recursive: true }); + await writeFile( + path.join(sourceDir, '.agentv', 'targets.yaml'), + `targets: + - name: default + provider: mock +`, + 'utf8', + ); + await writeFile( + path.join(sourceDir, 'evals', 'missing-template.eval.yaml'), + `workspace: + template: ../does-not-exist +tests: + - id: missing-template + input: hello + assertions: + - type: contains + value: Mock +`, + 'utf8', + ); + + const result = await runCli(sourceDir, [ + 'eval', + 'bundle', + 'evals/missing-template.eval.yaml', + '--out', + bundleDir, + ]); + + expect(result.exitCode).toBe(1); + const output = `${result.stdout}\n${result.stderr}`; + expect(output).toContain('Cannot bundle eval'); + expect(output).toContain('workspace.template for test "missing-template"'); + expect(output).toContain('not found'); + }, 30_000); +}); diff --git a/apps/cli/test/unit/preprocess-argv.test.ts b/apps/cli/test/unit/preprocess-argv.test.ts index 4aaf8eedc..2c4963e70 100644 --- a/apps/cli/test/unit/preprocess-argv.test.ts +++ b/apps/cli/test/unit/preprocess-argv.test.ts @@ -26,6 +26,11 @@ describe('preprocessArgv', () => { expect(preprocessArgv(argv)).toEqual(argv); }); + it('does not insert `run` for eval bundle', () => { + const argv = ['node', 'agentv', 'eval', 'bundle', 'evals/demo.eval.yaml', '--out', 'bundle']; + expect(preprocessArgv(argv)).toEqual(argv); + }); + it('does not insert `run` when eval is followed by --help', () => { const argv = ['node', 'agentv', 'eval', '--help']; expect(preprocessArgv(argv)).toEqual(argv);