diff --git a/apps/cli/src/commands/grade/index.ts b/apps/cli/src/commands/grade/index.ts index 1e15a0537..fcc74d7cc 100644 --- a/apps/cli/src/commands/grade/index.ts +++ b/apps/cli/src/commands/grade/index.ts @@ -9,11 +9,20 @@ import { readFile, stat } from 'node:fs/promises'; import path from 'node:path'; import { + EXECUTION_TRACE_SCHEMA_VERSION, type PreparedAttemptMetadata, type ResolvedTarget, + type Trace, + type TranscriptJsonLine, + buildTraceFromMessages, deriveCategory, gradePreparedEvalCase, loadTestSuite, + readTraceEnvelopeReplayRecords, + readTranscriptJsonl, + traceEnvelopeToTraceSummary, + traceEnvelopeToTranscriptMessages, + traceFromTranscriptJsonLines, writeArtifactsFromResults, } from '@agentv/core'; import { command, number, oneOf, option, optional, positional, string } from 'cmd-ts'; @@ -23,40 +32,11 @@ import { buildDefaultRunDir } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; import { selectMultipleTargets } from '../eval/targets.js'; -interface SetupStepWire { - readonly name: string; - readonly status: 'ok' | 'skipped' | 'warning'; - readonly message?: string; -} - -interface RepoPinWire { - readonly path?: string; - readonly repo?: string; - readonly commit?: string; - readonly base_commit?: string; - readonly ancestor?: number; - readonly sparse?: readonly string[]; -} - -interface BaselineWire { +interface PreparedBaseline { readonly status: 'initialized' | 'unavailable'; readonly commit?: string; } -interface PrepareManifestWire { - readonly schema_version: 1; - readonly eval_path: string; - readonly test_id: string; - readonly target: string; - readonly workspace_path: string; - readonly prompt_path: string; - readonly setup_status: 'ok'; - readonly setup_steps: readonly SetupStepWire[]; - readonly repo_pins: readonly RepoPinWire[]; - readonly baseline: BaselineWire; - readonly created_at: string; -} - interface PreparedManifest { readonly schemaVersion: 1; readonly evalPath: string; @@ -65,9 +45,7 @@ interface PreparedManifest { readonly workspacePath: string; readonly promptPath: string; readonly setupStatus: 'ok'; - readonly setupSteps: readonly SetupStepWire[]; - readonly repoPins: readonly RepoPinWire[]; - readonly baseline: BaselineWire; + readonly baseline: PreparedBaseline; readonly createdAt: string; readonly manifestPath: string; readonly preparedDir: string; @@ -80,6 +58,7 @@ interface GradePreparedResult { readonly executionStatus: string; readonly workspacePath: string; readonly manifestPath: string; + readonly tracePath?: string; readonly outputDir: string; readonly indexPath: string; } @@ -91,10 +70,22 @@ interface GradePreparedResultWire { readonly execution_status: string; readonly workspace_path: string; readonly manifest_path: string; + readonly trace_path?: string; readonly output_dir: string; readonly index_path: string; } +interface PreparedTraceInput { + readonly trace: Trace; + readonly sourcePath: string; +} + +interface TranscriptLineGroup { + readonly testId: string; + readonly target: string; + readonly lines: readonly TranscriptJsonLine[]; +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } @@ -123,7 +114,7 @@ function expectArray( return value; } -function expectBaseline(value: unknown, manifestPath: string): BaselineWire { +function expectBaseline(value: unknown, manifestPath: string): PreparedBaseline { if (!isRecord(value)) { throw invalidManifest(manifestPath, "missing object field 'baseline'"); } @@ -165,6 +156,8 @@ function fromManifestWire(value: unknown, manifestPath: string): PreparedManifes const preparedDir = path.dirname(manifestPath); const resolveManifestPath = (rawPath: string) => path.isAbsolute(rawPath) ? rawPath : path.resolve(preparedDir, rawPath); + expectArray(value, 'setup_steps', manifestPath); + expectArray(value, 'repo_pins', manifestPath); return { schemaVersion: 1, @@ -174,8 +167,6 @@ function fromManifestWire(value: unknown, manifestPath: string): PreparedManifes workspacePath: resolveManifestPath(expectString(value, 'workspace_path', manifestPath)), promptPath: resolveManifestPath(expectString(value, 'prompt_path', manifestPath)), setupStatus, - setupSteps: expectArray(value, 'setup_steps', manifestPath) as readonly SetupStepWire[], - repoPins: expectArray(value, 'repo_pins', manifestPath) as readonly RepoPinWire[], baseline: expectBaseline(value.baseline, manifestPath), createdAt: expectString(value, 'created_at', manifestPath), manifestPath, @@ -265,13 +256,17 @@ function assertMatchesManifest(options: { return options.testId ?? options.manifest.testId; } -function toPreparedAttemptMetadata(manifest: PreparedManifest): PreparedAttemptMetadata { +function toPreparedAttemptMetadata( + manifest: PreparedManifest, + tracePath: string | undefined, +): PreparedAttemptMetadata { return { source: 'manual', manifestPath: manifest.manifestPath, preparedDir: manifest.preparedDir, workspacePath: manifest.workspacePath, promptPath: manifest.promptPath, + ...(tracePath !== undefined && { tracePath }), target: manifest.target, preparedAt: manifest.createdAt, setupStatus: manifest.setupStatus, @@ -288,6 +283,7 @@ function toCommandOutputWire(result: GradePreparedResult): GradePreparedResultWi execution_status: result.executionStatus, workspace_path: result.workspacePath, manifest_path: result.manifestPath, + ...(result.tracePath !== undefined && { trace_path: result.tracePath }), output_dir: result.outputDir, index_path: result.indexPath, }; @@ -298,16 +294,224 @@ function printHumanOutput(result: GradePreparedResult): void { console.log(`Score: ${result.score.toFixed(3)} (${result.executionStatus})`); console.log(`Workspace: ${result.workspacePath}`); console.log(`Manifest: ${result.manifestPath}`); + if (result.tracePath) { + console.log(`Trace: ${result.tracePath}`); + } console.log(`Artifact workspace: ${result.outputDir}`); console.log(`Index: ${result.indexPath}`); } +function isTraceEnvelopeDocument(value: unknown): boolean { + return isRecord(value) && value.schema_version === EXECUTION_TRACE_SCHEMA_VERSION; +} + +function isTranscriptJsonLine(value: unknown): boolean { + return ( + isRecord(value) && + typeof value.test_id === 'string' && + typeof value.target === 'string' && + typeof value.message_index === 'number' && + isRecord(value.source) + ); +} + +function parseFirstJsonLine(raw: string): unknown | undefined { + const firstLine = raw + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line.length > 0); + if (!firstLine) { + return undefined; + } + try { + return JSON.parse(firstLine); + } catch { + return undefined; + } +} + +function looksLikeTraceEnvelopeJsonText(trimmed: string): boolean { + if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) { + return false; + } + return ( + trimmed.includes('"schema_version"') && + trimmed.includes(JSON.stringify(EXECUTION_TRACE_SCHEMA_VERSION)) + ); +} + +function traceEnvelopeRecordMatchesPreparedAttempt( + record: Awaited>[number], + options: { readonly testId: string; readonly target: string }, +): boolean { + if (record.envelope.eval.testId !== options.testId) { + return false; + } + const sourceTarget = record.envelope.eval.sourceTarget ?? record.envelope.eval.target; + return sourceTarget === options.target || record.envelope.eval.target === options.target; +} + +function selectTraceEnvelopeRecord( + records: Awaited>, + options: { readonly sourcePath: string; readonly testId: string; readonly target: string }, +) { + if (records.length === 0) { + throw new Error(`Trace file has no execution trace records: ${options.sourcePath}`); + } + const matchingTest = records.filter((record) => record.envelope.eval.testId === options.testId); + const candidates = matchingTest.filter((record) => + traceEnvelopeRecordMatchesPreparedAttempt(record, options), + ); + + if (candidates.length === 1) { + return candidates[0]; + } + + const detail = + candidates.length === 0 + ? matchingTest.length === 0 + ? `no record matches test_id '${options.testId}'` + : `${matchingTest.length} record(s) match test_id '${options.testId}' but none match target '${options.target}'` + : `${candidates.length} records match test_id '${options.testId}' and target '${options.target}'`; + throw new Error( + `Trace file ${options.sourcePath} is ambiguous for prepared grading: ${detail}. Pass a single-record trace file or a file with one matching test/target record.`, + ); +} + +function traceFromEnvelopeRecord( + record: Awaited>[number], +): Trace { + const summary = traceEnvelopeToTraceSummary(record.envelope); + return buildTraceFromMessages({ + output: traceEnvelopeToTranscriptMessages(record.envelope), + summary: summary.trace, + tokenUsage: summary.tokenUsage, + costUsd: summary.costUsd, + durationMs: summary.durationMs, + startTime: summary.startTime, + endTime: summary.endTime, + provider: record.envelope.source.provider, + target: record.envelope.eval.target, + testId: record.envelope.eval.testId, + conversationId: record.envelope.eval.runId, + metadata: { + trace_source_path: record.sourcePath, + trace_source_format: EXECUTION_TRACE_SCHEMA_VERSION, + trace_artifact_id: record.envelope.artifactId, + ...(record.lineNumber !== undefined && { trace_source_line: record.lineNumber }), + }, + }); +} + +function groupTranscriptLinesByTestTarget( + lines: readonly TranscriptJsonLine[], +): TranscriptLineGroup[] { + const groups = new Map(); + + for (const line of lines) { + const key = `${line.test_id}\0${line.target}`; + const group = + groups.get(key) ?? + (() => { + const created = { testId: line.test_id, target: line.target, lines: [] }; + groups.set(key, created); + return created; + })(); + group.lines.push(line); + } + + return [...groups.values()]; +} + +function selectTranscriptLines( + lines: Awaited>, + options: { readonly sourcePath: string; readonly testId: string; readonly target: string }, +) { + const groups = groupTranscriptLinesByTestTarget(lines); + if (groups.length === 0) { + throw new Error(`Trace file has no transcript rows: ${options.sourcePath}`); + } + const matchingTest = groups.filter((group) => group.testId === options.testId); + const candidates = matchingTest.filter((group) => group.target === options.target); + + if (candidates.length === 1) { + return candidates[0].lines; + } + + const detail = + candidates.length === 0 + ? matchingTest.length === 0 + ? `no transcript group matches test_id '${options.testId}'` + : `${matchingTest.length} transcript group(s) match test_id '${options.testId}' but none match target '${options.target}'` + : `${candidates.length} transcript groups match test_id '${options.testId}' and target '${options.target}'`; + throw new Error( + `Trace file ${options.sourcePath} is ambiguous for prepared grading: ${detail}. Pass a single-session transcript or a transcript with one matching test/target group.`, + ); +} + +async function readPreparedTrace( + tracePath: string, + options: { readonly testId: string; readonly target: string }, +): Promise { + const sourcePath = path.resolve(tracePath); + const raw = await readFile(sourcePath, 'utf8'); + const trimmed = raw.trim(); + if (trimmed.length === 0) { + throw new Error(`Trace file is empty: ${sourcePath}`); + } + + const firstLine = parseFirstJsonLine(raw); + if (isTraceEnvelopeDocument(firstLine)) { + const records = await readTraceEnvelopeReplayRecords(sourcePath); + return { + sourcePath, + trace: traceFromEnvelopeRecord( + selectTraceEnvelopeRecord(records, { sourcePath, ...options }), + ), + }; + } + + if (isTranscriptJsonLine(firstLine)) { + const lines = selectTranscriptLines(await readTranscriptJsonl(sourcePath), { + sourcePath, + ...options, + }); + const trace = traceFromTranscriptJsonLines(lines); + return { + sourcePath, + trace: { + ...trace, + metadata: { + ...trace.metadata, + trace_source_path: sourcePath, + trace_source_format: 'agentv.transcript.jsonl', + }, + }, + }; + } + + if (looksLikeTraceEnvelopeJsonText(trimmed)) { + const records = await readTraceEnvelopeReplayRecords(sourcePath); + return { + sourcePath, + trace: traceFromEnvelopeRecord( + selectTraceEnvelopeRecord(records, { sourcePath, ...options }), + ), + }; + } + + throw new Error( + `Unsupported trace format at ${sourcePath}. Expected agentv.trace.v1 JSON/JSONL or AgentV transcript JSONL.`, + ); +} + async function gradePreparedAttempt(options: { readonly evalPath: string; readonly testId?: string; readonly preparedPath: string; readonly outputDir?: string; readonly responsePath?: string; + readonly tracePath?: string; readonly experiment?: string; readonly graderTarget?: string; readonly model?: string; @@ -320,6 +524,10 @@ async function gradePreparedAttempt(options: { await ensureDirectoryExists(manifest.workspacePath, 'Prepared workspace'); await ensureFileExists(manifest.promptPath, 'Prepared prompt'); + const preparedTrace = + options.tracePath !== undefined + ? await readPreparedTrace(options.tracePath, { testId, target: manifest.target }) + : undefined; const evalDir = path.dirname(evalPath); const repoRoot = await findRepoRoot(evalDir); @@ -370,11 +578,12 @@ async function gradePreparedAttempt(options: { workspacePath: manifest.workspacePath, baselineCommit: manifest.baseline.commit, response, + trace: preparedTrace?.trace, verbose: options.verbose, graderTarget: options.graderTarget, model: options.model, threshold: options.threshold ?? suite.threshold, - preparedAttempt: toPreparedAttemptMetadata(manifest), + preparedAttempt: toPreparedAttemptMetadata(manifest, preparedTrace?.sourcePath), }); const artifacts = await writeArtifactsFromResults([result], runDir, { @@ -391,6 +600,7 @@ async function gradePreparedAttempt(options: { executionStatus: result.executionStatus, workspacePath: manifest.workspacePath, manifestPath: manifest.manifestPath, + tracePath: preparedTrace?.sourcePath, outputDir: runDir, indexPath: artifacts.indexPath, }; @@ -426,6 +636,12 @@ export const gradeCommand = command({ long: 'response', description: 'Optional final response text file from the human or external agent', }), + trace: option({ + type: optional(string), + long: 'trace', + description: + 'Optional AgentV trace/session file for trace-aware graders (execution trace JSON/JSONL or transcript JSONL)', + }), experiment: option({ type: optional(string), long: 'experiment', @@ -459,6 +675,7 @@ export const gradeCommand = command({ prepared, output, response, + trace, experiment, graderTarget, model, @@ -471,6 +688,7 @@ export const gradeCommand = command({ preparedPath: prepared, outputDir: output, responsePath: response, + tracePath: trace, experiment, graderTarget, model, diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index e7b2bac49..27758fa28 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, it, setDefaultTimeout } from 'bun:test'; import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -13,6 +13,8 @@ const __dirname = path.dirname(__filename); const projectRoot = path.resolve(__dirname, '../../../../..'); const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); +setDefaultTimeout(15_000); + async function exists(filePath: string): Promise { return access(filePath).then( () => true, @@ -235,21 +237,21 @@ describe('agentv grade prepared attempts', () => { ).rejects.toThrow(/Invalid prepared manifest/); }); - it('marks trajectory grading unavailable when no trace is supplied', async () => { + it('marks trace-based tool grading unavailable when no trace is supplied', async () => { const { evalPath, targetMarker } = await writeFixtureProject( tempDir, ` - name: expected-tool-sequence type: tool-trajectory - mode: in_order + mode: exact expected: - tool: Read args: path: app.txt `, ); - const preparedDir = path.join(tempDir, 'prepared', 'trajectory'); - const runDir = path.join(tempDir, 'runs', 'trajectory'); + const preparedDir = path.join(tempDir, 'prepared', 'trace-tools'); + const runDir = path.join(tempDir, 'runs', 'trace-tools'); await runCli( ['prepare', evalPath, '--test-id', 'case-1', '--target', 'codex', '--out', preparedDir], @@ -279,4 +281,224 @@ describe('agentv grade prepared attempts', () => { passed: false, }); }); + + it('grades tool-sequence assertions from a supplied transcript trace', async () => { + const { evalPath, targetMarker } = await writeFixtureProject( + tempDir, + ` +- name: expected-tool-sequence + type: tool-trajectory + mode: in_order + expected: + - tool: Read + args: + path: app.txt +`, + ); + const preparedDir = path.join(tempDir, 'prepared', 'trace-tools-with-trace'); + const runDir = path.join(tempDir, 'runs', 'trace-tools-with-trace'); + const tracePath = path.join(tempDir, 'trace.jsonl'); + + await runCli( + ['prepare', evalPath, '--test-id', 'case-1', '--target', 'codex', '--out', preparedDir], + tempDir, + ); + await writeFile( + path.join(preparedDir, 'workspace', 'app.txt'), + 'initial workspace file\nmanual edit\n', + 'utf8', + ); + await writeFile( + tracePath, + `${[ + { + test_id: 'case-1', + target: 'other-agent', + message_index: 0, + role: 'assistant', + content: 'decoy', + tool_calls: [{ tool: 'Write', input: { path: 'app.txt' }, id: 'call-write-app' }], + source: { + provider: 'other-agent', + session_id: 'manual-session-decoy', + }, + }, + { + test_id: 'case-1', + target: 'codex', + message_index: 0, + role: 'assistant', + content: 'done', + tool_calls: [ + { + tool: 'Read', + input: { path: 'app.txt' }, + id: 'call-read-app', + output: 'initial workspace file', + }, + ], + source: { + provider: 'codex', + session_id: 'manual-session-1', + }, + }, + ] + .map((line) => JSON.stringify(line)) + .join('\n')}\n`, + 'utf8', + ); + + const result = await runCli( + [ + 'grade', + evalPath, + '--test-id', + 'case-1', + '--prepared', + preparedDir, + '--trace', + tracePath, + '--output', + runDir, + '--format', + 'json', + ], + tempDir, + ); + + const output = JSON.parse(result.stdout); + expect(output).toMatchObject({ + test_id: 'case-1', + target: 'codex', + score: 1, + execution_status: 'ok', + trace_path: tracePath, + }); + expect(await exists(targetMarker)).toBe(false); + + const row = JSON.parse((await readFile(path.join(runDir, 'index.jsonl'), 'utf8')).trim()); + const answerPath = row.answer_path ?? row.response_path ?? row.output_path; + expect(typeof answerPath).toBe('string'); + expect((await readFile(path.join(runDir, answerPath), 'utf8')).trim()).toBe('done'); + expect(row.score).toBe(1); + expect(row.scores[0]).toMatchObject({ + name: 'expected-tool-sequence', + type: 'tool-trajectory', + score: 1, + assertions: [{ text: 'Found Read at position 0', passed: true }], + }); + expect(row.metadata.prepared_attempt.trace_path).toBe(tracePath); + }); + + it('rejects a single transcript trace for the wrong target', async () => { + const { evalPath } = await writeFixtureProject( + tempDir, + ` +- name: expected-tool-sequence + type: tool-trajectory + mode: in_order + expected: + - tool: Read +`, + ); + const preparedDir = path.join(tempDir, 'prepared', 'wrong-transcript-target'); + const tracePath = path.join(tempDir, 'wrong-transcript-target.jsonl'); + + await runCli( + ['prepare', evalPath, '--test-id', 'case-1', '--target', 'codex', '--out', preparedDir], + tempDir, + ); + await writeFile( + tracePath, + `${JSON.stringify({ + test_id: 'case-1', + target: 'other-agent', + message_index: 0, + role: 'assistant', + content: 'done', + tool_calls: [{ tool: 'Read', input: { path: 'app.txt' }, id: 'call-read-app' }], + source: { + provider: 'other-agent', + session_id: 'manual-session-other', + }, + })}\n`, + 'utf8', + ); + + await expect( + runCli( + ['grade', evalPath, '--test-id', 'case-1', '--prepared', preparedDir, '--trace', tracePath], + tempDir, + ), + ).rejects.toThrow(/none match target 'codex'/); + }); + + it('rejects a single execution trace for the wrong target', async () => { + const { evalPath } = await writeFixtureProject( + tempDir, + ` +- name: expected-tool-sequence + type: tool-trajectory + mode: in_order + expected: + - tool: Read +`, + ); + const preparedDir = path.join(tempDir, 'prepared', 'wrong-envelope-target'); + const tracePath = path.join(tempDir, 'wrong-envelope-target.json'); + + await runCli( + ['prepare', evalPath, '--test-id', 'case-1', '--target', 'codex', '--out', preparedDir], + tempDir, + ); + await writeFile( + tracePath, + `${JSON.stringify( + { + schema_version: 'agentv.trace.v1', + artifact_id: 'execution-trace-wrong-target', + created_at: '2026-06-18T00:00:00.000Z', + eval: { + test_id: 'case-1', + target: 'other-agent', + }, + trace: { + format: 'otlp_openinference_spans', + trace_id: 'trace-wrong-target', + root_span_id: 'span-root', + spans: [ + { + trace_id: 'trace-wrong-target', + span_id: 'span-root', + name: 'invoke_agent', + kind: 'INTERNAL', + start_time_unix_nano: '0', + end_time_unix_nano: '1', + status: { code: 'OK' }, + attributes: {}, + }, + ], + }, + source: { + kind: 'agentv_run', + provider: 'other-agent', + }, + capture: { + content: 'metadata', + redaction_level: 'none', + }, + }, + null, + 2, + )}\n`, + 'utf8', + ); + + await expect( + runCli( + ['grade', evalPath, '--test-id', 'case-1', '--prepared', preparedDir, '--trace', tracePath], + tempDir, + ), + ).rejects.toThrow(/none match target 'codex'/); + }); }); diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index f2283f09b..8fab67fd7 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -85,7 +85,6 @@ function payload(projectDir: string, runDir: string) { target: 'mock', timestamp: '2026-06-13T00:00:00.000Z', trace: { - schemaVersion: 'agentv.trajectory.v1', messages: [], events: [], eventCount: 0, diff --git a/apps/cli/test/commands/trace/trace.test.ts b/apps/cli/test/commands/trace/trace.test.ts index f974df20a..d0db2cbcb 100644 --- a/apps/cli/test/commands/trace/trace.test.ts +++ b/apps/cli/test/commands/trace/trace.test.ts @@ -206,7 +206,6 @@ describe('trace utils', () => { expect(results).toHaveLength(1); expect(results[0].test_id).toBe('test-2'); expect(results[0].trace).toMatchObject({ - schema_version: 'agentv.trajectory.v1', event_count: 0, messages: [], events: [], @@ -222,7 +221,6 @@ describe('trace utils', () => { expect(results).toHaveLength(1); expect(results[0].test_id).toBe('test-2'); expect(results[0].trace).toMatchObject({ - schema_version: 'agentv.trajectory.v1', event_count: 0, messages: [], events: [], diff --git a/apps/cli/test/fixtures/mock-run-evaluation.ts b/apps/cli/test/fixtures/mock-run-evaluation.ts index 98eda7a2c..c0957ca5d 100644 --- a/apps/cli/test/fixtures/mock-run-evaluation.ts +++ b/apps/cli/test/fixtures/mock-run-evaluation.ts @@ -72,7 +72,6 @@ function evalCaseIds(evalCases: ReadonlyArray | undefined): readonly st function buildTrace(targetName: string, testId: string, output: string): Record { const message = { role: 'assistant', content: output }; return { - schemaVersion: 'agentv.trajectory.v1', eventCount: 2, toolCalls: {}, errorCount: 0, diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 75d63d33e..990d5b89f 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -132,6 +132,15 @@ case directories are still useful for organizing bulky prompts, fixtures, or tests while authoring an eval, but they are optional input organization rather than a separate artifact schema. +### Manual or External-Agent Attempts + +Use `agentv prepare` when you want AgentV to set up one eval case but a human, +external agent, or separate harness should perform the work. The workflow is: +prepare the workspace and prompt, run the external attempt in that workspace, +then grade the final state with `agentv grade --prepared` without rerunning the +target provider. See [Prepare](/docs/tools/prepare/) for the full workflow, +manifest shape, and optional trace/session input with `--trace`. + ### Trace Persistence Export execution traces (tool calls, timing, spans) to files for debugging and analysis: diff --git a/apps/web/src/content/docs/docs/tools/prepare.mdx b/apps/web/src/content/docs/docs/tools/prepare.mdx new file mode 100644 index 000000000..ed7a14f98 --- /dev/null +++ b/apps/web/src/content/docs/docs/tools/prepare.mdx @@ -0,0 +1,103 @@ +--- +title: Prepare +description: Prepare one eval case for a human or external agent, then grade the finished workspace. +sidebar: + order: 4 +--- + +`agentv prepare` materializes one eval case without launching the target provider. Use it when a human, a separate agent process, or another harness should attempt the task in the same workspace state AgentV would have provided immediately before target execution. + +This is the manual-attempt workflow: + +```bash +agentv prepare evals/foo.eval.yaml --test-id case-1 --target codex --out /tmp/agentv-case-1 +``` + +The prepared directory contains: + +```text +/tmp/agentv-case-1/ + workspace/ # materialized template/repos/hooks state + prompt.md # safe task prompt for the human or external agent + agentv_prepare.json # snake_case manifest for audit and later grading +``` + +`prepare` runs setup only: workspace `before_all`, target `before_all`, workspace `before_each`, and target `before_each`. It does not launch the agent, run graders, mark an eval complete, or expose hidden expected outputs and grader internals in `prompt.md`. + +## Grade the Attempt + +After the human or external agent finishes editing files in `workspace/`, grade the final state without rerunning the target: + +```bash +agentv grade evals/foo.eval.yaml \ + --test-id case-1 \ + --prepared /tmp/agentv-case-1 \ + --output .agentv/results/runs/manual-case-1 +``` + +`grade` reads `agentv_prepare.json`, verifies it matches the eval/test, captures workspace changes from the prepared baseline when available, and runs the eval's graders against the final workspace. The target provider is not invoked. + +If the external agent produced a final answer outside the workspace, pass it as a text file: + +```bash +agentv grade evals/foo.eval.yaml \ + --test-id case-1 \ + --prepared /tmp/agentv-case-1 \ + --response /tmp/agentv-case-1/final-response.md +``` + +## Add Trace or Session Evidence + +Trace-aware graders can use a local trace/session artifact from the manual attempt: + +```bash +agentv grade evals/foo.eval.yaml \ + --test-id case-1 \ + --prepared /tmp/agentv-case-1 \ + --trace /tmp/agentv-case-1/session.jsonl +``` + +Supported `--trace` inputs: + +| Format | Typical source | +|--------|----------------| +| `agentv.trace.v1` JSON or JSONL | `outputs/execution-trace.json` from an AgentV run or replay/export workflow | +| AgentV transcript JSONL | `agentv import claude`, `agentv import codex`, or `agentv import copilot` output | + +Single-record trace files are accepted directly. Multi-record files are matched by `test_id` and target. The selected trace is projected into AgentV's normal `trace` and `messages` grader context, so `tool-trajectory`, execution-metrics, and code graders receive the same shape they see during eval runs. + +Use `--response` when the final answer text should be graded independently of the trace. If `--response` is omitted and the trace contains an assistant message with content, AgentV uses the last assistant message as the candidate answer. + +## Observability Boundary + +`prepare` is not a replacement for live observability. Configure live tracing in the harness or target itself: + +- Use provider-native settings, target hooks, or environment variables to enable session logs. +- Use AgentV's OTLP options during normal eval runs, such as `--otel-file` or `--export-otel`, when AgentV is the runner. +- For Opik, Phoenix, Langfuse, or another backend, treat their traces as external artifacts that can be imported or projected back into AgentV later. + +AgentV remains responsible for eval definitions, workspace setup, grading, result bundles, and CI gates. Live trace storage, dashboards, and provider-specific run monitoring belong in the observability backend or the external harness. + +There is no `agentv watch` command. + +## Manifest + +`agentv_prepare.json` uses snake_case keys because it is a disk artifact: + +```json +{ + "schema_version": 1, + "eval_path": "/repo/evals/foo.eval.yaml", + "test_id": "case-1", + "target": "codex", + "workspace_path": "/tmp/agentv-case-1/workspace", + "prompt_path": "/tmp/agentv-case-1/prompt.md", + "setup_status": "ok", + "setup_steps": [], + "repo_pins": [], + "baseline": { "status": "initialized", "commit": "..." }, + "created_at": "2026-06-18T00:00:00.000Z" +} +``` + +Keep the prepared directory with the generated run directory when sharing review evidence. The `index.jsonl` row written by `grade` includes `metadata.prepared_attempt` with the manifest path, workspace path, prompt path, baseline status, and optional trace path. diff --git a/docs/adr/2026-06-11-phoenix-observability-adapter.md b/docs/adr/2026-06-11-phoenix-observability-adapter.md index d39703229..29c2232e1 100644 --- a/docs/adr/2026-06-11-phoenix-observability-adapter.md +++ b/docs/adr/2026-06-11-phoenix-observability-adapter.md @@ -6,7 +6,7 @@ Status: Proposed ## Context -AgentV exports evaluation traces through generic OpenTelemetry/OTLP plumbing and is adding a derived trajectory contract for post-hoc trace evaluation. A focused follow-up proposed adding a Phoenix OTel backend preset for `--otel-backend phoenix`, but that raised a scope concern: Phoenix project routing, collector endpoint conventions, API keys, dataset concepts, and experiment behavior are backend-specific. +AgentV exports evaluation traces through generic OpenTelemetry/OTLP plumbing and is adding a trace artifact contract for post-hoc trace evaluation. A focused follow-up proposed adding a Phoenix OTel backend preset for `--otel-backend phoenix`, but that raised a scope concern: Phoenix project routing, collector endpoint conventions, API keys, dataset concepts, and experiment behavior are backend-specific. AgentV's architecture principles prefer a lightweight core with extension points and adapters. Built-ins should be universal primitives that most users compose. Backend-specific observability integrations should not make AgentV core behave like a hosted trace or experiment platform. @@ -25,7 +25,7 @@ AgentV core should own: - generic OTLP/HTTP export configuration; - OTLP JSON file export; -- derived trajectory types and wire conversion; +- trace artifact types and boundary conversion; - generic OTLP/OpenInference import/export mapping where it is backend-neutral; - small registry/discovery primitives for extension points. @@ -94,7 +94,7 @@ Negative: ## Tracker impact -- `av-vwa.6` remains valid: core should map derived trajectories to and from generic OTLP/OpenInference shapes, while Phoenix-specific dataset, experiment, project, and span-kind behavior stays in adapter space. +- `av-vwa.6` remains valid: core should map trace artifacts to and from generic OTLP/OpenInference shapes, while Phoenix-specific dataset, experiment, project, and span-kind behavior stays in adapter space. - `av-vwa.6.1` should be revised from adding a Phoenix preset in core to adding the minimal observability backend extension seam plus a Phoenix resolver in the Phoenix adapter. If the extension seam is not approved, defer the bead and document generic OTLP environment-variable configuration for Phoenix instead. ## Open questions diff --git a/docs/plans/replay-target-workflow-handoff.md b/docs/plans/replay-target-workflow-handoff.md index 046108346..cd2034914 100644 --- a/docs/plans/replay-target-workflow-handoff.md +++ b/docs/plans/replay-target-workflow-handoff.md @@ -24,7 +24,7 @@ This work owns the replay target database loop: - preserve target output/messages/tool calls/transcript/usage/cost/duration where available, - prove replay makes zero live target calls with live-provider environment variables blanked. -The broader derived trajectory contract remains a separate architecture unit. This replay loop should not invent a competing full trace schema. +The broader trace artifact contract remains a separate architecture unit. This replay loop should not invent a competing full trace schema. ## Existing Useful Surface diff --git a/docs/plans/trace-evaluation-architecture.md b/docs/plans/trace-evaluation-architecture.md index deb6b72e7..0842d4e46 100644 --- a/docs/plans/trace-evaluation-architecture.md +++ b/docs/plans/trace-evaluation-architecture.md @@ -9,12 +9,12 @@ date: 2026-06-04 ## Summary -Build AgentV's trace evaluation architecture around a versioned, provider-neutral trajectory contract. AgentV should ingest traces from AgentV runs, OTLP/OpenInference/Phoenix exports, Pi sessions, and transcript-style agent logs; normalize them into one trajectory model; and run existing and future graders against that model without becoming a hosted observability platform. +Build AgentV's trace evaluation architecture around a versioned, provider-neutral trace artifact contract. AgentV should ingest traces from AgentV runs, OTLP/OpenInference/Phoenix exports, Pi sessions, and transcript-style agent logs; normalize them into one trace artifact model; and run existing and future graders against that model without becoming a hosted observability platform. -Update, 2026-06-17: `agentv.trace.v1` is the canonical AgentV trace artifact -namespace for the span-graph sidecar. The trajectory/read-model contract in this -older plan is a derived projection and uses `agentv.trajectory.v1` when it needs -a schema string. +Update, 2026-06-18: `agentv.trace.v1` is the only AgentV trace artifact +namespace for persisted trace sidecars. The trace read-model contract in +this older plan is a derived internal projection over that artifact, not a +separate versioned document that users produce. --- @@ -22,39 +22,39 @@ a schema string. AgentV already captures tool calls in provider `Message.toolCalls`, persists compact `TraceSummary` data, exports AgentV OTLP spans, and has early post-hoc trace commands. That is enough for local AgentV result inspection, but it is not enough to evaluate production traces, Phoenix traces, or third-party agent sessions with the same grader contract. -The best-practice direction is clear: larger players own trace stores, dashboards, datasets, and experiment UIs. AgentV's niche is the repo-local, declarative evaluation harness for coding agents and tool-using agents. To connect those worlds, AgentV needs a durable trajectory contract between raw trace sources and graders. +The best-practice direction is clear: larger players own trace stores, dashboards, datasets, and experiment UIs. AgentV's niche is the repo-local, declarative evaluation harness for coding agents and tool-using agents. To connect those worlds, AgentV needs a durable trace artifact contract between raw trace sources and graders. --- ## Requirements -**Canonical Trajectory Contract** +**Canonical Trace Artifact Contract** -- R1. AgentV must define a versioned derived trajectory object that preserves ordered tool calls, model turns, tool inputs, tool outputs, IDs, timing, status, errors, source metadata, and optional raw evidence. -- R2. The derived trajectory contract must be provider-neutral and must not require OpenTelemetry as the source format. +- R1. AgentV must define a versioned trace artifact that preserves ordered tool calls, model turns, tool inputs, tool outputs, IDs, timing, status, errors, source metadata, and optional raw evidence. +- R2. The trace artifact contract must be provider-neutral and must not require OpenTelemetry as the source format. - R3. The contract must support branchable session sources by selecting an evaluation path before grading. - R4. The contract must retain enough metadata to explain grader evidence without requiring graders to parse provider-native payloads. -- R5. All persisted trajectory wire formats must use `snake_case`; TypeScript internals must use `camelCase`. +- R5. All persisted trace artifacts must use `snake_case`; TypeScript internals must use `camelCase`. **Standards and Integrations** - R6. AgentV OTLP export must continue to emit standards-aligned GenAI spans, especially `invoke_agent`, `chat`, and `execute_tool` operations. -- R7. AgentV must map derived trajectories to and from OTLP/OpenInference-style traces without making Phoenix-specific assumptions in core. +- R7. AgentV must map trace artifacts to and from OTLP/OpenInference-style traces without making Phoenix-specific assumptions in core. - R8. Phoenix integration must use AgentV as the eval-definition and grader layer while Phoenix remains the trace/dataset/experiment backend. - R9. Unsupported or lossy mappings must be explicit in conversion reports instead of silently approximated. **Post-Hoc Trace And Transcript Evaluation** -- R10. Existing deterministic trace graders, including `tool-trajectory` and `execution-metrics`, must run against derived trajectories, not only live provider output messages or compact summaries. +- R10. Existing deterministic trace graders, including `tool-trajectory` and `execution-metrics`, must run against trace artifacts, not only live provider output messages or compact summaries. - R11. Post-hoc evaluation must accept AgentV run artifacts, AgentV OTLP files, Phoenix/Langfuse/OTLP exports, imported coding-agent transcripts, Pi session JSONL, and compact transcript JSONL through source-specific adapters. -- R12. Grader output must cite derived trajectory evidence, such as matched tool call IDs, positions, timing, or source event IDs. -- R13. Trace evaluation must preserve current lightweight result output by deriving compact summaries from derived trajectories. +- R12. Grader output must cite trace artifact evidence, such as matched tool call IDs, positions, timing, or source event IDs. +- R13. Trace evaluation must preserve current lightweight result output by deriving compact summaries from trace artifacts. **Replay** -- R14. Derived trajectory artifacts and transcripts must support replay as well as grading: AgentV should be able to replay recorded model/tool messages as provider output for eval suites without invoking a live agent. +- R14. Trace artifacts and transcripts must support replay as well as grading: AgentV should be able to replay recorded model/tool messages as provider output for eval suites without invoking a live agent. - R15. Replay must preserve source provider metadata, ordered messages, tool calls, tool outputs, token usage, duration, cost, and redaction state where present. -- R16. Replay and grading must share the same derived source artifact so users do not maintain separate transcript and trace formats for the same session. +- R16. Replay and grading must share the same trace artifact so users do not maintain separate transcript and trace formats for the same session. - R17. Replay must be configurable as a normal target substitute, such as a replay target alias replacing a live coding-agent target without changing eval YAML or grader configuration. - R18. Replay lookup must use strict test/run identity, including at least suite or eval identity, test ID, target identity, and attempt or variant when present; missing or ambiguous records must fail loudly. - R19. AgentV must support recording live target responses into replayable JSONL fixtures so a later replay run can execute the same graders without live LLM calls. @@ -75,11 +75,11 @@ The best-practice direction is clear: larger players own trace stores, dashboard ## Key Technical Decisions -- **Start from realistic characterization evals:** The first implementation phase should collect a small set of real trace fixtures and write evals that answer useful agent-quality questions. The derived trajectory contract should be pressure-tested by those evals before broad schema or adapter work expands. -- **Normalize first, grade second:** Graders should consume AgentV's trajectory contract. Importers translate raw sources into the contract; exporters translate the contract into OTLP/OpenInference/Phoenix shapes. This avoids coupling graders to Phoenix, Pi, VS Code, or provider-specific logs. +- **Start from realistic characterization evals:** The first implementation phase should collect a small set of real trace fixtures and write evals that answer useful agent-quality questions. The trace artifact contract should be pressure-tested by those evals before broad schema or adapter work expands. +- **Normalize first, grade second:** Graders should consume AgentV's trace artifact contract. Importers translate raw sources into the contract; exporters translate the contract into OTLP/OpenInference/Phoenix shapes. This avoids coupling graders to Phoenix, Pi, VS Code, or provider-specific logs. - **OTel is an interchange layer, not the canonical model:** VS Code and industry tooling make OTLP/HTTP and GenAI span semantics important, but entireio-style logs and Pi sessions prove valuable traces are often transcript or lifecycle JSON. AgentV should support OTel strongly without making it mandatory. -- **Tool trajectory is turn-centric, not span-centric:** The derived trajectory object should model sessions, turns, messages, tool calls, tool results, and selected branches as a projection over the canonical trace artifact. -- **Coding-agent transcripts are trace sources:** `agentv import claude`, `agentv import codex`, `agentv import copilot`, and `agentv eval --transcript` already establish transcript import as offline grading infrastructure. The architecture should extend that path into trajectory normalization instead of creating a separate trace-only mechanism. +- **Tool sequence grading is turn-centric, not span-centric:** The trace artifact should model sessions, turns, messages, tool calls, tool results, and selected branches as a projection over the canonical trace artifact. +- **Coding-agent transcripts are trace sources:** `agentv import claude`, `agentv import codex`, `agentv import copilot`, and `agentv eval --transcript` already establish transcript import as offline grading infrastructure. The architecture should extend that path into trace artifact normalization instead of creating a separate trace-only mechanism. - **Replay fixtures are not cached grader results:** AgentV replay should return previously recorded target output and then run graders fresh. This is closer to curated transcript replay than result-cache reuse, and it preserves realistic partial or failed behavior for evaluator development. - **Replay is target substitution, not only eval mode:** `agentv eval --transcript` is useful, but the showcase should prove a replay target can replace a live target in the same eval configuration. That keeps replay aligned with AgentV's target composition model. - **Strict lookup over fuzzy cache matching:** A replay database should fail on missing or ambiguous records rather than silently falling back to similar prompts. Smooth demo fallback can come later if real use demands it. @@ -93,11 +93,11 @@ The best-practice direction is clear: larger players own trace stores, dashboard ## High-Level Technical Design -AgentV should introduce a derived trajectory layer between raw sources and evaluation. +AgentV should introduce a trace artifact layer between raw sources and evaluation. ```mermaid flowchart TB - A[AgentV run output] --> N[Derived trajectory] + A[AgentV run output] --> N[Trace artifact] B[OTLP / OpenInference export] --> N C[Phoenix trace export] --> N D[Pi session JSONL] --> N @@ -115,16 +115,15 @@ flowchart TB Q --> R ``` -The derived trajectory model should stay a projection over the canonical +The trace artifact model should stay a projection over the canonical `agentv.trace.v1` sidecar plus derived read models: -- The full trajectory is the derived artifact for grading, replay, and explanation: ordered model turns, tool calls/results, branch metadata, source event IDs, content redaction state, and raw evidence handles. -- The compact summary is a derived compatibility/read model for cheap result storage and dashboard aggregation: counts, durations, token usage, cost, error count, and tool-call counts. It must be recomputable from a full trajectory and should not be authored as separate trace state when the trajectory is available. +- The trace artifact is the derived artifact for grading, replay, and explanation: ordered model turns, tool calls/results, branch metadata, source event IDs, content redaction state, and raw evidence handles. +- The compact summary is a derived compatibility/read model for cheap result storage and dashboard aggregation: counts, durations, token usage, cost, error count, and tool-call counts. It must be recomputable from a trace artifact and should not be authored as separate trace state when the artifact is available. -Directional wire shape: +Directional internal projection shape: ```yaml -schema_version: agentv.trajectory.v1 source: kind: pi_session | agentv_run | otlp | phoenix | langfuse | imported_transcript | compact_transcript path: traces/session.jsonl @@ -176,30 +175,30 @@ The exact schema belongs in implementation, but these concepts should be stable: - **Patterns:** Use real shapes before generalizing. After the replay showcase works, add one AgentV native run with tool calls, one Pi session-style JSONL fixture, one VS Code/OTel-style `invoke_agent` -> `chat` -> `execute_tool` fixture, and optionally one entireio compact transcript fixture when file-change evidence is in scope. - **Existing Surface:** Include one fixture produced by the existing transcript import pipeline (`agentv import claude`, `agentv import codex`, or `agentv import copilot`) so the showcase proves imported coding-agent transcripts are first-class trace sources. - **Test Scenarios:** The initial evals should ask whether the agent called the right tools in order, avoided unnecessary tools, edited expected files when file-change evidence exists, recovered from a tool error, stayed inside cost/latency bounds when metrics exist, and produced grader evidence tied back to source events. -- **Verification:** If the proposed trajectory contract cannot express these fixtures cleanly, revise the contract before expanding adapters. If a field is unused by these evals, keep it optional or defer it. +- **Verification:** If the proposed trace artifact contract cannot express these fixtures cleanly, revise the contract before expanding adapters. If a field is unused by these evals, keep it optional or defer it. -### U1. Derived Trajectory Model +### U1. Trace Artifact Model -- **Goal:** Introduce the core TypeScript model, Zod validation, and snake_case wire conversion for full trajectories. +- **Goal:** Introduce the core TypeScript model, Zod validation, and snake_case boundary conversion for trace artifacts. - **Files:** `packages/core/src/evaluation/trace.ts`, `packages/core/src/evaluation/types.ts`, `packages/eval/src/schemas.ts`, new focused files under `packages/core/src/evaluation/trace/` if the existing file becomes too large. -- **Patterns:** Follow the existing `TraceSummary`, `TokenUsage`, and project wire conversion conventions. Keep internal fields camelCase and wire fields snake_case. -- **Test Scenarios:** Add tests that validate round-trip conversion, version rejection, missing optional content, inferred duration flags, branch metadata, and raw evidence handles. -- **Verification:** Unit tests should prove summaries can be derived from full trajectories without changing current summary behavior, and that derived trajectory artifacts do not embed a separate summary payload. +- **Patterns:** Follow the existing `TraceSummary`, `TokenUsage`, and project boundary conversion conventions. Keep internal fields camelCase and persisted fields snake_case. +- **Test Scenarios:** Add tests that validate round-trip conversion, missing optional content, inferred duration flags, branch metadata, and raw evidence handles. +- **Verification:** Unit tests should prove summaries can be derived from trace artifacts without changing current summary behavior, and that trace artifacts do not embed a separate summary payload. -### U2. Trajectory Extraction From AgentV Runs +### U2. Trace Artifact Extraction From AgentV Runs -- **Goal:** Convert live provider `Message[]` output and existing AgentV result artifacts into derived trajectories. +- **Goal:** Convert live provider `Message[]` output and existing AgentV result artifacts into trace artifacts. - **Files:** `packages/core/src/evaluation/orchestrator.ts`, `packages/core/src/evaluation/providers/types.ts`, `packages/core/src/evaluation/trace.ts`, `apps/cli/src/commands/inspect/utils.ts`. - **Patterns:** Preserve `Message.toolCalls` as the primary high-fidelity source. Keep `TraceSummary` as derived output, not the only trace representation. - **Test Scenarios:** Cover assistant tool calls with IDs, calls without IDs, tool outputs, token usage, model metadata, errors, missing timing, and no-tool-call runs. -- **Verification:** Existing eval result baselines should continue to include compact trace summaries while new full trajectory artifacts are available where configured. +- **Verification:** Existing eval result baselines should continue to include compact trace summaries while full trace artifacts are available where configured. ### U3. OTLP and OpenInference Import/Export Mapping -- **Goal:** Map derived trajectories to and from OTLP JSON/HTTP-compatible spans using GenAI and OpenInference-compatible semantics where available. +- **Goal:** Map trace artifacts to and from OTLP JSON/HTTP-compatible spans using GenAI and OpenInference-compatible semantics where available. - **Files:** `packages/core/src/observability/otel-exporter.ts`, `packages/core/src/observability/otlp-json-file-exporter.ts`, `apps/cli/src/commands/inspect/utils.ts`, tests under `packages/core/test/observability/` and `apps/cli/test/`. - **Patterns:** Continue human-readable span names (`invoke_agent `, `chat `, `execute_tool `) plus machine-stable attributes. Use `gen_ai.operation.name`, `gen_ai.tool.name`, `gen_ai.tool.call.id`, token usage attributes, and `agentv.*` only where standards do not cover the concept. -- **Test Scenarios:** Import an AgentV OTLP file into a trajectory, export it back, and verify tool call order, call IDs, token usage, durations, redaction state, and grader score events survive when representable. +- **Test Scenarios:** Import an AgentV OTLP file into a trace artifact, export it back, and verify tool call order, call IDs, token usage, durations, redaction state, and grader score events survive when representable. - **Verification:** `agentv trace show` and `agentv trace score` should work from OTLP artifacts without requiring an `agentv.score` attribute for trace-only evaluation. ### U4. Phoenix Adapter Trace Evaluation Path @@ -207,16 +206,16 @@ The exact schema belongs in implementation, but these concepts should be stable: - **Goal:** Extend the Phoenix integration so Phoenix can be a trace source and experiment backend while AgentV remains the grader/eval definition layer. - **Files:** `packages/phoenix-adapter/src/`, `packages/phoenix-adapter/docs/support-matrix.md`, `packages/phoenix-adapter/docs/e2e-verification.md`, related CLI entry points if promoted beyond package scripts. - **Patterns:** Keep unsupported mappings visible, as the current Phoenix adapter already does. Do not move Phoenix-specific dataset or experiment concepts into AgentV core. -- **Test Scenarios:** Convert a Phoenix/OTLP trace export into derived trajectories, run deterministic trace graders, report unsupported evaluator families, and emit a Phoenix experiment/report with AgentV grader results where supported. +- **Test Scenarios:** Convert a Phoenix/OTLP trace export into trace artifacts, run deterministic trace graders, report unsupported evaluator families, and emit a Phoenix experiment/report with AgentV grader results where supported. - **Verification:** Dry-run conversion must work offline. Live Phoenix smoke can remain a separately documented manual check. ### U5. Pi Session Importer -- **Goal:** Import Pi session JSONL, including Hugging Face `pi-mono` style files, into derived trajectories. +- **Goal:** Import Pi session JSONL, including Hugging Face `pi-mono` style files, into trace artifacts. - **Files:** `packages/core/src/evaluation/providers/pi-cli.ts`, `packages/core/src/evaluation/providers/pi-coding-agent.ts`, likely new importer files under `packages/core/src/import/` or `packages/core/src/evaluation/trace/importers/`. - **Patterns:** Reuse existing Pi parsing where possible, but add branch/path selection. Fold `toolResult` messages into matching assistant tool calls by ID. Treat `bashExecution` mapping as explicit policy. - **Test Scenarios:** Cover session header parsing, branch selection, assistant `toolCall` blocks, separate `toolResult` entries, `bashExecution`, inline images, thinking blocks, token usage, cost, and inferred timing. -- **Verification:** A small fixture derived from Pi's public session format should produce a deterministic ordered trajectory and compact summary. +- **Verification:** A small fixture derived from Pi's public session format should produce a deterministic ordered trace artifact and compact summary. ### U6. Compact Transcript and Lifecycle Log Importer @@ -230,17 +229,17 @@ The exact schema belongs in implementation, but these concepts should be stable: - **Goal:** Upgrade the current transcript import and replay path so imported Claude, Codex, and Copilot sessions can be normalized and graded as trajectories. - **Files:** `apps/cli/src/commands/import/`, `packages/core/src/import/`, `packages/core/src/import/transcript-provider.ts`, `apps/cli/src/commands/eval/run-eval.ts`. -- **Patterns:** Reuse `TranscriptEntry`, `TranscriptJsonLine`, and `TranscriptProvider` instead of inventing a parallel transcript format. Preserve `agentv eval --transcript` compatibility while adding trajectory derivation. -- **Test Scenarios:** Import one Claude, Codex, and Copilot fixture; replay each through `--transcript`; derive derived trajectories; run `tool-trajectory`; verify tool IDs, order, tool outputs, source provider metadata, duration, cost, token usage, and redaction state where available. +- **Patterns:** Reuse `TranscriptEntry`, `TranscriptJsonLine`, and `TranscriptProvider` instead of inventing a parallel transcript format. Preserve `agentv eval --transcript` compatibility while adding trace artifact derivation. +- **Test Scenarios:** Import one Claude, Codex, and Copilot fixture; replay each through `--transcript`; derive trace artifacts; run `tool-trajectory`; verify tool IDs, order, tool outputs, source provider metadata, duration, cost, token usage, and redaction state where available. - **Verification:** Existing transcript-provider tests keep passing, and new trace-evaluation tests prove imported transcripts and OTLP/Phoenix exports feed the same grader path. -### U6c. Replay From Derived Trajectory +### U6c. Replay From Trace Artifact -- **Goal:** Let derived trajectories act as replay sources, not only grader inputs. -- **Files:** `packages/core/src/import/transcript-provider.ts`, `packages/core/src/import/types.ts`, trajectory model files from U1, `apps/cli/src/commands/eval/run-eval.ts`. -- **Patterns:** Treat replay as a projection from derived trajectory to provider `Message[]`. Do not duplicate storage by keeping one trace file for graders and a separate transcript file for replay when the derived artifact can serve both. -- **Test Scenarios:** Replay a derived trajectory generated from AgentV output, an imported transcript, and an OTLP/Phoenix-style trace. Verify each produces the expected provider output messages and identical compact summaries where data is representable. -- **Verification:** Existing `agentv eval --transcript` behavior remains compatible, with a migration path to replay derived trajectory artifacts directly. +- **Goal:** Let trace artifacts act as replay sources, not only grader inputs. +- **Files:** `packages/core/src/import/transcript-provider.ts`, `packages/core/src/import/types.ts`, trace artifact model files from U1, `apps/cli/src/commands/eval/run-eval.ts`. +- **Patterns:** Treat replay as a projection from a trace artifact to provider `Message[]`. Do not duplicate storage by keeping one trace file for graders and a separate transcript file for replay when the trace artifact can serve both. +- **Test Scenarios:** Replay a trace artifact generated from AgentV output, an imported transcript, and an OTLP/Phoenix-style trace. Verify each produces the expected provider output messages and identical compact summaries where data is representable. +- **Verification:** Existing `agentv eval --transcript` behavior remains compatible, with a migration path to replay trace artifacts directly. ### U6d. Replay Target Database Loop @@ -260,10 +259,10 @@ The exact schema belongs in implementation, but these concepts should be stable: ### U7. Grader Context Upgrade -- **Goal:** Let built-in and code graders receive derived trajectories in addition to compact summaries and output messages. +- **Goal:** Let built-in and code graders receive trace artifacts in addition to compact summaries and output messages. - **Files:** `packages/core/src/evaluation/graders/types.ts`, `packages/core/src/evaluation/graders/tool-trajectory.ts`, `packages/core/src/evaluation/graders/execution-metrics.ts`, `packages/core/src/evaluation/graders/code-grader.ts`, `packages/eval/src/index.ts`, `packages/eval/src/schemas.ts`. -- **Patterns:** Keep backward compatibility: existing graders that only read `trace` or `output` continue to work. New trajectory-aware graders use the richer object. -- **Test Scenarios:** Existing `tool-trajectory` modes should pass from live output and from derived trajectory input. Argument matching, ordering, latency, status/error matching, and evidence text should be covered. +- **Patterns:** Keep existing graders that only read `trace` or `output` working. Trace-aware graders use the richer object. +- **Test Scenarios:** Existing `tool-trajectory` modes should pass from live output and from trace artifact input. Argument matching, ordering, latency, status/error matching, and evidence text should be covered. - **Verification:** `trace score` should run `tool-trajectory` against imported traces, not only metrics-only graders. ### U8. CLI and Artifact Workflow @@ -286,13 +285,13 @@ The exact schema belongs in implementation, but these concepts should be stable: ## Acceptance Examples -- AE1. **Covers R1, R10, R13.** Given an AgentV run with ordered tool calls, when the run is converted to a derived trajectory, then `tool-trajectory` can grade the order and the result still has the same compact summary counts. -- AE2. **Covers R2, R7, R11.** Given an OTLP trace export with `execute_tool` spans, when `agentv trace score` runs, then AgentV imports the spans into a trajectory and grades tool usage without requiring a Phoenix server. +- AE1. **Covers R1, R10, R13.** Given an AgentV run with ordered tool calls, when the run is converted to a trace artifact, then `tool-trajectory` can grade the order and the result still has the same compact summary counts. +- AE2. **Covers R2, R7, R11.** Given an OTLP trace export with `execute_tool` spans, when `agentv trace score` runs, then AgentV imports the spans into a trace artifact and grades tool usage without requiring a Phoenix server. - AE3. **Covers R3, R11.** Given a branchable Pi session, when a selected leaf is provided or inferred, then only the selected branch path is graded and omitted branch IDs are recorded. - AE4. **Covers R8, R9.** Given a Phoenix trace export containing unsupported evaluator semantics, when the Phoenix adapter runs, then the report lists unsupported mappings instead of treating them as successful conversions. -- AE5. **Covers R23.** Given content capture is disabled, when a trace with tool arguments and results is exported, then the trajectory preserves IDs, names, timing, and redaction state but does not persist raw content. -- AE6. **Covers R12.** Given a failed tool trajectory expectation, when AgentV reports the grader result, then the assertion cites the actual matched or missing tool call positions and source event IDs. -- AE7. **Covers R10, R11.** Given a transcript imported with `agentv import codex`, `agentv import claude`, or `agentv import copilot`, when the transcript is evaluated post-hoc, then the same trajectory graders used for OTLP/Phoenix traces can score it. +- AE5. **Covers R23.** Given content capture is disabled, when a trace with tool arguments and results is exported, then the trace artifact preserves IDs, names, timing, and redaction state but does not persist raw content. +- AE6. **Covers R12.** Given a failed tool sequence expectation, when AgentV reports the grader result, then the assertion cites the actual matched or missing tool call positions and source event IDs. +- AE7. **Covers R10, R11.** Given a transcript imported with `agentv import codex`, `agentv import claude`, or `agentv import copilot`, when the transcript is evaluated post-hoc, then the same trace graders used for OTLP/Phoenix traces can score it. - AE8. **Covers R14, R15, R16.** Given the same derived artifact, when AgentV uses it for replay and for grading, then replayed provider output and grader evidence come from the same ordered messages and tool calls. - AE9. **Covers R17, R18, R19.** Given a live target run has been recorded into replay JSONL, when the same eval runs with a replay target alias and shuffled fixture records, then AgentV selects the exact matching record, makes no live LLM call, and runs graders fresh. - AE10. **Covers R18.** Given a replay database has no matching record or has multiple matching records for the current test and target, when replay runs, then AgentV fails with an actionable error instead of falling back to a near match. @@ -324,8 +323,8 @@ Outside this product's identity: ## System-Wide Impact -- **Core data model:** `TraceSummary` becomes a derived compact view of full trajectories, not the highest-fidelity trace contract. -- **Graders:** Trace-aware graders move from output-message parsing toward trajectory parsing while keeping compatibility with current eval results. +- **Core data model:** `TraceSummary` becomes a derived compact view of trace artifacts, not the highest-fidelity trace contract. +- **Graders:** Trace-aware graders move from output-message parsing toward trace artifact parsing while keeping compatibility with current eval results. - **CLI:** Post-hoc trace commands become a real evaluation path, not just inspection of AgentV-generated artifacts. - **Observability:** OTel export remains standards-aligned and becomes round-trippable enough for offline scoring. - **Adapters:** Phoenix, Pi, and transcript importers stay outside primitive grader logic, preserving AgentV's lightweight core. @@ -335,9 +334,9 @@ Outside this product's identity: ## Risks And Dependencies -- **Standards churn:** GenAI and OpenInference semantic conventions continue to evolve. Mitigation: keep the AgentV trajectory contract stable and isolate standards mapping in adapters. +- **Standards churn:** GenAI and OpenInference semantic conventions continue to evolve. Mitigation: keep the AgentV trace artifact contract stable and isolate standards mapping in adapters. - **Lossy imports:** Some sources lack tool timing, outputs, status, or branch data. Mitigation: record provenance and warning metadata instead of fabricating precision. -- **Artifact size:** Full trajectories can be large. Mitigation: keep compact summaries as default result metadata and store full trajectories as optional sidecar artifacts. +- **Artifact size:** Full trace artifacts can be large. Mitigation: keep compact summaries as default result metadata and store full trace artifacts as optional sidecar artifacts. - **Privacy mistakes:** Tool args/results may contain secrets or PII. Mitigation: default to redacted content and centralize capture policy. - **Phoenix scope creep:** It is tempting to mirror Phoenix concepts in core. Mitigation: keep Phoenix-specific dataset and experiment behavior in `packages/phoenix-adapter/`. @@ -346,7 +345,7 @@ Outside this product's identity: ## Sources And Research - `packages/core/src/evaluation/trace.ts` defines current `TraceSummary`, tool trajectory config, and execution metric helpers. -- `packages/core/src/evaluation/providers/types.ts` defines provider `Message` and `ToolCall`, the current richest AgentV trajectory source. +- `packages/core/src/evaluation/providers/types.ts` defines provider `Message` and `ToolCall`, the current richest AgentV trace source. - `packages/core/src/evaluation/graders/tool-trajectory.ts` currently grades from output messages first, then compact trace summaries. - `apps/cli/src/commands/inspect/utils.ts` currently imports AgentV OTLP JSON into raw results but mostly reconstructs summaries and requires AgentV result metadata for full support. - `apps/cli/src/commands/import/` already exposes `claude`, `codex`, and `copilot` transcript import subcommands for offline grading. diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index b2f16a004..41ee9610e 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -510,6 +510,7 @@ export interface PreparedAttemptMetadata { readonly preparedDir?: string; readonly workspacePath: string; readonly promptPath?: string; + readonly tracePath?: string; readonly target: string; readonly preparedAt?: string; readonly setupStatus?: string; @@ -532,6 +533,7 @@ export interface GradePreparedEvalCaseOptions { readonly workspacePath: string; readonly baselineCommit?: string; readonly response?: string; + readonly trace?: Trace; readonly verbose?: boolean; readonly threshold?: number; readonly preparedAttempt: PreparedAttemptMetadata; @@ -575,6 +577,7 @@ export async function gradePreparedEvalCase( workspacePath, baselineCommit, response, + trace: preparedTrace, verbose, threshold: caseThreshold, preparedAttempt, @@ -615,19 +618,22 @@ export async function gradePreparedEvalCase( } } - const candidate = response ?? ''; + const candidate = response ?? extractLastAssistantContent(preparedTrace?.messages ?? []) ?? ''; const input = buildResultInput(promptInputs); const outputMessages: readonly Message[] = - candidate.length > 0 ? [{ role: 'assistant' as const, content: candidate }] : []; - const resultTrace = buildTraceFromMessages({ - input, - output: outputMessages, - finalOutput: candidate, - provider: provider.kind, - target: target.name, - testId: evalCase.id, - conversationId: evalCase.conversation_id, - }); + preparedTrace?.messages ?? + (candidate.length > 0 ? [{ role: 'assistant' as const, content: candidate }] : []); + const resultTrace = + preparedTrace ?? + buildTraceFromMessages({ + input, + output: outputMessages, + finalOutput: candidate, + provider: provider.kind, + target: target.name, + testId: evalCase.id, + conversationId: evalCase.conversation_id, + }); try { const gradeStartedAt = nowFn(); @@ -642,6 +648,13 @@ export async function gradePreparedEvalCase( promptInputs, now: gradeStartedAt, agentTimeoutMs, + output: preparedTrace ? outputMessages : undefined, + trace: preparedTrace ? resultTrace : undefined, + costUsd: preparedTrace ? resultTrace.costUsd : undefined, + durationMs: preparedTrace ? resultTrace.durationMs : undefined, + tokenUsage: preparedTrace ? resultTrace.tokenUsage : undefined, + startTime: preparedTrace ? resultTrace.startTime : undefined, + endTime: preparedTrace ? resultTrace.endTime : undefined, targetResolver: runtime.targetResolver, availableTargets: runtime.availableTargets, fileChanges, diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index ad2e552db..27a1f659c 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -388,6 +388,7 @@ function toIndexPreparedAttempt(value: unknown): Record | undef prepared_dir: value.preparedDir, workspace_path: value.workspacePath, prompt_path: value.promptPath, + trace_path: value.tracePath, target: value.target, prepared_at: value.preparedAt, setup_status: value.setupStatus, diff --git a/packages/core/src/evaluation/trace-envelope.ts b/packages/core/src/evaluation/trace-envelope.ts index 38aa70097..34cd89f69 100644 --- a/packages/core/src/evaluation/trace-envelope.ts +++ b/packages/core/src/evaluation/trace-envelope.ts @@ -24,14 +24,13 @@ import { createHash } from 'node:crypto'; import { z } from 'zod'; import type { Message, ToolCall } from './providers/types.js'; -import { - NORMALIZED_TRAJECTORY_SCHEMA_VERSION, - type TokenUsage, - type TraceArtifact, - type TraceComputeResult, - type TraceEvent, - type TraceSourceKind, - type TraceSummary, +import type { + TokenUsage, + TraceArtifact, + TraceComputeResult, + TraceEvent, + TraceSourceKind, + TraceSummary, } from './trace.js'; import type { EvaluationResult, EvaluationVerdict, GraderKind } from './types.js'; @@ -367,7 +366,6 @@ export interface TraceEnvelopeToolTrajectoryItem { } export interface TraceEnvelopeToolTrajectoryView { - readonly schemaVersion: typeof NORMALIZED_TRAJECTORY_SCHEMA_VERSION; readonly traceId: string; readonly rootSpanId: string; readonly tools: readonly TraceEnvelopeToolTrajectoryItem[]; @@ -1494,7 +1492,6 @@ export function traceEnvelopeToToolTrajectoryView( }); return { - schemaVersion: NORMALIZED_TRAJECTORY_SCHEMA_VERSION, traceId: envelope.trace.traceId, rootSpanId: envelope.trace.rootSpanId, tools, @@ -1612,7 +1609,6 @@ export function traceEnvelopeToTraceArtifact(envelope: TraceEnvelope): TraceArti const summary = traceEnvelopeToTraceSummary(envelope); return { - schemaVersion: NORMALIZED_TRAJECTORY_SCHEMA_VERSION, source: { kind: envelope.source.kind as TraceSourceKind, path: envelope.source.path, diff --git a/packages/core/src/evaluation/trace.ts b/packages/core/src/evaluation/trace.ts index 86430af4a..ca93236f2 100644 --- a/packages/core/src/evaluation/trace.ts +++ b/packages/core/src/evaluation/trace.ts @@ -17,11 +17,9 @@ * serializer such as `toSnakeCaseDeep()`. */ import { z } from 'zod'; -import type { Message } from './providers/types.js'; +import type { Message, ToolCall } from './providers/types.js'; -export const NORMALIZED_TRAJECTORY_SCHEMA_VERSION = 'agentv.trajectory.v1' as const; - -export const NORMALIZED_TRACE_SOURCE_KINDS = [ +export const TRACE_SOURCE_KINDS = [ 'agentv_run', 'otlp', 'phoenix', @@ -31,7 +29,7 @@ export const NORMALIZED_TRACE_SOURCE_KINDS = [ 'compact_transcript', ] as const; -export const NORMALIZED_TRACE_EVENT_TYPES = [ +export const TRACE_EVENT_TYPES = [ 'message', 'model_turn', 'tool_call', @@ -40,30 +38,15 @@ export const NORMALIZED_TRACE_EVENT_TYPES = [ 'error', ] as const; -export const NORMALIZED_TOOL_STATUSES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const; - -export const NORMALIZED_REDACTION_LEVELS = ['none', 'partial', 'full'] as const; +export const TRACE_TOOL_STATUSES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const; -export const TRACE_SCHEMA_VERSION = NORMALIZED_TRAJECTORY_SCHEMA_VERSION; -export const TRACE_SOURCE_KINDS = NORMALIZED_TRACE_SOURCE_KINDS; -export const TRACE_EVENT_TYPES = NORMALIZED_TRACE_EVENT_TYPES; -export const TRACE_TOOL_STATUSES = NORMALIZED_TOOL_STATUSES; -export const TRACE_REDACTION_LEVELS = NORMALIZED_REDACTION_LEVELS; +export const TRACE_REDACTION_LEVELS = ['none', 'partial', 'full'] as const; export type TraceSourceKind = (typeof TRACE_SOURCE_KINDS)[number]; export type TraceEventType = (typeof TRACE_EVENT_TYPES)[number]; export type TraceToolStatus = (typeof TRACE_TOOL_STATUSES)[number]; export type TraceRedactionLevel = (typeof TRACE_REDACTION_LEVELS)[number]; -/** @deprecated Use TraceSourceKind. */ -export type NormalizedTraceSourceKind = TraceSourceKind; -/** @deprecated Use TraceEventType. */ -export type NormalizedTraceEventType = TraceEventType; -/** @deprecated Use TraceToolStatus. */ -export type NormalizedToolStatus = TraceToolStatus; -/** @deprecated Use TraceRedactionLevel. */ -export type NormalizedRedactionLevel = TraceRedactionLevel; - export interface TraceSource { readonly kind: TraceSourceKind; readonly path?: string; @@ -173,14 +156,13 @@ export interface TraceEvent { } /** - * Derived trajectory artifact shape used by import/replay helpers. + * Derived trace artifact shape used by import/replay helpers. * * New evaluation results use `Trace` below: final answer in `output`, full * transcript under `trace.messages`, structured spans under `trace.events`, and * provider-native session identifiers in `trace.metadata`. */ export interface TraceArtifact { - readonly schemaVersion: typeof NORMALIZED_TRAJECTORY_SCHEMA_VERSION; readonly source: TraceSource; readonly session: TraceSession; readonly branch?: TraceBranch; @@ -193,12 +175,6 @@ export interface TraceArtifact { readonly metadata?: Readonly>; } -/** - * @deprecated Use `Trace` for evaluation results or `TraceArtifact` for - * derived import/replay trajectory artifacts. - */ -export type NormalizedTrajectory = TraceArtifact; - function omitUndefinedProperties>(value: T): T { return Object.fromEntries( Object.entries(value).filter(([, property]) => property !== undefined), @@ -213,13 +189,13 @@ const TokenUsageWireSchema = z.object({ reasoning: z.number().optional(), }); -export const NormalizedRedactionStateWireSchema = z.object({ - level: z.enum(NORMALIZED_REDACTION_LEVELS), +export const TraceRedactionStateWireSchema = z.object({ + level: z.enum(TRACE_REDACTION_LEVELS), fields: z.array(z.string()).optional(), reason: z.string().optional(), }); -export const NormalizedTraceErrorWireSchema = z.object({ +export const TraceErrorWireSchema = z.object({ message: z.string(), name: z.string().optional(), code: z.string().optional(), @@ -227,8 +203,8 @@ export const NormalizedTraceErrorWireSchema = z.object({ metadata: MetadataWireSchema.optional(), }); -export const NormalizedTraceSourceWireSchema = z.object({ - kind: z.enum(NORMALIZED_TRACE_SOURCE_KINDS), +export const TraceSourceWireSchema = z.object({ + kind: z.enum(TRACE_SOURCE_KINDS), path: z.string().optional(), url: z.string().optional(), provider: z.string().optional(), @@ -237,7 +213,7 @@ export const NormalizedTraceSourceWireSchema = z.object({ metadata: MetadataWireSchema.optional(), }); -export const NormalizedTraceSessionWireSchema = z.object({ +export const TraceSessionWireSchema = z.object({ session_id: z.string().optional(), conversation_id: z.string().optional(), cwd: z.string().optional(), @@ -246,7 +222,7 @@ export const NormalizedTraceSessionWireSchema = z.object({ metadata: MetadataWireSchema.optional(), }); -export const NormalizedTraceBranchWireSchema = z.object({ +export const TraceBranchWireSchema = z.object({ selected_leaf_id: z.string().optional(), selected_path_ids: z.array(z.string()).optional(), included_event_ids: z.array(z.string()).optional(), @@ -254,7 +230,7 @@ export const NormalizedTraceBranchWireSchema = z.object({ selection_reason: z.string().optional(), }); -export const NormalizedTraceSourceRefWireSchema = z.object({ +export const TraceSourceRefWireSchema = z.object({ event_id: z.string().optional(), message_id: z.string().optional(), span_id: z.string().optional(), @@ -265,7 +241,7 @@ export const NormalizedTraceSourceRefWireSchema = z.object({ metadata: MetadataWireSchema.optional(), }); -export const NormalizedRawEvidenceWireSchema = z.object({ +export const TraceRawEvidenceWireSchema = z.object({ kind: z.string(), ref: z.string().optional(), media_type: z.string().optional(), @@ -274,16 +250,16 @@ export const NormalizedRawEvidenceWireSchema = z.object({ metadata: MetadataWireSchema.optional(), }); -export const NormalizedTraceMessageWireSchema = z.object({ +export const TraceMessageWireSchema = z.object({ role: z.string(), name: z.string().optional(), content: z.unknown().optional(), - redaction: NormalizedRedactionStateWireSchema.optional(), + redaction: TraceRedactionStateWireSchema.optional(), token_usage: TokenUsageWireSchema.optional(), metadata: MetadataWireSchema.optional(), }); -export const NormalizedTraceModelWireSchema = z.object({ +export const TraceModelWireSchema = z.object({ provider: z.string().optional(), name: z.string().optional(), invocation_id: z.string().optional(), @@ -291,42 +267,41 @@ export const NormalizedTraceModelWireSchema = z.object({ metadata: MetadataWireSchema.optional(), }); -export const NormalizedTraceToolWireSchema = z.object({ +export const TraceToolWireSchema = z.object({ name: z.string(), call_id: z.string().optional(), input: z.unknown().optional(), output: z.unknown().optional(), - status: z.enum(NORMALIZED_TOOL_STATUSES).optional(), - error: NormalizedTraceErrorWireSchema.optional(), - redaction: NormalizedRedactionStateWireSchema.optional(), + status: z.enum(TRACE_TOOL_STATUSES).optional(), + error: TraceErrorWireSchema.optional(), + redaction: TraceRedactionStateWireSchema.optional(), metadata: MetadataWireSchema.optional(), }); -export const NormalizedTraceEventWireSchema = z.object({ +export const TraceEventWireSchema = z.object({ event_id: z.string(), parent_event_id: z.string().optional(), ordinal: z.number().int().nonnegative(), - type: z.enum(NORMALIZED_TRACE_EVENT_TYPES), + type: z.enum(TRACE_EVENT_TYPES), timestamp: z.string().optional(), duration_ms: z.number().nonnegative().optional(), duration_inferred: z.boolean().optional(), turn_index: z.number().int().nonnegative().optional(), - message: NormalizedTraceMessageWireSchema.optional(), - model: NormalizedTraceModelWireSchema.optional(), - tool: NormalizedTraceToolWireSchema.optional(), - error: NormalizedTraceErrorWireSchema.optional(), - source_ref: NormalizedTraceSourceRefWireSchema.optional(), - raw_evidence: z.array(NormalizedRawEvidenceWireSchema).optional(), - redaction: NormalizedRedactionStateWireSchema.optional(), + message: TraceMessageWireSchema.optional(), + model: TraceModelWireSchema.optional(), + tool: TraceToolWireSchema.optional(), + error: TraceErrorWireSchema.optional(), + source_ref: TraceSourceRefWireSchema.optional(), + raw_evidence: z.array(TraceRawEvidenceWireSchema).optional(), + redaction: TraceRedactionStateWireSchema.optional(), metadata: MetadataWireSchema.optional(), }); -export const NormalizedTrajectoryWireSchema = z.object({ - schema_version: z.literal(NORMALIZED_TRAJECTORY_SCHEMA_VERSION), - source: NormalizedTraceSourceWireSchema, - session: NormalizedTraceSessionWireSchema, - branch: NormalizedTraceBranchWireSchema.optional(), - events: z.array(NormalizedTraceEventWireSchema), +export const TraceArtifactWireSchema = z.object({ + source: TraceSourceWireSchema, + session: TraceSessionWireSchema, + branch: TraceBranchWireSchema.optional(), + events: z.array(TraceEventWireSchema), token_usage: TokenUsageWireSchema.optional(), cost_usd: z.number().optional(), duration_ms: z.number().optional(), @@ -335,52 +310,34 @@ export const NormalizedTrajectoryWireSchema = z.object({ metadata: MetadataWireSchema.optional(), }); -export type NormalizedTrajectoryWire = z.infer; -export type NormalizedTraceEventWire = z.infer; - -export const TraceRedactionStateWireSchema = NormalizedRedactionStateWireSchema; -export const TraceErrorWireSchema = NormalizedTraceErrorWireSchema; -export const TraceSourceWireSchema = NormalizedTraceSourceWireSchema; -export const TraceSessionWireSchema = NormalizedTraceSessionWireSchema; -export const TraceBranchWireSchema = NormalizedTraceBranchWireSchema; -export const TraceSourceRefWireSchema = NormalizedTraceSourceRefWireSchema; -export const TraceRawEvidenceWireSchema = NormalizedRawEvidenceWireSchema; -export const TraceMessageWireSchema = NormalizedTraceMessageWireSchema; -export const TraceModelWireSchema = NormalizedTraceModelWireSchema; -export const TraceToolWireSchema = NormalizedTraceToolWireSchema; -export const TraceEventWireSchema = NormalizedTraceEventWireSchema; -export const TraceArtifactWireSchema = NormalizedTrajectoryWireSchema; - -export type TraceArtifactWire = NormalizedTrajectoryWire; -export type TraceEventWire = NormalizedTraceEventWire; - -export function toNormalizedTrajectoryWire(trajectory: TraceArtifact): NormalizedTrajectoryWire { - return NormalizedTrajectoryWireSchema.parse( +export type TraceEventWire = z.infer; +export type TraceArtifactWire = z.infer; + +export function toTraceArtifactWire(artifact: TraceArtifact): TraceArtifactWire { + return TraceArtifactWireSchema.parse( omitUndefinedProperties({ - schema_version: trajectory.schemaVersion, - source: toNormalizedTraceSourceWire(trajectory.source), - session: toNormalizedTraceSessionWire(trajectory.session), - branch: trajectory.branch ? toNormalizedTraceBranchWire(trajectory.branch) : undefined, - events: trajectory.events.map(toNormalizedTraceEventWire), - token_usage: trajectory.tokenUsage, - cost_usd: trajectory.costUsd, - duration_ms: trajectory.durationMs, - started_at: trajectory.startedAt, - ended_at: trajectory.endedAt, - metadata: trajectory.metadata, + source: toTraceSourceWire(artifact.source), + session: toTraceSessionWire(artifact.session), + branch: artifact.branch ? toTraceBranchWire(artifact.branch) : undefined, + events: artifact.events.map(toTraceEventWire), + token_usage: artifact.tokenUsage, + cost_usd: artifact.costUsd, + duration_ms: artifact.durationMs, + started_at: artifact.startedAt, + ended_at: artifact.endedAt, + metadata: artifact.metadata, }), ); } -export function fromNormalizedTrajectoryWire(input: unknown): TraceArtifact { - const wire = NormalizedTrajectoryWireSchema.parse(input); +export function fromTraceArtifactWire(input: unknown): TraceArtifact { + const wire = TraceArtifactWireSchema.parse(input); return { - schemaVersion: wire.schema_version, - source: fromNormalizedTraceSourceWire(wire.source), - session: fromNormalizedTraceSessionWire(wire.session), - branch: wire.branch ? fromNormalizedTraceBranchWire(wire.branch) : undefined, - events: wire.events.map(fromNormalizedTraceEventWire), + source: fromTraceSourceWire(wire.source), + session: fromTraceSessionWire(wire.session), + branch: wire.branch ? fromTraceBranchWire(wire.branch) : undefined, + events: wire.events.map(fromTraceEventWire), tokenUsage: wire.token_usage, costUsd: wire.cost_usd, durationMs: wire.duration_ms, @@ -390,15 +347,7 @@ export function fromNormalizedTrajectoryWire(input: unknown): TraceArtifact { }; } -export function toTraceArtifactWire(artifact: TraceArtifact): TraceArtifactWire { - return toNormalizedTrajectoryWire(artifact); -} - -export function fromTraceArtifactWire(input: unknown): TraceArtifact { - return fromNormalizedTrajectoryWire(input); -} - -function toNormalizedTraceSourceWire(source: TraceSource) { +function toTraceSourceWire(source: TraceSource) { return omitUndefinedProperties({ kind: source.kind, path: source.path, @@ -410,9 +359,7 @@ function toNormalizedTraceSourceWire(source: TraceSource) { }); } -function fromNormalizedTraceSourceWire( - source: z.infer, -): TraceSource { +function fromTraceSourceWire(source: z.infer): TraceSource { return { kind: source.kind, path: source.path, @@ -424,7 +371,7 @@ function fromNormalizedTraceSourceWire( }; } -function toNormalizedTraceSessionWire(session: TraceSession) { +function toTraceSessionWire(session: TraceSession) { return omitUndefinedProperties({ session_id: session.sessionId, conversation_id: session.conversationId, @@ -435,9 +382,7 @@ function toNormalizedTraceSessionWire(session: TraceSession) { }); } -function fromNormalizedTraceSessionWire( - session: z.infer, -): TraceSession { +function fromTraceSessionWire(session: z.infer): TraceSession { return { sessionId: session.session_id, conversationId: session.conversation_id, @@ -448,7 +393,7 @@ function fromNormalizedTraceSessionWire( }; } -function toNormalizedTraceBranchWire(branch: TraceBranch) { +function toTraceBranchWire(branch: TraceBranch) { return omitUndefinedProperties({ selected_leaf_id: branch.selectedLeafId, selected_path_ids: branch.selectedPathIds, @@ -458,9 +403,7 @@ function toNormalizedTraceBranchWire(branch: TraceBranch) { }); } -function fromNormalizedTraceBranchWire( - branch: z.infer, -): TraceBranch { +function fromTraceBranchWire(branch: z.infer): TraceBranch { return { selectedLeafId: branch.selected_leaf_id, selectedPathIds: branch.selected_path_ids, @@ -470,8 +413,8 @@ function fromNormalizedTraceBranchWire( }; } -function toNormalizedTraceEventWire(event: TraceEvent): NormalizedTraceEventWire { - return NormalizedTraceEventWireSchema.parse( +function toTraceEventWire(event: TraceEvent): TraceEventWire { + return TraceEventWireSchema.parse( omitUndefinedProperties({ event_id: event.eventId, parent_event_id: event.parentEventId, @@ -481,19 +424,19 @@ function toNormalizedTraceEventWire(event: TraceEvent): NormalizedTraceEventWire duration_ms: event.durationMs, duration_inferred: event.durationInferred, turn_index: event.turnIndex, - message: event.message ? toNormalizedTraceMessageWire(event.message) : undefined, - model: event.model ? toNormalizedTraceModelWire(event.model) : undefined, - tool: event.tool ? toNormalizedTraceToolWire(event.tool) : undefined, - error: event.error ? toNormalizedTraceErrorWire(event.error) : undefined, - source_ref: event.sourceRef ? toNormalizedTraceSourceRefWire(event.sourceRef) : undefined, - raw_evidence: event.rawEvidence?.map(toNormalizedRawEvidenceWire), + message: event.message ? toTraceMessageWire(event.message) : undefined, + model: event.model ? toTraceModelWire(event.model) : undefined, + tool: event.tool ? toTraceToolWire(event.tool) : undefined, + error: event.error ? toTraceErrorWire(event.error) : undefined, + source_ref: event.sourceRef ? toTraceSourceRefWire(event.sourceRef) : undefined, + raw_evidence: event.rawEvidence?.map(toTraceRawEvidenceWire), redaction: event.redaction, metadata: event.metadata, }), ); } -function fromNormalizedTraceEventWire(event: NormalizedTraceEventWire): TraceEvent { +function fromTraceEventWire(event: TraceEventWire): TraceEvent { return { eventId: event.event_id, parentEventId: event.parent_event_id, @@ -503,18 +446,18 @@ function fromNormalizedTraceEventWire(event: NormalizedTraceEventWire): TraceEve durationMs: event.duration_ms, durationInferred: event.duration_inferred, turnIndex: event.turn_index, - message: event.message ? fromNormalizedTraceMessageWire(event.message) : undefined, - model: event.model ? fromNormalizedTraceModelWire(event.model) : undefined, - tool: event.tool ? fromNormalizedTraceToolWire(event.tool) : undefined, - error: event.error ? fromNormalizedTraceErrorWire(event.error) : undefined, - sourceRef: event.source_ref ? fromNormalizedTraceSourceRefWire(event.source_ref) : undefined, - rawEvidence: event.raw_evidence?.map(fromNormalizedRawEvidenceWire), + message: event.message ? fromTraceMessageWire(event.message) : undefined, + model: event.model ? fromTraceModelWire(event.model) : undefined, + tool: event.tool ? fromTraceToolWire(event.tool) : undefined, + error: event.error ? fromTraceErrorWire(event.error) : undefined, + sourceRef: event.source_ref ? fromTraceSourceRefWire(event.source_ref) : undefined, + rawEvidence: event.raw_evidence?.map(fromTraceRawEvidenceWire), redaction: event.redaction, metadata: event.metadata, }; } -function toNormalizedTraceMessageWire(message: TraceMessage) { +function toTraceMessageWire(message: TraceMessage) { return omitUndefinedProperties({ role: message.role, name: message.name, @@ -525,9 +468,7 @@ function toNormalizedTraceMessageWire(message: TraceMessage) { }); } -function fromNormalizedTraceMessageWire( - message: z.infer, -): TraceMessage { +function fromTraceMessageWire(message: z.infer): TraceMessage { return { role: message.role, name: message.name, @@ -538,7 +479,7 @@ function fromNormalizedTraceMessageWire( }; } -function toNormalizedTraceModelWire(model: TraceModel) { +function toTraceModelWire(model: TraceModel) { return omitUndefinedProperties({ provider: model.provider, name: model.name, @@ -548,9 +489,7 @@ function toNormalizedTraceModelWire(model: TraceModel) { }); } -function fromNormalizedTraceModelWire( - model: z.infer, -): TraceModel { +function fromTraceModelWire(model: z.infer): TraceModel { return { provider: model.provider, name: model.name, @@ -560,7 +499,7 @@ function fromNormalizedTraceModelWire( }; } -function toNormalizedTraceToolWire(tool: TraceTool) { +function toTraceToolWire(tool: TraceTool) { return omitUndefinedProperties({ name: tool.name, call_id: tool.callId, @@ -573,9 +512,7 @@ function toNormalizedTraceToolWire(tool: TraceTool) { }); } -function fromNormalizedTraceToolWire( - tool: z.infer, -): TraceTool { +function fromTraceToolWire(tool: z.infer): TraceTool { return { name: tool.name, callId: tool.call_id, @@ -588,7 +525,7 @@ function fromNormalizedTraceToolWire( }; } -function toNormalizedTraceErrorWire(error: TraceError) { +function toTraceErrorWire(error: TraceError) { return omitUndefinedProperties({ message: error.message, name: error.name, @@ -598,9 +535,7 @@ function toNormalizedTraceErrorWire(error: TraceError) { }); } -function fromNormalizedTraceErrorWire( - error: z.infer, -): TraceError { +function fromTraceErrorWire(error: z.infer): TraceError { return { message: error.message, name: error.name, @@ -610,7 +545,7 @@ function fromNormalizedTraceErrorWire( }; } -function toNormalizedTraceSourceRefWire(sourceRef: TraceSourceRef) { +function toTraceSourceRefWire(sourceRef: TraceSourceRef) { return omitUndefinedProperties({ event_id: sourceRef.eventId, message_id: sourceRef.messageId, @@ -623,8 +558,8 @@ function toNormalizedTraceSourceRefWire(sourceRef: TraceSourceRef) { }); } -function fromNormalizedTraceSourceRefWire( - sourceRef: z.infer, +function fromTraceSourceRefWire( + sourceRef: z.infer, ): TraceSourceRef { return { eventId: sourceRef.event_id, @@ -638,7 +573,7 @@ function fromNormalizedTraceSourceRefWire( }; } -function toNormalizedRawEvidenceWire(evidence: TraceRawEvidence) { +function toTraceRawEvidenceWire(evidence: TraceRawEvidence) { return omitUndefinedProperties({ kind: evidence.kind, ref: evidence.ref, @@ -649,8 +584,8 @@ function toNormalizedRawEvidenceWire(evidence: TraceRawEvidence) { }); } -function fromNormalizedRawEvidenceWire( - evidence: z.infer, +function fromTraceRawEvidenceWire( + evidence: z.infer, ): TraceRawEvidence { return { kind: evidence.kind, @@ -662,32 +597,6 @@ function fromNormalizedRawEvidenceWire( }; } -// Deprecated compatibility names retained for callers that imported the older -// normalized-trace terminology. New code should use the AgentV-owned Trace* -// names above. -/** @deprecated Use TraceSource. */ -export type NormalizedTraceSource = TraceSource; -/** @deprecated Use TraceSession. */ -export type NormalizedTraceSession = TraceSession; -/** @deprecated Use TraceBranch. */ -export type NormalizedTraceBranch = TraceBranch; -/** @deprecated Use TraceSourceRef. */ -export type NormalizedTraceSourceRef = TraceSourceRef; -/** @deprecated Use TraceRawEvidence. */ -export type NormalizedRawEvidence = TraceRawEvidence; -/** @deprecated Use TraceRedactionState. */ -export type NormalizedRedactionState = TraceRedactionState; -/** @deprecated Use TraceError. */ -export type NormalizedTraceError = TraceError; -/** @deprecated Use TraceMessage. */ -export type NormalizedTraceMessage = TraceMessage; -/** @deprecated Use TraceModel. */ -export type NormalizedTraceModel = TraceModel; -/** @deprecated Use TraceTool. */ -export type NormalizedTraceTool = TraceTool; -/** @deprecated Use TraceEvent. */ -export type NormalizedTraceEvent = TraceEvent; - /** * Token usage metrics from provider execution. */ @@ -734,7 +643,6 @@ export interface TraceSummary { * the canonical trace artifact and derive this shape from it. */ export interface Trace extends TraceSummary { - readonly schemaVersion: typeof TRACE_SCHEMA_VERSION; /** Complete chat transcript used for transcript-aware graders. */ readonly messages: readonly Message[]; /** Structured event stream derived from the same messages and metrics. */ @@ -917,7 +825,6 @@ export function buildTraceFromMessages(options: BuildTraceOptions = {}): Trace { } return { - schemaVersion: TRACE_SCHEMA_VERSION, eventCount: summary.eventCount, toolCalls: summary.toolCalls, errorCount: summary.errorCount + (options.error ? 1 : 0), @@ -1011,7 +918,7 @@ export interface ToolTrajectoryGraderConfig { } /** - * Expected tool call item in a trajectory sequence. + * Expected tool call item in a tool sequence. */ export interface ToolTrajectoryExpectedItem { readonly tool: string; @@ -1041,7 +948,7 @@ interface MessageLike { /** * Compute a lightweight summary from provider output messages. - * Used for legacy/default result persistence when no full trajectory is present. + * Used for legacy/default result persistence when no trace artifact is present. * * Derives timing information from span boundaries: * - startTime: earliest startTime across all messages and tool calls @@ -1130,35 +1037,35 @@ export function computeTraceSummary(messages: readonly MessageLike[]): TraceComp } /** - * Return the trajectory events selected for grading. + * Return the trace artifact events selected for grading. * * Importers should already store the selected branch path in `events`. When a * source also carries explicit `branch.includedEventIds`, honor it here so * branchable transcripts cannot accidentally grade omitted alternatives. */ -export function getSelectedTrajectoryEvents(trajectory: TraceArtifact): readonly TraceEvent[] { - if (!trajectory.branch?.includedEventIds || trajectory.branch.includedEventIds.length === 0) { - return trajectory.events; +export function getSelectedTraceArtifactEvents(artifact: TraceArtifact): readonly TraceEvent[] { + if (!artifact.branch?.includedEventIds || artifact.branch.includedEventIds.length === 0) { + return artifact.events; } - const includedIds = new Set(trajectory.branch.includedEventIds); - return trajectory.events.filter((event) => includedIds.has(event.eventId)); + const includedIds = new Set(artifact.branch.includedEventIds); + return artifact.events.filter((event) => includedIds.has(event.eventId)); } /** - * Derive the existing compact TraceSummary shape from a full trajectory. + * Derive the existing compact TraceSummary shape from a trace artifact. * - * This is the canonical bridge from the high-fidelity trajectory contract to the - * backward-compatible summary/read model. Keep the projection one-way: importers - * and replay should preserve TraceArtifact or Trace, while existing result readers - * can continue consuming the derived TraceSummary shape unchanged. + * This is the canonical bridge from the high-fidelity trace artifact to the + * summary/read model. Keep the projection one-way: importers and replay should + * preserve TraceArtifact or Trace, while existing result readers can continue + * consuming the derived TraceSummary shape unchanged. * * The summary keeps the current lightweight contract: eventCount is the number * of tool-call events, toolCalls is counted by tool name, toolDurations carries * per-tool milliseconds when present, and llmCallCount counts model turns. */ -export function computeTraceSummaryFromTrajectory(trajectory: TraceArtifact): TraceComputeResult { - const selectedEvents = getSelectedTrajectoryEvents(trajectory); +export function computeTraceSummaryFromTraceArtifact(artifact: TraceArtifact): TraceComputeResult { + const selectedEvents = getSelectedTraceArtifactEvents(artifact); const hasModelTurnEvents = selectedEvents.some((event) => event.type === 'model_turn'); const toolCallCounts: Record = {}; const toolDurations: Record = {}; @@ -1215,11 +1122,197 @@ export function computeTraceSummaryFromTrajectory(trajectory: TraceArtifact): Tr llmCallCount, ...(hasAnyDuration ? { toolDurations } : {}), }, - tokenUsage: trajectory.tokenUsage, - costUsd: trajectory.costUsd, - durationMs: trajectory.durationMs, - startTime: trajectory.startedAt ?? earliestStart?.toISOString(), - endTime: trajectory.endedAt ?? latestEnd?.toISOString(), + tokenUsage: artifact.tokenUsage, + costUsd: artifact.costUsd, + durationMs: artifact.durationMs, + startTime: artifact.startedAt ?? earliestStart?.toISOString(), + endTime: artifact.endedAt ?? latestEnd?.toISOString(), + }; +} + +function eventEndIso(event: TraceEvent): string | undefined { + const start = parseTimestamp(event.timestamp); + return deriveEventEnd(start, event.durationMs)?.toISOString(); +} + +function messageFromTraceArtifactEvent(event: TraceEvent, fallbackRole = 'assistant'): Message { + return { + role: event.message?.role ?? fallbackRole, + name: event.message?.name, + content: event.message?.content as Message['content'], + startTime: event.timestamp, + endTime: eventEndIso(event), + durationMs: event.durationMs, + metadata: omitUndefinedProperties({ + event_id: event.eventId, + source_ref: event.sourceRef, + ...event.message?.metadata, + }), + tokenUsage: event.message?.tokenUsage ?? event.model?.tokenUsage, + }; +} + +function toolCallFromTraceArtifactEvent(event: TraceEvent): ToolCall | undefined { + const tool = event.tool; + if (!tool) { + return undefined; + } + + return { + tool: tool.name, + id: tool.callId, + input: tool.input, + output: tool.output, + startTime: event.timestamp, + endTime: eventEndIso(event), + durationMs: event.durationMs, + }; +} + +function attachTraceArtifactToolCall( + messages: Message[], + eventMessageIndex: Map, + event: TraceEvent, +): void { + const toolCall = toolCallFromTraceArtifactEvent(event); + if (!toolCall) { + return; + } + + const parentIndex = event.parentEventId ? eventMessageIndex.get(event.parentEventId) : undefined; + let fallbackIndex = -1; + for (let index = messages.length - 1; index >= 0; index--) { + if (messages[index].role === 'assistant') { + fallbackIndex = index; + break; + } + } + const messageIndex = parentIndex ?? fallbackIndex; + + if (messageIndex >= 0) { + const message = messages[messageIndex]; + messages[messageIndex] = { + ...message, + toolCalls: [...(message.toolCalls ?? []), toolCall], + }; + return; + } + + const syntheticIndex = messages.length; + messages.push({ + role: 'assistant', + startTime: event.timestamp, + endTime: eventEndIso(event), + durationMs: event.durationMs, + metadata: { source: 'trace_artifact_tool_call', event_id: event.eventId }, + toolCalls: [toolCall], + }); + eventMessageIndex.set(event.eventId, syntheticIndex); +} + +function mergeTraceArtifactFinalResponse( + messages: Message[], + eventMessageIndex: Map, + event: TraceEvent, +): boolean { + if (!event.parentEventId || !event.message) { + return false; + } + + const parentIndex = eventMessageIndex.get(event.parentEventId); + if (parentIndex === undefined) { + return false; + } + + const parent = messages[parentIndex]; + const finalMessage = messageFromTraceArtifactEvent(event); + eventMessageIndex.set(event.eventId, parentIndex); + + if (sameMessageContent(parent, finalMessage)) { + return true; + } + + if (parent.content === undefined && finalMessage.content !== undefined) { + messages[parentIndex] = { + ...parent, + name: parent.name ?? finalMessage.name, + content: finalMessage.content, + endTime: parent.endTime ?? finalMessage.endTime, + tokenUsage: parent.tokenUsage ?? finalMessage.tokenUsage, + metadata: omitUndefinedProperties({ + ...parent.metadata, + final_response_event_id: event.eventId, + final_response_source_ref: event.sourceRef, + }), + }; + return true; + } + + return false; +} + +/** + * Project a trace artifact into the result-local Trace read model used by + * graders. Message events preserve transcript content when available; + * model-turn events provide assistant message anchors for child tool calls; + * standalone tool calls attach to the nearest assistant message. + */ +export function traceFromTraceArtifact(artifact: TraceArtifact): Trace { + const events = [...getSelectedTraceArtifactEvents(artifact)].sort( + (first, second) => first.ordinal - second.ordinal, + ); + const messages: Message[] = []; + const eventMessageIndex = new Map(); + + for (const event of events) { + if (event.type === 'message' && event.message) { + eventMessageIndex.set(event.eventId, messages.length); + messages.push(messageFromTraceArtifactEvent(event)); + continue; + } + + if (event.type === 'model_turn') { + eventMessageIndex.set(event.eventId, messages.length); + messages.push(messageFromTraceArtifactEvent(event, 'assistant')); + continue; + } + + if (event.type === 'final_response' && event.message) { + if (mergeTraceArtifactFinalResponse(messages, eventMessageIndex, event)) { + continue; + } + eventMessageIndex.set(event.eventId, messages.length); + messages.push(messageFromTraceArtifactEvent(event)); + continue; + } + + if (event.type === 'tool_call') { + attachTraceArtifactToolCall(messages, eventMessageIndex, event); + } + } + + const computed = computeTraceSummaryFromTraceArtifact(artifact); + const conversationId = artifact.session.sessionId ?? artifact.session.conversationId; + return { + eventCount: computed.trace.eventCount, + toolCalls: computed.trace.toolCalls, + errorCount: computed.trace.errorCount, + llmCallCount: computed.trace.llmCallCount, + ...(computed.trace.toolDurations ? { toolDurations: computed.trace.toolDurations } : {}), + messages, + events, + tokenUsage: computed.tokenUsage, + costUsd: computed.costUsd, + durationMs: computed.durationMs, + startTime: computed.startTime, + endTime: computed.endTime, + metadata: { + ...(artifact.source.provider ? { provider: artifact.source.provider } : {}), + ...(conversationId ? { provider_session_id: conversationId } : {}), + source: artifact.source, + session: artifact.session, + ...artifact.metadata, + }, }; } diff --git a/packages/core/test/evaluation/trace-envelope.test.ts b/packages/core/test/evaluation/trace-envelope.test.ts index 205b46c0c..6b15eaff7 100644 --- a/packages/core/test/evaluation/trace-envelope.test.ts +++ b/packages/core/test/evaluation/trace-envelope.test.ts @@ -18,11 +18,7 @@ import { traceEnvelopeToTraceSummary, traceEnvelopeToTranscriptMessages, } from '../../src/evaluation/trace-envelope.js'; -import { - NORMALIZED_TRAJECTORY_SCHEMA_VERSION, - buildTraceFromMessages, - computeTraceSummary, -} from '../../src/evaluation/trace.js'; +import { buildTraceFromMessages, computeTraceSummary } from '../../src/evaluation/trace.js'; import type { EvaluationResult } from '../../src/evaluation/types.js'; function jsonComparable(value: unknown): unknown { @@ -312,7 +308,7 @@ describe('execution trace artifact v1', () => { }); }); - it('projects Message[], TraceSummary, trajectory, tool grader input, and OTLP JSON from spans', () => { + it('projects Message[], TraceSummary, trace artifact, tool grader input, and OTLP JSON from spans', () => { const output: readonly Message[] = [ { role: 'assistant', @@ -333,7 +329,7 @@ describe('execution trace artifact v1', () => { const envelope = buildTraceEnvelopeFromEvaluationResult(result); const messages = traceEnvelopeToMessages(envelope); const summary = traceEnvelopeToTraceSummary(envelope); - const trajectory = traceEnvelopeToTraceArtifact(envelope); + const traceArtifact = traceEnvelopeToTraceArtifact(envelope); const compact = traceEnvelopeToToolTrajectoryView(envelope); const otlp = traceEnvelopeToOtlpJson(envelope); const grader = new ToolTrajectoryGrader({ @@ -347,12 +343,12 @@ describe('execution trace artifact v1', () => { expect(summary.trace).toEqual(computeTraceSummary(output).trace); expect(messages[0]?.toolCalls?.map((toolCall) => toolCall.tool)).toEqual(['Read', 'Edit']); - expect(trajectory.events.map((event) => event.type)).toEqual([ + expect(traceArtifact.events.map((event) => event.type)).toEqual([ 'model_turn', 'tool_call', 'tool_call', ]); - expect(compact.schemaVersion).toBe(NORMALIZED_TRAJECTORY_SCHEMA_VERSION); + expect(compact).not.toHaveProperty('schemaVersion'); expect(compact.tools.map((tool) => [tool.position, tool.tool, tool.toolCallId])).toEqual([ [0, 'Read', 'call-read'], [1, 'Edit', 'call-edit'], diff --git a/packages/core/test/evaluation/trace-trajectory.test.ts b/packages/core/test/evaluation/trace-trajectory.test.ts index d6294ca70..2b4e978aa 100644 --- a/packages/core/test/evaluation/trace-trajectory.test.ts +++ b/packages/core/test/evaluation/trace-trajectory.test.ts @@ -1,22 +1,21 @@ import { describe, expect, it } from 'bun:test'; import { - NORMALIZED_TRAJECTORY_SCHEMA_VERSION, - type NormalizedTrajectory, - NormalizedTrajectoryWireSchema, + type TraceArtifact, + TraceArtifactWireSchema, computeTraceSummary, - computeTraceSummaryFromTrajectory, - fromNormalizedTrajectoryWire, - toNormalizedTrajectoryWire, + computeTraceSummaryFromTraceArtifact, + fromTraceArtifactWire, + toTraceArtifactWire, + traceFromTraceArtifact, } from '../../src/evaluation/trace.js'; function jsonComparable(value: unknown): unknown { return JSON.parse(JSON.stringify(value)); } -function buildTrajectory(): NormalizedTrajectory { +function buildTraceArtifact(): TraceArtifact { return { - schemaVersion: NORMALIZED_TRAJECTORY_SCHEMA_VERSION, source: { kind: 'pi_session', path: 'traces/pi-session.jsonl', @@ -135,13 +134,13 @@ function buildTrajectory(): NormalizedTrajectory { }; } -describe('derived trajectory contract', () => { - it('round-trips between internal camelCase and snake_case wire format', () => { - const trajectory = buildTrajectory(); +describe('derived trace artifact projection', () => { + it('round-trips between internal camelCase and boundary snake_case shape', () => { + const artifact = buildTraceArtifact(); - const wire = toNormalizedTrajectoryWire(trajectory); + const wire = toTraceArtifactWire(artifact); - expect(wire.schema_version).toBe('agentv.trajectory.v1'); + expect(wire).not.toHaveProperty('schema_version'); expect(wire.source.kind).toBe('pi_session'); expect(wire.session.session_id).toBe('session-123'); expect(wire.branch?.selected_leaf_id).toBe('leaf-success'); @@ -150,25 +149,13 @@ describe('derived trajectory contract', () => { expect(wire.events[2]?.source_ref?.raw_kind).toBe('pi.toolCall'); expect(wire.events[2]?.raw_evidence?.[0]?.media_type).toBe('application/json'); - const roundTrip = fromNormalizedTrajectoryWire(wire); + const roundTrip = fromTraceArtifactWire(wire); - expect(jsonComparable(roundTrip)).toEqual(jsonComparable(trajectory)); - }); - - it('rejects unsupported schema versions', () => { - const wire = toNormalizedTrajectoryWire(buildTrajectory()); - - expect(() => - fromNormalizedTrajectoryWire({ - ...wire, - schema_version: 'agentv.trace.v2', - }), - ).toThrow(); + expect(jsonComparable(roundTrip)).toEqual(jsonComparable(artifact)); }); it('validates missing optional content without fabricating fields', () => { - const trajectory: NormalizedTrajectory = { - schemaVersion: NORMALIZED_TRAJECTORY_SCHEMA_VERSION, + const artifact: TraceArtifact = { source: { kind: 'imported_transcript' }, session: { sessionId: 'minimal-session' }, events: [ @@ -187,17 +174,17 @@ describe('derived trajectory contract', () => { ], }; - const wire = toNormalizedTrajectoryWire(trajectory); - const roundTrip = fromNormalizedTrajectoryWire(wire); + const wire = toTraceArtifactWire(artifact); + const roundTrip = fromTraceArtifactWire(wire); expect(wire.events[0]?.message).not.toHaveProperty('content'); expect(wire.events[1]?.tool).not.toHaveProperty('input'); - expect(jsonComparable(roundTrip)).toEqual(jsonComparable(trajectory)); + expect(jsonComparable(roundTrip)).toEqual(jsonComparable(artifact)); }); it('preserves branch metadata and derives summaries from the selected branch path', () => { - const trajectory: NormalizedTrajectory = { - ...buildTrajectory(), + const artifact: TraceArtifact = { + ...buildTraceArtifact(), branch: { selectedLeafId: 'leaf-success', includedEventIds: ['evt-read'], @@ -205,8 +192,8 @@ describe('derived trajectory contract', () => { }, }; - const wire = toNormalizedTrajectoryWire(trajectory); - const summary = computeTraceSummaryFromTrajectory(fromNormalizedTrajectoryWire(wire)); + const wire = toTraceArtifactWire(artifact); + const summary = computeTraceSummaryFromTraceArtifact(fromTraceArtifactWire(wire)); expect(wire.branch).toEqual({ selected_leaf_id: 'leaf-success', @@ -219,7 +206,7 @@ describe('derived trajectory contract', () => { }); it('preserves redaction, raw evidence, source refs, inferred timing, and tool errors', () => { - const wire = toNormalizedTrajectoryWire(buildTrajectory()); + const wire = toTraceArtifactWire(buildTraceArtifact()); const readEvent = wire.events.find((event) => event.event_id === 'evt-read'); const errorEvent = wire.events.find((event) => event.event_id === 'evt-write-error'); @@ -249,7 +236,7 @@ describe('derived trajectory contract', () => { code: 'EACCES', }); - const summary = computeTraceSummaryFromTrajectory(fromNormalizedTrajectoryWire(wire)); + const summary = computeTraceSummaryFromTraceArtifact(fromTraceArtifactWire(wire)); expect(summary.trace.errorCount).toBe(1); expect(summary.trace.toolDurations).toEqual({ @@ -258,7 +245,7 @@ describe('derived trajectory contract', () => { }); }); - it('derives the existing TraceSummary shape from full trajectories', () => { + it('derives the existing TraceSummary shape from trace artifacts', () => { const messages = [ { role: 'user', startTime: '2026-06-08T10:00:00Z' }, { @@ -280,8 +267,7 @@ describe('derived trajectory contract', () => { ], }, ]; - const trajectory: NormalizedTrajectory = { - schemaVersion: NORMALIZED_TRAJECTORY_SCHEMA_VERSION, + const artifact: TraceArtifact = { source: { kind: 'agentv_run' }, session: { sessionId: 'agentv-run-1' }, events: [ @@ -319,15 +305,15 @@ describe('derived trajectory contract', () => { }; const fromMessages = computeTraceSummary(messages); - const fromTrajectory = computeTraceSummaryFromTrajectory(trajectory); + const fromArtifact = computeTraceSummaryFromTraceArtifact(artifact); - expect(fromTrajectory).toEqual(fromMessages); + expect(fromArtifact).toEqual(fromMessages); }); - it('keeps TraceSummary as a derived read model outside trajectory wire state', () => { - const trajectory = buildTrajectory(); - const wire = toNormalizedTrajectoryWire(trajectory); - const summary = computeTraceSummaryFromTrajectory(fromNormalizedTrajectoryWire(wire)); + it('keeps TraceSummary as a derived read model outside trace artifact state', () => { + const artifact = buildTraceArtifact(); + const wire = toTraceArtifactWire(artifact); + const summary = computeTraceSummaryFromTraceArtifact(fromTraceArtifactWire(wire)); expect(wire).not.toHaveProperty('trace'); expect(wire).not.toHaveProperty('summary'); @@ -348,8 +334,7 @@ describe('derived trajectory contract', () => { }); it('counts LLM turns once when message and model_turn events coexist', () => { - const trajectory: NormalizedTrajectory = { - schemaVersion: NORMALIZED_TRAJECTORY_SCHEMA_VERSION, + const artifact: TraceArtifact = { source: { kind: 'agentv_run' }, session: { sessionId: 'agentv-run-2' }, events: [ @@ -375,15 +360,63 @@ describe('derived trajectory contract', () => { ], }; - const summary = computeTraceSummaryFromTrajectory(trajectory); + const summary = computeTraceSummaryFromTraceArtifact(artifact); expect(summary.trace.llmCallCount).toBe(1); expect(summary.trace.errorCount).toBe(0); }); + it('projects child final responses onto contentless model turns', () => { + const artifact: TraceArtifact = { + source: { kind: 'imported_transcript', provider: 'codex' }, + session: { sessionId: 'session-with-final' }, + events: [ + { + eventId: 'evt-user', + ordinal: 0, + type: 'message', + message: { role: 'user', content: 'Inspect app.txt' }, + }, + { + eventId: 'evt-model', + ordinal: 1, + type: 'model_turn', + timestamp: '2026-06-18T08:00:00Z', + }, + { + eventId: 'evt-read', + parentEventId: 'evt-model', + ordinal: 2, + type: 'tool_call', + tool: { name: 'Read', callId: 'call-read', input: { path: 'app.txt' } }, + }, + { + eventId: 'evt-final', + parentEventId: 'evt-model', + ordinal: 3, + type: 'final_response', + timestamp: '2026-06-18T08:00:01Z', + message: { role: 'assistant', content: 'done' }, + }, + ], + }; + + const trace = traceFromTraceArtifact(artifact); + const assistant = trace.messages.find((message) => message.role === 'assistant'); + + expect(assistant?.content).toBe('done'); + expect(assistant?.toolCalls?.map((toolCall) => toolCall.tool)).toEqual(['Read']); + expect(trace.events.map((event) => event.eventId)).toEqual([ + 'evt-user', + 'evt-model', + 'evt-read', + 'evt-final', + ]); + }); + it('exposes a Zod schema for direct wire validation', () => { - const wire = toNormalizedTrajectoryWire(buildTrajectory()); + const wire = toTraceArtifactWire(buildTraceArtifact()); - expect(NormalizedTrajectoryWireSchema.parse(wire).events).toHaveLength(4); + expect(TraceArtifactWireSchema.parse(wire).events).toHaveLength(4); }); }); diff --git a/packages/eval/src/index.ts b/packages/eval/src/index.ts index e1ff4eef9..947d895c1 100644 --- a/packages/eval/src/index.ts +++ b/packages/eval/src/index.ts @@ -40,13 +40,7 @@ export { CodeGraderInputSchema, CodeGraderResultSchema, - NORMALIZED_REDACTION_LEVELS, - NORMALIZED_TOOL_STATUSES, - NORMALIZED_TRACE_EVENT_TYPES, - NORMALIZED_TRACE_SOURCE_KINDS, - NORMALIZED_TRAJECTORY_SCHEMA_VERSION, TRACE_REDACTION_LEVELS, - TRACE_SCHEMA_VERSION, TRACE_SOURCE_KINDS, TRACE_EVENT_TYPES, TRACE_TOOL_STATUSES, @@ -64,18 +58,6 @@ export { TraceSourceRefSchema, TraceSourceSchema, TraceToolSchema, - NormalizedRawEvidenceSchema, - NormalizedRedactionStateSchema, - NormalizedTraceBranchSchema, - NormalizedTraceErrorSchema, - NormalizedTraceEventSchema, - NormalizedTraceMessageSchema, - NormalizedTraceModelSchema, - NormalizedTraceSessionSchema, - NormalizedTraceSourceRefSchema, - NormalizedTraceSourceSchema, - NormalizedTraceToolSchema, - NormalizedTrajectorySchema, MessageSchema, ToolCallSchema, TokenUsageSchema, @@ -98,18 +80,6 @@ export { type TraceSource, type TraceSourceRef, type TraceTool, - type NormalizedRawEvidence, - type NormalizedRedactionState, - type NormalizedTraceBranch, - type NormalizedTraceError, - type NormalizedTraceEvent, - type NormalizedTraceMessage, - type NormalizedTraceModel, - type NormalizedTraceSession, - type NormalizedTraceSource, - type NormalizedTraceSourceRef, - type NormalizedTraceTool, - type NormalizedTrajectory, type TraceSummary, type Trace, type Message, diff --git a/packages/eval/src/schemas.ts b/packages/eval/src/schemas.ts index a3a924a56..b1f45c5d8 100644 --- a/packages/eval/src/schemas.ts +++ b/packages/eval/src/schemas.ts @@ -46,9 +46,7 @@ export const TraceSummarySchema = z.object({ llmCallCount: z.number().optional(), }); -export const NORMALIZED_TRAJECTORY_SCHEMA_VERSION = 'agentv.trajectory.v1' as const; - -export const NORMALIZED_TRACE_SOURCE_KINDS = [ +export const TRACE_SOURCE_KINDS = [ 'agentv_run', 'otlp', 'phoenix', @@ -58,7 +56,7 @@ export const NORMALIZED_TRACE_SOURCE_KINDS = [ 'compact_transcript', ] as const; -export const NORMALIZED_TRACE_EVENT_TYPES = [ +export const TRACE_EVENT_TYPES = [ 'message', 'model_turn', 'tool_call', @@ -67,25 +65,19 @@ export const NORMALIZED_TRACE_EVENT_TYPES = [ 'error', ] as const; -export const NORMALIZED_TOOL_STATUSES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const; - -export const NORMALIZED_REDACTION_LEVELS = ['none', 'partial', 'full'] as const; +export const TRACE_TOOL_STATUSES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const; -export const TRACE_SCHEMA_VERSION = NORMALIZED_TRAJECTORY_SCHEMA_VERSION; -export const TRACE_SOURCE_KINDS = NORMALIZED_TRACE_SOURCE_KINDS; -export const TRACE_EVENT_TYPES = NORMALIZED_TRACE_EVENT_TYPES; -export const TRACE_TOOL_STATUSES = NORMALIZED_TOOL_STATUSES; -export const TRACE_REDACTION_LEVELS = NORMALIZED_REDACTION_LEVELS; +export const TRACE_REDACTION_LEVELS = ['none', 'partial', 'full'] as const; const MetadataSchema = z.record(z.string(), z.unknown()); -export const NormalizedRedactionStateSchema = z.object({ - level: z.enum(NORMALIZED_REDACTION_LEVELS), +export const TraceRedactionStateSchema = z.object({ + level: z.enum(TRACE_REDACTION_LEVELS), fields: z.array(z.string()).optional(), reason: z.string().optional(), }); -export const NormalizedTraceErrorSchema = z.object({ +export const TraceErrorSchema = z.object({ message: z.string(), name: z.string().optional(), code: z.string().optional(), @@ -93,8 +85,8 @@ export const NormalizedTraceErrorSchema = z.object({ metadata: MetadataSchema.optional(), }); -export const NormalizedTraceSourceSchema = z.object({ - kind: z.enum(NORMALIZED_TRACE_SOURCE_KINDS), +export const TraceSourceSchema = z.object({ + kind: z.enum(TRACE_SOURCE_KINDS), path: z.string().optional(), url: z.string().optional(), provider: z.string().optional(), @@ -103,7 +95,7 @@ export const NormalizedTraceSourceSchema = z.object({ metadata: MetadataSchema.optional(), }); -export const NormalizedTraceSessionSchema = z.object({ +export const TraceSessionSchema = z.object({ sessionId: z.string().optional(), conversationId: z.string().optional(), cwd: z.string().optional(), @@ -112,7 +104,7 @@ export const NormalizedTraceSessionSchema = z.object({ metadata: MetadataSchema.optional(), }); -export const NormalizedTraceBranchSchema = z.object({ +export const TraceBranchSchema = z.object({ selectedLeafId: z.string().optional(), selectedPathIds: z.array(z.string()).optional(), includedEventIds: z.array(z.string()).optional(), @@ -120,7 +112,7 @@ export const NormalizedTraceBranchSchema = z.object({ selectionReason: z.string().optional(), }); -export const NormalizedTraceSourceRefSchema = z.object({ +export const TraceSourceRefSchema = z.object({ eventId: z.string().optional(), messageId: z.string().optional(), spanId: z.string().optional(), @@ -131,7 +123,7 @@ export const NormalizedTraceSourceRefSchema = z.object({ metadata: MetadataSchema.optional(), }); -export const NormalizedRawEvidenceSchema = z.object({ +export const TraceRawEvidenceSchema = z.object({ kind: z.string(), ref: z.string().optional(), mediaType: z.string().optional(), @@ -140,16 +132,16 @@ export const NormalizedRawEvidenceSchema = z.object({ metadata: MetadataSchema.optional(), }); -export const NormalizedTraceMessageSchema = z.object({ +export const TraceMessageSchema = z.object({ role: z.string(), name: z.string().optional(), content: z.unknown().optional(), - redaction: NormalizedRedactionStateSchema.optional(), + redaction: TraceRedactionStateSchema.optional(), tokenUsage: TokenUsageSchema.optional(), metadata: MetadataSchema.optional(), }); -export const NormalizedTraceModelSchema = z.object({ +export const TraceModelSchema = z.object({ provider: z.string().optional(), name: z.string().optional(), invocationId: z.string().optional(), @@ -157,49 +149,47 @@ export const NormalizedTraceModelSchema = z.object({ metadata: MetadataSchema.optional(), }); -export const NormalizedTraceToolSchema = z.object({ +export const TraceToolSchema = z.object({ name: z.string(), callId: z.string().optional(), input: z.unknown().optional(), output: z.unknown().optional(), - status: z.enum(NORMALIZED_TOOL_STATUSES).optional(), - error: NormalizedTraceErrorSchema.optional(), - redaction: NormalizedRedactionStateSchema.optional(), + status: z.enum(TRACE_TOOL_STATUSES).optional(), + error: TraceErrorSchema.optional(), + redaction: TraceRedactionStateSchema.optional(), metadata: MetadataSchema.optional(), }); -export const NormalizedTraceEventSchema = z.object({ +export const TraceEventSchema = z.object({ eventId: z.string(), parentEventId: z.string().optional(), ordinal: z.number().int().nonnegative(), - type: z.enum(NORMALIZED_TRACE_EVENT_TYPES), + type: z.enum(TRACE_EVENT_TYPES), timestamp: z.string().optional(), durationMs: z.number().nonnegative().optional(), durationInferred: z.boolean().optional(), turnIndex: z.number().int().nonnegative().optional(), - message: NormalizedTraceMessageSchema.optional(), - model: NormalizedTraceModelSchema.optional(), - tool: NormalizedTraceToolSchema.optional(), - error: NormalizedTraceErrorSchema.optional(), - sourceRef: NormalizedTraceSourceRefSchema.optional(), - rawEvidence: z.array(NormalizedRawEvidenceSchema).optional(), - redaction: NormalizedRedactionStateSchema.optional(), + message: TraceMessageSchema.optional(), + model: TraceModelSchema.optional(), + tool: TraceToolSchema.optional(), + error: TraceErrorSchema.optional(), + sourceRef: TraceSourceRefSchema.optional(), + rawEvidence: z.array(TraceRawEvidenceSchema).optional(), + redaction: TraceRedactionStateSchema.optional(), metadata: MetadataSchema.optional(), }); /** - * Derived trajectory schema exposed to custom graders. + * Derived trace artifact shape used by import/replay helpers. * - * AgentV-owned persisted trajectory artifacts use the snake_case wire schemas - * and converters in @agentv/core. This SDK schema mirrors the internal - * camelCase model that grader authors receive. + * This is not a persisted public trace contract. Grader authors normally + * receive the result-local `Trace` shape below. */ -export const NormalizedTrajectorySchema = z.object({ - schemaVersion: z.literal(NORMALIZED_TRAJECTORY_SCHEMA_VERSION), - source: NormalizedTraceSourceSchema, - session: NormalizedTraceSessionSchema, - branch: NormalizedTraceBranchSchema.optional(), - events: z.array(NormalizedTraceEventSchema), +export const TraceArtifactSchema = z.object({ + source: TraceSourceSchema, + session: TraceSessionSchema, + branch: TraceBranchSchema.optional(), + events: z.array(TraceEventSchema), tokenUsage: TokenUsageSchema.optional(), costUsd: z.number().optional(), durationMs: z.number().optional(), @@ -208,19 +198,6 @@ export const NormalizedTrajectorySchema = z.object({ metadata: MetadataSchema.optional(), }); -export const TraceRedactionStateSchema = NormalizedRedactionStateSchema; -export const TraceErrorSchema = NormalizedTraceErrorSchema; -export const TraceSourceSchema = NormalizedTraceSourceSchema; -export const TraceSessionSchema = NormalizedTraceSessionSchema; -export const TraceBranchSchema = NormalizedTraceBranchSchema; -export const TraceSourceRefSchema = NormalizedTraceSourceRefSchema; -export const TraceRawEvidenceSchema = NormalizedRawEvidenceSchema; -export const TraceMessageSchema = NormalizedTraceMessageSchema; -export const TraceModelSchema = NormalizedTraceModelSchema; -export const TraceToolSchema = NormalizedTraceToolSchema; -export const TraceEventSchema = NormalizedTraceEventSchema; -export const TraceArtifactSchema = NormalizedTrajectorySchema; - /** * Tool call schema. */ @@ -298,7 +275,6 @@ export const MessageSchema = z.object({ * `messages` and structured execution events under `events`. */ export const TraceSchema = TraceSummarySchema.extend({ - schemaVersion: z.literal(TRACE_SCHEMA_VERSION), messages: z.array(MessageSchema), events: z.array(TraceEventSchema), tokenUsage: TokenUsageSchema.optional(), @@ -376,30 +352,6 @@ export type TraceError = z.infer; export type TraceSourceRef = z.infer; export type TraceRawEvidence = z.infer; export type TraceRedactionState = z.infer; -/** @deprecated Use TraceArtifact for legacy import/replay artifacts or Trace for evaluation results. */ -export type NormalizedTrajectory = TraceArtifact; -/** @deprecated Use TraceSource. */ -export type NormalizedTraceSource = TraceSource; -/** @deprecated Use TraceSession. */ -export type NormalizedTraceSession = TraceSession; -/** @deprecated Use TraceBranch. */ -export type NormalizedTraceBranch = TraceBranch; -/** @deprecated Use TraceEvent. */ -export type NormalizedTraceEvent = TraceEvent; -/** @deprecated Use TraceMessage. */ -export type NormalizedTraceMessage = TraceMessage; -/** @deprecated Use TraceModel. */ -export type NormalizedTraceModel = TraceModel; -/** @deprecated Use TraceTool. */ -export type NormalizedTraceTool = TraceTool; -/** @deprecated Use TraceError. */ -export type NormalizedTraceError = TraceError; -/** @deprecated Use TraceSourceRef. */ -export type NormalizedTraceSourceRef = TraceSourceRef; -/** @deprecated Use TraceRawEvidence. */ -export type NormalizedRawEvidence = TraceRawEvidence; -/** @deprecated Use TraceRedactionState. */ -export type NormalizedRedactionState = TraceRedactionState; export type Message = z.infer; export type ToolCall = z.infer; export type TokenUsage = z.infer; diff --git a/packages/eval/test/define-code-grader.test.ts b/packages/eval/test/define-code-grader.test.ts index 0a348e5a2..6f55257a2 100644 --- a/packages/eval/test/define-code-grader.test.ts +++ b/packages/eval/test/define-code-grader.test.ts @@ -15,7 +15,6 @@ import { } from '../src/schemas.js'; const makeTrace = (overrides: Record = {}) => ({ - schemaVersion: 'agentv.trajectory.v1', eventCount: 3, toolCalls: { read: 2, write: 1 }, errorCount: 0, diff --git a/packages/eval/test/define-prompt-template.test.ts b/packages/eval/test/define-prompt-template.test.ts index 0d6fe7818..1ed795d52 100644 --- a/packages/eval/test/define-prompt-template.test.ts +++ b/packages/eval/test/define-prompt-template.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it } from 'bun:test'; import { PromptTemplateInputSchema } from '../src/schemas.js'; const makeTrace = (overrides: Record = {}) => ({ - schemaVersion: 'agentv.trajectory.v1', eventCount: 3, toolCalls: { read: 2, write: 1 }, errorCount: 0,