write file changes diff sidecar

Author: christso

apps/cli/src/commands/eval/artifact-writer.ts

@@ -105,6 +105,7 @@ export function buildIndexArtifactEntry(
     transcriptPath?: string;
     transcriptRawPath?: string;
     metricsPath?: string;
+    fileChangesPath?: string;
     rawProviderLogPath?: string;
     responsePath?: string;
     taskBundle?: MaterializedTaskBundlePaths;

apps/cli/test/commands/eval/artifact-writer.test.ts

@@ -4,6 +4,7 @@ import { mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 
 import {
+  CANONICAL_FILE_CHANGES_ARTIFACT_PATH,
   CANONICAL_METRICS_ARTIFACT_PATH,
   CANONICAL_TRANSCRIPT_ARTIFACT_PATH,
   type EvalTest,
@@ -290,6 +291,10 @@ describe('buildGradingArtifact', () => {
       '@@ -1 +1 @@',
       '-old',
       '+new',
+      '--- a/deleted.ts',
+      '+++ /dev/null',
+      '@@ -1 +0,0 @@',
+      '-gone',
     ].join('\n');
 
     const result = makeResult({ fileChanges: diff });
@@ -298,6 +303,9 @@ describe('buildGradingArtifact', () => {
     expect(grading.workspace_changes).toBeDefined();
     expect(grading.workspace_changes?.files_created).toBe(1);
     expect(grading.workspace_changes?.files_modified).toBe(1);
+    expect(grading.workspace_changes?.files_deleted).toBe(1);
+    expect(grading.workspace_changes?.deleted_file_paths).toEqual(['deleted.ts']);
+    expect(grading.workspace_changes).not.toHaveProperty('diff_summary');
   });
 
   it('includes conversation when conversationId present', () => {
@@ -766,6 +774,17 @@ describe('parseJsonlResults', () => {
     expect(() => parseJsonlResults(content)).toThrow(/Use "artifact_pointers"/);
   });
 
+  it('rejects camelCase file changes path rows for the new wire field', () => {
+    const content = `${JSON.stringify({
+      test_id: 'file-changes-row',
+      target: 'codex',
+      score: 1,
+      fileChangesPath: 'file-changes-row/run-1/outputs/file_changes.diff',
+    })}\n`;
+
+    expect(() => parseJsonlResults(content)).toThrow(/Use "file_changes_path"/);
+  });
+
   it('does not treat parsed raw provider log pointers as fresh source artifacts', () => {
     const content = `${JSON.stringify({
       test_id: 'raw-log-case',
@@ -1398,6 +1417,10 @@ describe('writeArtifactsFromResults', () => {
       '+++ b/src/new.ts',
       '@@ -0,0 +1 @@',
       '+created',
+      '--- a/src/gone.ts',
+      '+++ /dev/null',
+      '@@ -1 +0,0 @@',
+      '-deleted',
     ].join('\n');
     const results = [
       makeResult({
@@ -1433,6 +1456,21 @@ describe('writeArtifactsFromResults', () => {
     const rowDir = expectRowDir(indexLine, 'summary-case');
 
     expect(indexLine?.metrics_path).toBe(`${rowDir}/run-1/metrics.json`);
+    expect(indexLine?.file_changes_path).toBe(
+      `${rowDir}/run-1/${CANONICAL_FILE_CHANGES_ARTIFACT_PATH}`,
+    );
+    await expect(
+      readFile(
+        runArtifactPath(testDir, indexLine, 'run-1', 'outputs', 'file_changes.diff'),
+        'utf8',
+      ),
+    ).resolves.toBe(fileChanges);
+
+    const runResult = JSON.parse(
+      await readFile(runArtifactPath(testDir, indexLine, 'run-1', 'result.json'), 'utf8'),
+    );
+    expect(runResult.file_changes_path).toBe('./outputs/file_changes.diff');
+    expect(runResult.output_paths.file_changes).toBe('./outputs/file_changes.diff');
 
     const summary = MetricsArtifactWireSchema.parse(
       JSON.parse(
@@ -1451,6 +1489,7 @@ describe('writeArtifactsFromResults', () => {
       transcript_path: 'transcript.jsonl',
       grading_path: 'grading.json',
       timing_path: 'timing.json',
+      file_changes_path: CANONICAL_FILE_CHANGES_ARTIFACT_PATH,
     });
     expect(summary.source_artifacts).not.toHaveProperty('trace_path');
     await expect(
@@ -1504,6 +1543,7 @@ describe('writeArtifactsFromResults', () => {
       source: 'file_changes',
     });
     expect(summary.metrics.files_created).toEqual(['src/new.ts']);
+    expect(summary.metrics.files_deleted).toEqual(['src/gone.ts']);
     expect(summary.metrics.web_fetches).toEqual([
       {
         url: 'https://example.com/spec',

apps/cli/test/commands/grade/grade-prepared.test.ts

@@ -197,8 +197,14 @@ describe('agentv grade prepared attempts', () => {
     });
     expect(typeof row.metadata.prepared_attempt.baseline_commit).toBe('string');
 
+    expect(row.file_changes_path).toMatch(/\/run-1\/outputs\/file_changes\.diff$/);
+    await expect(readFile(path.join(runDir, row.file_changes_path), 'utf8')).resolves.toContain(
+      '+manual edit',
+    );
+
     const grading = JSON.parse(await readFile(path.join(runDir, row.grading_path), 'utf8'));
-    expect(grading.workspace_changes.diff_summary).toContain('+manual edit');
+    expect(grading.workspace_changes).not.toHaveProperty('diff_summary');
+    expect(grading.workspace_changes.files_modified).toBeGreaterThanOrEqual(1);
   }, 20_000);
 
   it('fails clearly when the prepared manifest is missing', async () => {

apps/web/src/content/docs/docs/evaluation/running-evals.mdx

@@ -141,6 +141,7 @@ my-results/
       transcript.jsonl
       transcript-raw.jsonl
       outputs/answer.md
+      outputs/file_changes.diff  # when workspace changes are captured
     test/
       EVAL.yaml
       targets.yaml
@@ -149,13 +150,13 @@ my-results/
 ```
 
 The `index.jsonl` row links to these generated paths with snake_case fields such
-as `result_dir`, `test_dir`, `eval_path`, `targets_path`, `files_path`, and
-`graders_path`. Treat those paths as relative to the run directory. When you need
-a portable artifact for audit, review, Dashboard inspection, or rerun workflows,
-share the generated run directory and its `index.jsonl` manifest. Source-side
-case directories are still useful for organizing bulky prompts, fixtures, or
-tests while authoring an eval, but they are optional input organization rather
-than a separate artifact schema.
+as `result_dir`, `test_dir`, `eval_path`, `targets_path`, `files_path`,
+`file_changes_path`, and `graders_path`. Treat those paths as relative to the
+run directory. When you need a portable artifact for audit, review, Dashboard
+inspection, or rerun workflows, share the generated run directory and its
+`index.jsonl` manifest. Source-side case directories are still useful for
+organizing bulky prompts, fixtures, or tests while authoring an eval, but they
+are optional input organization rather than a separate artifact schema.
 
 For the full root layout, per-attempt sidecars, pointer rules, and integration
 guidance, use the [Result Artifact Contract](/docs/reference/result-artifacts/).

apps/web/src/content/docs/docs/reference/result-artifacts.mdx

@@ -45,6 +45,7 @@ The default local layout is:
           transcript-raw.jsonl
           outputs/
             answer.md
+            file_changes.diff
         run-2/
           result.json
           grading.json
@@ -54,6 +55,7 @@ The default local layout is:
           transcript-raw.jsonl
           outputs/
             answer.md
+            file_changes.diff
 ```
 
 The `<experiment>` and `<run_id>` directories are storage allocation. They help
@@ -83,6 +85,7 @@ query.
 | `result.json` | Compact per-attempt manifest for one attempt directory. | Loading one attempt without scanning the whole run index. |
 | `grading.json` | Grader outputs, assertions, rubric evidence, execution-metric grader facts, and scoring provenance. | Explaining why a row passed or failed. |
 | `metrics.json` | Derived executor behavior summary, such as tool calls, files touched, shell commands, errors, turns, and output sizes. | Dashboard behavior views, metric-style graders, adapter projections, and lightweight analysis. |
+| `outputs/file_changes.diff` | Full unified diff of workspace file changes when file changes are captured. | Human review and external artifact inspection; LLM and code graders still receive the same full diff through `file_changes`. |
 | `timing.json` | Duration, token usage, cost usage, and source labels such as `provider_reported`, `token_estimated`, `aggregate`, or `unavailable`. | Cost/latency reporting and provider-accounting audits. |
 | `transcript.jsonl` | AgentV-normalized transcript/timeline rows. | Portable human review, replay, transcript-aware graders, and tool-trajectory analysis. |
 | `transcript-raw.jsonl` | Native provider or harness evidence when available. | Parser debugging, forensic review, and preserving source bytes without making provider schemas public AgentV fields. |
@@ -132,6 +135,7 @@ Example row:
   "transcript_raw_path": "refund-eligibility--4f9a7c2d1b6e/run-1/transcript-raw.jsonl",
   "output_path": "refund-eligibility--4f9a7c2d1b6e/run-1/outputs/answer.md",
   "answer_path": "refund-eligibility--4f9a7c2d1b6e/run-1/outputs/answer.md",
+  "file_changes_path": "refund-eligibility--4f9a7c2d1b6e/run-1/outputs/file_changes.diff",
   "test_dir": "refund-eligibility--4f9a7c2d1b6e/test"
 }
 ```

apps/web/src/content/docs/docs/tools/results.mdx

@@ -130,8 +130,10 @@ token/cost usage.
 Every case uses aggregate `summary.json`, then stores attempt details under
 `run-N/`. Each `run-N/` contains a compact per-attempt manifest `result.json`,
 `grading.json`, `metrics.json`, `timing.json`, `transcript.jsonl`,
-`transcript-raw.jsonl`, and `outputs/answer.md`. The `result.json` file carries
-`grading_path`, `metrics_path`, transcript, and output paths.
+`transcript-raw.jsonl`, `outputs/answer.md`, and `outputs/file_changes.diff`
+when workspace changes were captured. The `result.json` file carries
+`grading_path`, `metrics_path`, transcript, output, and `file_changes_path`
+paths.
 
 `transcript-raw.jsonl` preserves native provider or harness transcript bytes
 when they are available, while `transcript.jsonl` is the normalized
@@ -141,10 +143,10 @@ systems can be linked through safe `external_trace` metadata when available.
 `summary.json` remains the run-level aggregate summary. `index.jsonl` is the
 canonical row index for the run: one row per result, attempt, or case, carrying
 lightweight explicit paths such as `transcript_path`, `transcript_raw_path`,
-and `metrics_path` plus artifact pointers only when detached payload publishing
-needs them. Dashboard search indexes, SQLite indexes, and other read models are
-derived projections over these run artifacts, not replacements for
-`index.jsonl`.
+`file_changes_path`, and `metrics_path` plus artifact pointers only when
+detached payload publishing needs them. Dashboard search indexes, SQLite
+indexes, and other read models are derived projections over these run artifacts,
+not replacements for `index.jsonl`.
 Duration, token, and cost usage remains in `timing.json`, including source
 labels such as `provider_reported`, `token_estimated`, `aggregate`, or
 `unavailable`.
@@ -154,7 +156,7 @@ while adding AgentV/Vercel-style detail:
 
 | Field group | Purpose |
 |-------------|---------|
-| `tool_calls`, `total_tool_calls`, `total_steps`, `errors_encountered`, `output_chars`, `transcript_chars`, `files_created` | Agent Skills-compatible executor metrics |
+| `tool_calls`, `total_tool_calls`, `total_steps`, `errors_encountered`, `output_chars`, `transcript_chars`, `files_created`, `files_deleted` | Agent Skills-compatible executor metrics |
 | `tool_call_events`, `tool_call_counts`, `tool_category_counts`, `shell_commands`, `files_read`, `files_modified`, `web_fetches`, `errors`, `reasoning_blocks`, `thinking_blocks`, `total_turns` | AgentV/Vercel-style behavior summary when source data includes it |
 
 Vercel `@vercel/agent-eval` `results.o11y` maps into AgentV like this:

docs/adr/0011-result-output-artifact-contract.md

@@ -61,6 +61,8 @@ An AgentV result output is a run-centric bundle with this root contract:
       transcript.jsonl
       transcript-raw.jsonl
       outputs/
+        answer.md             # when target output exists
+        file_changes.diff     # when workspace file changes exist
 ```
 
 `summary.json` and `index.jsonl` are complementary:
@@ -82,8 +84,8 @@ aggregate summaries.
 ordinary per-case sidecars through explicit fields such as `result_dir`,
 `summary_path`, `grading_path`, `metrics_path`, `timing_path`,
 `transcript_path`, `transcript_raw_path`, `answer_path`, `output_path`,
-`test_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` when
-those artifacts exist.
+`file_changes_path`, `test_dir`, `eval_path`, `targets_path`, `files_path`, and
+`graders_path` when those artifacts exist.
 
 `artifact_pointers` remain an offload indirection for large detached payload
 bytes. They are not the discovery path for ordinary sidecars that live in the

packages/core/src/evaluation/metrics.ts

@@ -152,6 +152,7 @@ export const MetricsWireSchema = z
     files_read: z.array(FileReferenceWireSchema),
     files_modified: z.array(FileReferenceWireSchema),
     files_created: z.array(z.string()),
+    files_deleted: z.array(z.string()).default([]),
     web_fetches: z.array(WebFetchWireSchema),
     errors: z.array(ExecutionErrorWireSchema),
     errors_encountered: z.number().int().nonnegative(),
@@ -184,6 +185,7 @@ export const MetricsArtifactWireSchema = z
         transcript_path: z.string().optional(),
         grading_path: z.string().optional(),
         timing_path: z.string().optional(),
+        file_changes_path: z.string().optional(),
       })
       .strict(),
     metrics: MetricsWireSchema,
@@ -515,11 +517,14 @@ function parseModifiedPathsFromDiff(fileChanges: string | undefined): string[] {
     return [];
   }
   const paths = new Set<string>();
-  for (const line of fileChanges.split('\n')) {
-    if (!line.startsWith('+++ b/')) {
+  const lines = fileChanges.split('\n');
+  for (let index = 0; index < lines.length - 1; index++) {
+    const oldLine = lines[index];
+    const newLine = lines[index + 1];
+    if (!oldLine.startsWith('--- a/') || !newLine?.startsWith('+++ b/')) {
       continue;
     }
-    const filePath = line.slice('+++ b/'.length).trim();
+    const filePath = newLine.slice('+++ b/'.length).trim();
     if (filePath && filePath !== '/dev/null') {
       paths.add(filePath);
     }
@@ -593,6 +598,26 @@ function buildFilesCreated(result: EvaluationResult, calls: readonly ToolCallRef
   return [...paths];
 }
 
+function parseDeletedPathsFromDiff(fileChanges: string | undefined): string[] {
+  if (!fileChanges) {
+    return [];
+  }
+  const paths = new Set<string>();
+  const lines = fileChanges.split('\n');
+  for (let index = 0; index < lines.length - 1; index++) {
+    const oldLine = lines[index];
+    const newLine = lines[index + 1];
+    if (!oldLine.startsWith('--- a/') || newLine !== '+++ /dev/null') {
+      continue;
+    }
+    const filePath = oldLine.slice('--- a/'.length).trim();
+    if (filePath) {
+      paths.add(filePath);
+    }
+  }
+  return [...paths];
+}
+
 function buildWebFetches(calls: readonly ToolCallRef[]) {
   return calls.flatMap((call) => {
     if (toolCategory(call.toolCall.tool) !== 'web_fetch') {
@@ -831,6 +856,7 @@ function buildMetrics(result: EvaluationResult) {
     files_read: buildFileReads(calls),
     files_modified: buildFileModifications(result, calls),
     files_created: buildFilesCreated(result, calls),
+    files_deleted: parseDeletedPathsFromDiff(result.fileChanges),
     web_fetches: buildWebFetches(calls),
     errors,
     errors_encountered: errors.length,
@@ -854,6 +880,7 @@ export function buildMetricsArtifact(
     transcriptPath?: string;
     gradingPath?: string;
     timingPath?: string;
+    fileChangesPath?: string;
     generatedAt?: string;
   } = {},
 ): MetricsArtifactWire {
@@ -876,6 +903,7 @@ export function buildMetricsArtifact(
         transcript_path: options.transcriptPath,
         grading_path: options.gradingPath,
         timing_path: options.timingPath,
+        file_changes_path: options.fileChangesPath,
       }),
       metrics: buildMetrics(result),
     }),

packages/core/src/evaluation/result-artifact-contract.ts

@@ -7,7 +7,8 @@
  * AgentV-owned artifacts belong when projected to a results ref, sidecar ref,
  * or object store. Use pointers for large detached payload bytes, not as the
  * discovery path for ordinary sidecars such as `metrics.json`; normal
- * sidecars should use explicit path fields such as `metrics_path`.
+ * sidecars should use explicit path fields such as `metrics_path` and
+ * `file_changes_path`.
  *
  * Git remote publishing treats the configured results branch as the
  * metadata/control plane and stores transcript payload bytes whose
@@ -27,6 +28,7 @@ export const AGENTV_RESULTS_REFS = {
 
 export const CANONICAL_TRANSCRIPT_ARTIFACT_PATH = 'transcript.jsonl' as const;
 export const CANONICAL_METRICS_ARTIFACT_PATH = 'metrics.json' as const;
+export const CANONICAL_FILE_CHANGES_ARTIFACT_PATH = 'outputs/file_changes.diff' as const;
 
 export const TRANSCRIPT_SCHEMA_VERSION = 'agentv.transcript.v1' as const;
 export const METRICS_SCHEMA_VERSION = 'agentv.metrics.v1' as const;

packages/core/src/evaluation/result-row-schema.ts

@@ -53,6 +53,7 @@ const RESULT_ROW_ALIASES = {
 
 const NEW_SNAKE_CASE_ONLY_FIELDS = {
   artifactPointers: 'artifact_pointers',
+  fileChangesPath: 'file_changes_path',
 } as const;
 
 const TRACE_SUMMARY_ALIASES = {