fix(results): restore index jsonl run manifest

Author: christso

.github/workflows/ci.yml

@@ -213,6 +213,7 @@ jobs:
     name: Validate Evals
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    needs: build
     steps:
       - uses: actions/checkout@v6
       - name: Setup Bun
@@ -223,8 +224,84 @@ jobs:
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
-      - name: Build
-        run: bun run build
+      - name: Download build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: agentv-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }}
+          path: ${{ runner.temp }}/agentv-build-artifact
+
+      - name: Restore build artifact
+        env:
+          ARTIFACT_DIR: ${{ runner.temp }}/agentv-build-artifact
+          EXPECTED_COMMIT_SHA: ${{ github.sha }}
+          EXPECTED_RUNNER_ARCH: ${{ runner.arch }}
+          EXPECTED_RUNNER_OS: ${{ runner.os }}
+        run: |
+          set -euo pipefail
+
+          bun -e '
+            import { createHash } from "node:crypto";
+            import { Buffer } from "node:buffer";
+            import { cpSync, existsSync, mkdirSync, rmSync } from "node:fs";
+            import path from "node:path";
+
+            const artifactDir = process.env.ARTIFACT_DIR;
+            if (!artifactDir) {
+              throw new Error("ARTIFACT_DIR is required");
+            }
+
+            const manifestPath = path.join(artifactDir, "manifest.json");
+            if (!existsSync(manifestPath)) {
+              throw new Error(`Build artifact manifest is missing: ${manifestPath}`);
+            }
+
+            const manifest = await Bun.file(manifestPath).json();
+            const rootPackageJson = await Bun.file("package.json").json();
+            const lockfile = await Bun.file("bun.lock").arrayBuffer();
+            const lockHash = createHash("sha256")
+              .update(Buffer.from(lockfile))
+              .digest("hex");
+
+            const expected = {
+              commit_sha: process.env.EXPECTED_COMMIT_SHA,
+              bun_lock_sha256: lockHash,
+              runner_os: process.env.EXPECTED_RUNNER_OS,
+              runner_arch: process.env.EXPECTED_RUNNER_ARCH,
+              bun_version_spec: rootPackageJson.packageManager ?? null,
+              bun_version: Bun.version,
+            };
+
+            for (const [key, value] of Object.entries(expected)) {
+              if (manifest[key] !== value) {
+                throw new Error(
+                  `Build artifact manifest mismatch for ${key}: expected ${value}, got ${manifest[key]}`,
+                );
+              }
+            }
+
+            const requiredPaths = [
+              "packages/core/dist",
+              "packages/sdk/dist",
+              "apps/cli/dist",
+              "apps/dashboard/dist",
+            ];
+            const includedPaths = new Set(manifest.included_paths ?? []);
+
+            for (const relativePath of requiredPaths) {
+              if (!includedPaths.has(`${relativePath}/**`)) {
+                throw new Error(`Build artifact manifest does not include ${relativePath}/**`);
+              }
+
+              const source = path.join(artifactDir, relativePath);
+              if (!existsSync(source)) {
+                throw new Error(`Build artifact path is missing: ${source}`);
+              }
+
+              rmSync(relativePath, { recursive: true, force: true });
+              mkdirSync(path.dirname(relativePath), { recursive: true });
+              cpSync(source, relativePath, { recursive: true });
+            }
+          '
 
       - name: Check evals directories have eval files
         run: bun scripts/validate-eval-dirs.ts

CONCEPTS.md

@@ -24,11 +24,11 @@ Shared domain vocabulary for this project — entities, named processes, and sta
 
 **Workspace** — The task environment an eval prepares for the agent: repositories, templates, fixture files, and lifecycle hooks. It is not prompt input; use `input` for instructions and `workspace.repos[]` for multi-repo workspaces the agent can inspect or modify through tools.
 
-**Run manifest** — The root `run_manifest.jsonl` file in a run bundle. It is the dashboard and tooling loading contract for per-case result rows and artifact locations, including fields such as `result_dir`, `task_dir`, `summary_path`, and `grading_path`.
+**Run manifest** — The root `index.jsonl` file in a run bundle. It is the dashboard and tooling loading contract for per-case result rows and artifact locations, including fields such as `result_dir`, `task_dir`, `summary_path`, and `grading_path`.
 
 **Result source identity** — The stable source identity for a result row: repo-relative `eval_path`, `test_id`, and `target`. `suite` and `name` are display metadata, not storage or routing identity.
 
-**Result directory** — The `result_dir` field in a `run_manifest.jsonl` row. It is a run-local directory allocation for that row's sidecars and outputs. Consumers discover it from `run_manifest.jsonl` and must not infer it from suite names, display names, test IDs, or targets.
+**Result directory** — The `result_dir` field in a `index.jsonl` row. It is a run-local directory allocation for that row's sidecars and outputs. Consumers discover it from `index.jsonl` and must not infer it from suite names, display names, test IDs, or targets.
 
 **Artifact sidecar** — A file beside or below a result directory that provides evidence for a result, such as `summary.json`, `grading.json`, `result.json`, transcripts, logs, or outputs. Sidecars are evidence, not the primary discovery mechanism for a run.
 

README.md

@@ -73,14 +73,14 @@ agentv eval evals/my-eval.yaml
 
 **5. Compare results across targets:**
 ```bash
-agentv compare .agentv/results/default/<timestamp>/run_manifest.jsonl
+agentv compare .agentv/results/default/<timestamp>/index.jsonl
 ```
 
 ## Output formats
 
 ```bash
-agentv eval evals/my-eval.yaml --output ./run   # writes ./run/run_manifest.jsonl
-cat ./run/run_manifest.jsonl                         # JSONL results for scripts/CI
+agentv eval evals/my-eval.yaml --output ./run   # writes ./run/index.jsonl
+cat ./run/index.jsonl                         # JSONL results for scripts/CI
 ```
 
 ## TypeScript SDK

ROADMAP.md

@@ -15,7 +15,7 @@ This roadmap translates [STRATEGY.md](STRATEGY.md) into the next few product pha
 
 ## Phase 1: Finish the artifact and local inspection foundation
 
-- Keep the canonical handoff surface centered on completed run bundles, `run_manifest.jsonl`, grading/timing/metrics artifacts, normalized transcripts, and optional `external_trace` link metadata.
+- Keep the canonical handoff surface centered on completed run bundles, `index.jsonl`, grading/timing/metrics artifacts, normalized transcripts, and optional `external_trace` link metadata.
 - Finish the vendor-neutral local export seams that let completed runs be re-read, compared, exported, and attached to non-Phoenix adapters without vendor-specific logic in core.
 - Keep OTLP/OpenInference mapping generic and reusable before building backend-specific upload or import paths.
 

STRATEGY.md

@@ -21,7 +21,7 @@ AgentV stays repo-native and workspace-native: it runs or imports evaluations ar
 
 - **Repo-native eval success** - Share of dogfood and example eval flows that run against real workspaces, hooks, repo materialization, or imported artifacts without extra infrastructure; measured by CI and manual UAT on canonical suites.
 - **Time to inspect a run** - Time from completed `agentv eval` to usable local review, compare, or report output from the canonical run bundle; measured through CLI and Dashboard/report workflows.
-- **Artifact portability coverage** - Share of integrations and follow-on workflows that consume `run_manifest.jsonl`, `summary.json`, trace sidecars, or imported run bundles instead of bespoke stores; measured by adapter smoke tests, docs, and example coverage.
+- **Artifact portability coverage** - Share of integrations and follow-on workflows that consume `index.jsonl`, `summary.json`, trace sidecars, or imported run bundles instead of bespoke stores; measured by adapter smoke tests, docs, and example coverage.
 - **Git-backed results reliability** - Success rate for publish, sync, resume, and WIP checkpoint flows across local branches and dedicated results repos; measured by integration tests and manual end-to-end verification.
 
 ## Tracks

apps/cli/src/cli.ts

@@ -6,7 +6,7 @@ import { runCli } from './index.js';
 // Forward SIGINT/SIGTERM to spawned provider subprocesses before exiting.
 // Without this, Dashboard's `child.kill('SIGTERM')` against the CLI orphans
 // any in-flight `claude`/`codex`/`pi`/`copilot` subprocess. The partial
-// `run_manifest.jsonl` is already row-by-row durable, so finished tests survive.
+// `index.jsonl` is already row-by-row durable, so finished tests survive.
 //
 // First signal: kill children, exit with the conventional 128+signal code.
 // Second signal within the same process: hard-exit so a hung child cannot

apps/cli/src/commands/eval/commands/run.ts

@@ -52,12 +52,12 @@ export const evalRunCommand = command({
       long: 'output',
       short: 'o',
       description:
-        'Run artifact directory (writes run_manifest.jsonl, summary.json, and per-case artifacts)',
+        'Run artifact directory (writes index.jsonl, summary.json, and per-case artifacts)',
     }),
     outputFormat: option({
       type: optional(string),
       long: 'output-format',
-      description: '[Removed] Run directories always write run_manifest.jsonl',
+      description: '[Removed] Run directories always write index.jsonl',
     }),
     experiment: option({
       type: optional(string),

apps/cli/src/commands/eval/result-layout.ts

@@ -1,15 +1,7 @@
-import { type Dirent, existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { type Dirent, existsSync, readdirSync, statSync } from 'node:fs';
 import path from 'node:path';
 
-export const RESULT_MANIFEST_FILENAME = 'run_manifest.jsonl';
-export const LEGACY_RESULT_INDEX_FILENAME = 'index.jsonl';
-// Backward-compatible export name retained for existing callers. New writes use
-// the row-level run manifest filename.
-export const RESULT_INDEX_FILENAME = RESULT_MANIFEST_FILENAME;
-export const RESULT_MANIFEST_FILENAMES = [
-  RESULT_MANIFEST_FILENAME,
-  LEGACY_RESULT_INDEX_FILENAME,
-] as const;
+export const RESULT_INDEX_FILENAME = 'index.jsonl';
 export const RUN_SUMMARY_FILENAME = 'summary.json';
 export const RESULTS_DIRNAME = 'results';
 export const DEFAULT_EXPERIMENT_NAME = 'default';
@@ -73,48 +65,13 @@ export function resolveRunIndexPath(runDir: string): string {
 }
 
 export function isRunManifestPath(filePath: string): boolean {
-  return RESULT_MANIFEST_FILENAMES.includes(
-    path.basename(filePath) as (typeof RESULT_MANIFEST_FILENAMES)[number],
-  );
-}
-
-function safeSummaryManifestPath(runDir: string, manifestPath: unknown): string | undefined {
-  if (typeof manifestPath !== 'string' || manifestPath.trim().length === 0) {
-    return undefined;
-  }
-  if (path.isAbsolute(manifestPath)) {
-    return undefined;
-  }
-  const normalized = path.normalize(manifestPath);
-  if (normalized.startsWith('..') || path.isAbsolute(normalized)) {
-    return undefined;
-  }
-  return path.join(runDir, normalized);
-}
-
-function resolveSummaryManifestPath(runDir: string): string | undefined {
-  try {
-    const summary = JSON.parse(readFileSync(path.join(runDir, RUN_SUMMARY_FILENAME), 'utf8')) as {
-      manifest_path?: unknown;
-    };
-    const manifestPath = safeSummaryManifestPath(runDir, summary.manifest_path);
-    return manifestPath && existsSync(manifestPath) ? manifestPath : undefined;
-  } catch {
-    return undefined;
-  }
+  return path.basename(filePath) === RESULT_INDEX_FILENAME;
 }
 
 export function resolveExistingRunPrimaryPath(runDir: string): string | undefined {
-  const summaryManifestPath = resolveSummaryManifestPath(runDir);
-  if (summaryManifestPath) {
-    return summaryManifestPath;
-  }
-
-  for (const filename of RESULT_MANIFEST_FILENAMES) {
-    const manifestPath = path.join(runDir, filename);
-    if (existsSync(manifestPath)) {
-      return manifestPath;
-    }
+  const indexPath = resolveRunIndexPath(runDir);
+  if (existsSync(indexPath)) {
+    return indexPath;
   }
 
   return undefined;
@@ -178,9 +135,7 @@ export function resolveWorkspaceOrFilePath(filePath: string): string {
       `Result workspace contains multiple run manifests; pass one bundle directory or manifest: ${filePath}`,
     );
   }
-  throw new Error(
-    `Result workspace is missing ${RESULT_MANIFEST_FILENAME} or legacy ${LEGACY_RESULT_INDEX_FILENAME}: ${filePath}`,
-  );
+  throw new Error(`Result workspace is missing ${RESULT_INDEX_FILENAME}: ${filePath}`);
 }
 
 export function resolveRunManifestPath(filePath: string): string {
@@ -190,7 +145,7 @@ export function resolveRunManifestPath(filePath: string): string {
 
   if (!isRunManifestPath(filePath)) {
     throw new Error(
-      `Expected a run workspace directory or ${RESULT_MANIFEST_FILENAME} manifest (legacy ${LEGACY_RESULT_INDEX_FILENAME} is also readable): ${filePath}`,
+      `Expected a run workspace directory or ${RESULT_INDEX_FILENAME} manifest: ${filePath}`,
     );
   }
 

apps/cli/src/commands/eval/run-eval.ts

@@ -140,7 +140,7 @@ interface NormalizedOptions {
   readonly keepWorkspaces: boolean;
   /** Removed: use --output instead */
   readonly artifacts?: string;
-  /** Removed: the run directory always uses run_manifest.jsonl */
+  /** Removed: the run directory always uses index.jsonl */
   readonly outputFormat?: string;
   readonly graderTarget?: string;
   readonly model?: string;

apps/cli/src/commands/grade/index.ts

@@ -625,7 +625,7 @@ export const gradeCommand = command({
       type: optional(string),
       long: 'output',
       short: 'o',
-      description: 'Run artifact directory (writes run_manifest.jsonl and per-test artifacts)',
+      description: 'Run artifact directory (writes index.jsonl and per-test artifacts)',
     }),
     response: option({
       type: optional(string),