Report total live benchmark run cost

.github/workflows/bench-live.yml

@@ -88,6 +88,7 @@ jobs:
                   ].join(" "));
                 }
                 console.log("");
+                console.log("Total run cost: $" + r.totalRunCostUsd.toFixed(6));
                 console.log("CLI " + r.cliVersion + " | endpoint " + r.apiBase + " | generated " + r.generatedAt);
               '
               echo '```'

bench/README.md

@@ -83,15 +83,16 @@ refuses to run without a key and without at least one model.
 ### Report
 
 `--json-out <path>` writes `{ generatedAt, cliVersion, apiBase, models[],
-modelAggregates[], cases[] }`. The `models` array is the exact schema the site
-consumes, one object per model:
+modelAggregates[], totalRunCostUsd, cases[] }`. The `models` array is the exact
+schema the site consumes, one object per model:
 
 ```
 { id, detectionRate, falsePositives, casesRun, meanCostUsdPerReview, meanDurationMs }
 ```
 
 `modelAggregates` is a superset with `totalCostUsd`, gate tallies, and error
-counts for the human table; `cases` is the per-`(model, case)` detail. Every run
+counts for the human table; `totalRunCostUsd` is the sum of included per-case
+costs across the full run; `cases` is the per-`(model, case)` detail. Every run
 also writes a timestamped copy under `.runs/live-models/` (gitignored). `--json`
 prints the full report instead of the human table.
 

bench/src/livemodels-score.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
 import { benchmarkCase, type BenchmarkCase, type Envelope } from "./harness";
 import {
   aggregateModel,
+  calculateTotalRunCostUsd,
   erroredLiveCase,
   estimateCasePromptTokens,
   findingHitsSeededRegion,
@@ -289,6 +290,60 @@ describe("aggregateModel", () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// Run cost total
+
+describe("calculateTotalRunCostUsd", () => {
+  test("sums included per-case costs and matches per-model totals", () => {
+    const c = defectCase({ line: 20 });
+    const modelA = [
+      scoreLiveCase({
+        case: c,
+        model: "a/model",
+        envelope: envelope({ usage: { promptTokens: 1000, completionTokens: 100 } }),
+        pricing,
+        exitCode: 0,
+        fidelityFailures: [],
+      }),
+      scoreLiveCase({
+        case: cleanCase(),
+        model: "a/model",
+        envelope: envelope({ usage: { promptTokens: 500, completionTokens: 50 } }),
+        pricing,
+        exitCode: 0,
+        fidelityFailures: [],
+      }),
+    ];
+    const modelB = [
+      scoreLiveCase({
+        case: c,
+        model: "b/model",
+        envelope: envelope({ usage: { promptTokens: 200, completionTokens: 20 } }),
+        pricing,
+        exitCode: 0,
+        fidelityFailures: [],
+      }),
+      scoreLiveCase({
+        case: cleanCase(),
+        model: "b/model",
+        envelope: envelope({ usage: { promptTokens: 999, completionTokens: 999 } }),
+        pricing: null,
+        exitCode: 0,
+        fidelityFailures: [],
+      }),
+      erroredLiveCase({ case: c, model: "b/model", exitCode: 2, error: "no valid v1 envelope (exit 2)" }),
+    ];
+
+    const results = [...modelA, ...modelB];
+    const perCaseTotal = results.reduce((sum, r) => sum + (r.costUsd ?? 0), 0);
+    const perModelTotal =
+      aggregateModel("a/model", modelA).totalCostUsd + aggregateModel("b/model", modelB).totalCostUsd;
+
+    expect(calculateTotalRunCostUsd(results)).toBeCloseTo(perCaseTotal, 12);
+    expect(calculateTotalRunCostUsd(results)).toBeCloseTo(perModelTotal, 12);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // Site schema shape
 

bench/src/livemodels-score.ts

@@ -278,6 +278,12 @@ export function aggregateModel(model: string, results: LiveModelCaseResult[]): L
   };
 }
 
+/** Total measured spend for a run, summing only cases with known pricing.
+ * Cases with null cost are excluded because their provider price was unknown. */
+export function calculateTotalRunCostUsd(results: LiveModelCaseResult[]): number {
+  return results.reduce((sum, r) => sum + (r.costUsd ?? 0), 0);
+}
+
 // ---------------------------------------------------------------------------
 // Pricing catalog parsing and cost projection
 

bench/src/livemodels.ts

@@ -33,6 +33,7 @@ import {
 } from "./harness";
 import {
   aggregateModel,
+  calculateTotalRunCostUsd,
   erroredLiveCase,
   pricingFromCatalog,
   scoreLiveCase,
@@ -83,6 +84,8 @@ export interface LiveModelsReport {
   /** Full per-model aggregates (superset of `models`) for the human table and
    * diagnostics. */
   modelAggregates: LiveModelAggregate[];
+  /** Total measured spend across all scored cases with known pricing. */
+  totalRunCostUsd: number;
   /** Per-case detail across every (model, case) pair. */
   cases: LiveModelCaseResult[];
 }
@@ -159,6 +162,7 @@ export async function runLiveModels(
     apiBase,
     models: aggregates.map(toSiteModelAggregate),
     modelAggregates: aggregates,
+    totalRunCostUsd: calculateTotalRunCostUsd(results),
     cases: results,
   };
 }
@@ -333,6 +337,8 @@ export function formatLiveModelsReport(report: LiveModelsReport): string {
     }
   }
   lines.push(
+    "",
+    `Total run cost: ${usd(report.totalRunCostUsd)}`,
     "",
     "detect = detection rate over defect fixtures; FP = findings that miss the seeded",
     "region (or any finding on a clean fixture). Fixtures are ours; no competitor peer",