diff --git a/.github/workflows/bench-live.yml b/.github/workflows/bench-live.yml index 835c8c5..e1f03c6 100644 --- a/.github/workflows/bench-live.yml +++ b/.github/workflows/bench-live.yml @@ -88,6 +88,7 @@ jobs: ].join(" ")); } console.log(""); + console.log("Total run cost: $" + r.totalRunCostUsd.toFixed(6)); console.log("CLI " + r.cliVersion + " | endpoint " + r.apiBase + " | generated " + r.generatedAt); ' echo '```' diff --git a/bench/README.md b/bench/README.md index fd23d3a..c4f27ab 100644 --- a/bench/README.md +++ b/bench/README.md @@ -83,15 +83,16 @@ refuses to run without a key and without at least one model. ### Report `--json-out ` writes `{ generatedAt, cliVersion, apiBase, models[], -modelAggregates[], cases[] }`. The `models` array is the exact schema the site -consumes, one object per model: +modelAggregates[], totalRunCostUsd, cases[] }`. The `models` array is the exact +schema the site consumes, one object per model: ``` { id, detectionRate, falsePositives, casesRun, meanCostUsdPerReview, meanDurationMs } ``` `modelAggregates` is a superset with `totalCostUsd`, gate tallies, and error -counts for the human table; `cases` is the per-`(model, case)` detail. Every run +counts for the human table; `totalRunCostUsd` is the sum of included per-case +costs across the full run; `cases` is the per-`(model, case)` detail. Every run also writes a timestamped copy under `.runs/live-models/` (gitignored). `--json` prints the full report instead of the human table. diff --git a/bench/src/livemodels-score.test.ts b/bench/src/livemodels-score.test.ts index 2a473b7..a934375 100644 --- a/bench/src/livemodels-score.test.ts +++ b/bench/src/livemodels-score.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { benchmarkCase, type BenchmarkCase, type Envelope } from "./harness"; import { aggregateModel, + calculateTotalRunCostUsd, erroredLiveCase, estimateCasePromptTokens, findingHitsSeededRegion, @@ -289,6 +290,60 @@ describe("aggregateModel", () => { }); }); +// --------------------------------------------------------------------------- +// Run cost total + +describe("calculateTotalRunCostUsd", () => { + test("sums included per-case costs and matches per-model totals", () => { + const c = defectCase({ line: 20 }); + const modelA = [ + scoreLiveCase({ + case: c, + model: "a/model", + envelope: envelope({ usage: { promptTokens: 1000, completionTokens: 100 } }), + pricing, + exitCode: 0, + fidelityFailures: [], + }), + scoreLiveCase({ + case: cleanCase(), + model: "a/model", + envelope: envelope({ usage: { promptTokens: 500, completionTokens: 50 } }), + pricing, + exitCode: 0, + fidelityFailures: [], + }), + ]; + const modelB = [ + scoreLiveCase({ + case: c, + model: "b/model", + envelope: envelope({ usage: { promptTokens: 200, completionTokens: 20 } }), + pricing, + exitCode: 0, + fidelityFailures: [], + }), + scoreLiveCase({ + case: cleanCase(), + model: "b/model", + envelope: envelope({ usage: { promptTokens: 999, completionTokens: 999 } }), + pricing: null, + exitCode: 0, + fidelityFailures: [], + }), + erroredLiveCase({ case: c, model: "b/model", exitCode: 2, error: "no valid v1 envelope (exit 2)" }), + ]; + + const results = [...modelA, ...modelB]; + const perCaseTotal = results.reduce((sum, r) => sum + (r.costUsd ?? 0), 0); + const perModelTotal = + aggregateModel("a/model", modelA).totalCostUsd + aggregateModel("b/model", modelB).totalCostUsd; + + expect(calculateTotalRunCostUsd(results)).toBeCloseTo(perCaseTotal, 12); + expect(calculateTotalRunCostUsd(results)).toBeCloseTo(perModelTotal, 12); + }); +}); + // --------------------------------------------------------------------------- // Site schema shape diff --git a/bench/src/livemodels-score.ts b/bench/src/livemodels-score.ts index 8525105..6284c3d 100644 --- a/bench/src/livemodels-score.ts +++ b/bench/src/livemodels-score.ts @@ -278,6 +278,12 @@ export function aggregateModel(model: string, results: LiveModelCaseResult[]): L }; } +/** Total measured spend for a run, summing only cases with known pricing. + * Cases with null cost are excluded because their provider price was unknown. */ +export function calculateTotalRunCostUsd(results: LiveModelCaseResult[]): number { + return results.reduce((sum, r) => sum + (r.costUsd ?? 0), 0); +} + // --------------------------------------------------------------------------- // Pricing catalog parsing and cost projection diff --git a/bench/src/livemodels.ts b/bench/src/livemodels.ts index c77e37f..ffff4b2 100644 --- a/bench/src/livemodels.ts +++ b/bench/src/livemodels.ts @@ -33,6 +33,7 @@ import { } from "./harness"; import { aggregateModel, + calculateTotalRunCostUsd, erroredLiveCase, pricingFromCatalog, scoreLiveCase, @@ -83,6 +84,8 @@ export interface LiveModelsReport { /** Full per-model aggregates (superset of `models`) for the human table and * diagnostics. */ modelAggregates: LiveModelAggregate[]; + /** Total measured spend across all scored cases with known pricing. */ + totalRunCostUsd: number; /** Per-case detail across every (model, case) pair. */ cases: LiveModelCaseResult[]; } @@ -159,6 +162,7 @@ export async function runLiveModels( apiBase, models: aggregates.map(toSiteModelAggregate), modelAggregates: aggregates, + totalRunCostUsd: calculateTotalRunCostUsd(results), cases: results, }; } @@ -333,6 +337,8 @@ export function formatLiveModelsReport(report: LiveModelsReport): string { } } lines.push( + "", + `Total run cost: ${usd(report.totalRunCostUsd)}`, "", "detect = detection rate over defect fixtures; FP = findings that miss the seeded", "region (or any finding on a clean fixture). Fixtures are ours; no competitor peer",