From c7edfbe16210029e0a054b8dbe7e327b4843ffc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 20:32:11 +0000 Subject: [PATCH 1/3] fix: seed demo data deterministically Replace Date.now() PRNG seeding in demoAnthropic/demoOpenAI with a fixed default seed (DEMO_SEED = 20260717) accepted as a parameter, so one demo run generates one coherent org, a real 6-month trend, and an identical hero savings number on every run. Add a determinism test asserting two full 6-month demo runs produce byte-identical reports. --- src/__tests__/demo-determinism.test.ts | 109 +++++++++++++++++++++++++ src/lib/demo.ts | 20 ++++- 2 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 src/__tests__/demo-determinism.test.ts diff --git a/src/__tests__/demo-determinism.test.ts b/src/__tests__/demo-determinism.test.ts new file mode 100644 index 0000000..6495931 --- /dev/null +++ b/src/__tests__/demo-determinism.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect } from "vitest"; +import { demoAnthropic, demoOpenAI, DEMO_SEED } from "@/lib/demo"; +import { agg, findIssues } from "@/lib/anthropic/analysis"; +import { aggOpenAI, findIssuesOpenAI } from "@/lib/openai/analysis"; +import { tc } from "@/lib/anthropic/pricing"; +import { tcOpenAI } from "@/lib/openai/pricing"; + +// Fixed base period so the test itself is clock-independent. Mirrors +// startDemo's loop: the base month plus the 5 months before it. +const BASE_YEAR = 2026; +const BASE_MONTH = 6; // July (0-indexed) + +function demoMonths(): { y: number; m: number }[] { + const out: { y: number; m: number }[] = []; + for (let i = 5; i >= 0; i--) { + let m = BASE_MONTH - i; + let y = BASE_YEAR; + while (m < 0) { + m += 12; + y--; + } + out.push({ y, m }); + } + return out; +} + +// Same report-building pipeline startDemo runs per month (rule engine path). +function runAnthropicDemo() { + return demoMonths().map(({ y, m }) => { + const d = demoAnthropic(y, m, DEMO_SEED); + const bk = agg(d.bk); + const bm = agg(d.bm); + const src = bk.length ? bk : bm; + const buckets = d.rawBk.length ? d.rawBk : d.rawBm; + + let spend = 0, + ti = 0, + to = 0; + for (const a of bm.length ? bm : src) { + spend += tc(a.model, a.inp, a.out); + ti += a.inp; + to += a.out; + } + + const findings = findIssues(src, d.ws, buckets); + return { + org: d.org, + spend, + savings: findings.reduce((s, f) => s + f.sav, 0), + tokens: ti + to, + findings, + }; + }); +} + +function runOpenAIDemo() { + return demoMonths().map(({ y, m }) => { + const d = demoOpenAI(y, m, DEMO_SEED); + const rows = aggOpenAI(d.usage); + + let spend = 0; + if (d.costs && d.costs.data.length > 0) { + for (const bucket of d.costs.data) { + for (const result of bucket.results) { + spend += result.amount.value; + } + } + } else { + for (const row of rows) { + spend += tcOpenAI(row.model, row.inp, row.out); + } + } + + const findings = findIssuesOpenAI(rows, d.projects); + return { + org: d.org, + spend, + savings: findings.reduce((s, f) => s + f.sav, 0), + findings, + }; + }); +} + +describe("demo mode determinism", () => { + it("two runs of the full 6-month Anthropic demo produce byte-identical reports", () => { + expect(JSON.stringify(runAnthropicDemo())).toBe( + JSON.stringify(runAnthropicDemo()) + ); + }); + + it("two runs of the full 6-month OpenAI demo produce byte-identical reports", () => { + expect(JSON.stringify(runOpenAIDemo())).toBe( + JSON.stringify(runOpenAIDemo()) + ); + }); + + it("all 6 months of a run come from the same org", () => { + const anth = runAnthropicDemo(); + expect(new Set(anth.map((r) => r.org.name)).size).toBe(1); + const oai = runOpenAIDemo(); + expect(new Set(oai.map((r) => r.org.name)).size).toBe(1); + }); + + it("a different seed produces different data", () => { + const a = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED); + const b = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED + 1); + expect(JSON.stringify(a.bm)).not.toBe(JSON.stringify(b.bm)); + }); +}); diff --git a/src/lib/demo.ts b/src/lib/demo.ts index ebd981f..914452c 100644 --- a/src/lib/demo.ts +++ b/src/lib/demo.ts @@ -5,6 +5,12 @@ import type { OpenAICostsData, } from "@/lib/openai/api"; +// Fixed default seed so every demo run generates the same org, the same +// 6-month trend, and the same hero savings number. Thread one seed through +// all monthly calls of a run — re-seeding per month would produce a different +// fake org each month. +export const DEMO_SEED = 20260717; + // ─── PRNG ──────────────────────────────────────────────────────────────────── function makeRand(seed: number): () => number { @@ -175,8 +181,11 @@ function genAnthropicEntries( return entries; } -export function demoAnthropic(year: number, month: number): PullResult { - const seed = Date.now() & 0x7fffffff; +export function demoAnthropic( + year: number, + month: number, + seed: number = DEMO_SEED +): PullResult { const profile = newProfile(seed); const org: Organization = { id: "demo_org_01", name: profile.orgName }; @@ -319,8 +328,11 @@ const OAI_SERVICES: ServiceConfig[] = [ }, ]; -export function demoOpenAI(year: number, month: number): OpenAIPullResult { - const seed = Date.now() & 0x7fffffff; +export function demoOpenAI( + year: number, + month: number, + seed: number = DEMO_SEED +): OpenAIPullResult { const profile = newProfile(seed); const rand = makeRand(seed ^ (year * 12 + month) ^ hashStr("oai")); const { scale } = profile; From f7d7d5a09fdd342b3782ae66bc6ff8fb2ade542f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 20:32:24 +0000 Subject: [PATCH 2/3] feat: fall back to rule engine when llm analysis fails findIssuesLLM now returns findings plus the NIM call's own token usage (with an estimated cost for the ROI footer). New analyzeWithFallback wrapper runs the LLM path and, on any failure, falls back to the deterministic rule engine and stamps a visible notice on the report. Reports gain optional engine ('rules' | 'llm'), notice, and llmUsage fields; the home page stamps them everywhere it builds a report and threads DEMO_SEED through demo calls. Add a test forcing the LLM call to throw, asserting the rules ran and the notice is set. --- src/__tests__/nim-fallback.test.ts | 81 ++++++++++++++++++++++++++++++ src/app/page.tsx | 76 +++++++++++++++++++--------- src/lib/nim/analysis.ts | 73 ++++++++++++++++++++++++--- src/types/analysis.ts | 16 ++++++ 4 files changed, 216 insertions(+), 30 deletions(-) create mode 100644 src/__tests__/nim-fallback.test.ts diff --git a/src/__tests__/nim-fallback.test.ts b/src/__tests__/nim-fallback.test.ts new file mode 100644 index 0000000..bfaf219 --- /dev/null +++ b/src/__tests__/nim-fallback.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect, vi } from "vitest"; +import { analyzeWithFallback, LLM_FALLBACK_NOTICE } from "@/lib/nim/analysis"; +import type { Finding } from "@/types"; +import { Severity, AnthropicCategory } from "@/types/analysis"; + +const ruleFinding: Finding = { + id: "key1-ws1-caching", + name: "key1", + ws: "production", + model: "claude-opus-4-6", + ml: "Opus 4.6", + inp: 10_000_000, + out: 100_000, + cached: 0, + reqs: 1000, + ao: 100, + ai: 10_000, + ratio: 100, + cr: 0, + cur: 200, + opt: 120, + sav: 80, + reason: "high input volume with no cache reads", + action: "add cache_control breakpoints", + sev: Severity.WARNING, + cat: AnthropicCategory.PROMPT_CACHING, + conf: 0.8, + impact: "$80.00/mo (40%)", + activeDays: 25, + temporal: { + burstiness: 0.2, + consistency: 0.8, + batchCandidate: false, + meanDaily: 300_000, + }, +}; + +describe("analyzeWithFallback", () => { + it("falls back to the rule engine and sets the notice when the LLM throws", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + let rulesRan = false; + + const out = await analyzeWithFallback( + () => Promise.reject(new Error("NIM analysis failed (503): down")), + () => { + rulesRan = true; + return [ruleFinding]; + } + ); + + expect(rulesRan).toBe(true); + expect(out.findings).toEqual([ruleFinding]); + expect(out.engine).toBe("rules"); + expect(out.notice).toBe(LLM_FALLBACK_NOTICE); + expect(out.llmUsage).toBeUndefined(); + vi.restoreAllMocks(); + }); + + it("returns the LLM findings and usage with engine 'llm' on success", async () => { + const usage = { + promptTokens: 4000, + completionTokens: 800, + costUsd: 0.00192, + }; + let rulesRan = false; + + const out = await analyzeWithFallback( + async () => ({ findings: [ruleFinding], usage }), + () => { + rulesRan = true; + return []; + } + ); + + expect(rulesRan).toBe(false); + expect(out.findings).toEqual([ruleFinding]); + expect(out.engine).toBe("llm"); + expect(out.notice).toBeUndefined(); + expect(out.llmUsage).toEqual(usage); + }); +}); diff --git a/src/app/page.tsx b/src/app/page.tsx index 3770ccf..41b5db2 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -10,11 +10,15 @@ import { pull as pullAnthropic } from "@/lib/anthropic/api"; import { pull as pullOpenAI } from "@/lib/openai/api"; import { agg, findIssues } from "@/lib/anthropic/analysis"; import { aggOpenAI, findIssuesOpenAI } from "@/lib/openai/analysis"; -import { findIssuesLLM } from "@/lib/nim/analysis"; +import { + analyzeWithFallback, + findIssuesLLM, + type AnalysisOutcome, +} from "@/lib/nim/analysis"; import { toSummariesAnthropic, toSummariesOpenAI } from "@/lib/nim/adapters"; import { tc } from "@/lib/anthropic/pricing"; import { tcOpenAI } from "@/lib/openai/pricing"; -import { demoAnthropic, demoOpenAI } from "@/lib/demo"; +import { demoAnthropic, demoOpenAI, DEMO_SEED } from "@/lib/demo"; import type { Report } from "@/types"; import Footer from "@/components/Footer"; import { FadeUp } from "@/components/motion/FadeUp"; @@ -254,8 +258,8 @@ function HomeContent() { }; // Choose NIM-guided LLM analysis when the setting is on, else the rule engine. - // NIM errors propagate to the caller so they surface in the UI rather than - // silently falling back. Used by both the real analysis and the demo. + // A NIM failure falls back to the deterministic rules and stamps a notice on + // the report instead of erroring out. Used by both the real analysis and the demo. const nimOn = () => useNim && nimAvailable; const analyzeAnthropic = async ( @@ -263,32 +267,40 @@ function HomeContent() { ws: Parameters[1], buckets: Parameters[2], spend: number - ) => { + ): Promise => { if (nimOn()) { setStep("Analyzing usage with NVIDIA NIM..."); - return findIssuesLLM(toSummariesAnthropic(src, ws, buckets), { - vendor: "anthropic", - totalSpend: spend, - workspaceCount: ws?.length || 0, - }); + return analyzeWithFallback( + () => + findIssuesLLM(toSummariesAnthropic(src, ws, buckets), { + vendor: "anthropic", + totalSpend: spend, + workspaceCount: ws?.length || 0, + }), + () => findIssues(src, ws, buckets) + ); } - return findIssues(src, ws, buckets); + return { findings: findIssues(src, ws, buckets), engine: "rules" }; }; const analyzeOpenAI = async ( rows: Parameters[0], projects: Parameters[1], spend: number - ) => { + ): Promise => { if (nimOn()) { setStep("Analyzing usage with NVIDIA NIM..."); - return findIssuesLLM(toSummariesOpenAI(rows, projects), { - vendor: "openai", - totalSpend: spend, - workspaceCount: projects?.length || 0, - }); + return analyzeWithFallback( + () => + findIssuesLLM(toSummariesOpenAI(rows, projects), { + vendor: "openai", + totalSpend: spend, + workspaceCount: projects?.length || 0, + }), + () => findIssuesOpenAI(rows, projects) + ); } - return findIssuesOpenAI(rows, projects); + return { findings: findIssuesOpenAI(rows, projects), engine: "rules" }; }; const startAnalysis = async () => { @@ -353,7 +365,8 @@ function HomeContent() { } } - const findings = await analyzeOpenAI(rows, d.projects, spend); + const analysis = await analyzeOpenAI(rows, d.projects, spend); + const findings = analysis.findings; let ti = 0; let to = 0; @@ -428,6 +441,9 @@ function HomeContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: analysis.engine, + notice: analysis.notice, + llmUsage: analysis.llmUsage, }; storage.saveAnalysis( @@ -470,7 +486,8 @@ function HomeContent() { to += a.out; } - const findings = await analyzeAnthropic(src, d.ws, buckets, spend); + const analysis = await analyzeAnthropic(src, d.ws, buckets, spend); + const findings = analysis.findings; const wb = agg(d.bw); @@ -518,6 +535,9 @@ function HomeContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: analysis.engine, + notice: analysis.notice, + llmUsage: analysis.llmUsage, }; // Save to localStorage @@ -569,7 +589,7 @@ function HomeContent() { y--; } - const d = demoOpenAI(y, m); + const d = demoOpenAI(y, m, DEMO_SEED); const rows = aggOpenAI(d.usage); let spend = 0; @@ -585,7 +605,8 @@ function HomeContent() { } } - const findings = await analyzeOpenAI(rows, d.projects, spend); + const analysis = await analyzeOpenAI(rows, d.projects, spend); + const findings = analysis.findings; let ti = 0, to = 0; @@ -635,6 +656,9 @@ function HomeContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: analysis.engine, + notice: analysis.notice, + llmUsage: analysis.llmUsage, }; storage.saveAnalysis( @@ -664,7 +688,7 @@ function HomeContent() { y--; } - const d = demoAnthropic(y, m); + const d = demoAnthropic(y, m, DEMO_SEED); const bk = agg(d.bk); const bm = agg(d.bm); const src = bk.length ? bk : bm; @@ -679,7 +703,8 @@ function HomeContent() { to += a.out; } - const findings = await analyzeAnthropic(src, d.ws, buckets, spend); + const analysis = await analyzeAnthropic(src, d.ws, buckets, spend); + const findings = analysis.findings; const wb = agg(d.bw); const wa: Record = {}; @@ -719,6 +744,9 @@ function HomeContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: analysis.engine, + notice: analysis.notice, + llmUsage: analysis.llmUsage, }; storage.saveAnalysis( diff --git a/src/lib/nim/analysis.ts b/src/lib/nim/analysis.ts index 216b2d1..f0f2a40 100644 --- a/src/lib/nim/analysis.ts +++ b/src/lib/nim/analysis.ts @@ -9,12 +9,23 @@ * here so dollar figures stay grounded and the model can't hallucinate numbers. */ -import type { Finding, TemporalPattern } from "@/types"; +import type { + AnalysisEngine, + Finding, + LlmUsage, + TemporalPattern, +} from "@/types"; import { Severity } from "@/types/analysis"; /** Default NIM-hosted model. OpenAI-compatible chat completions. */ export const NIM_DEFAULT_MODEL = "meta/llama-3.3-70b-instruct"; +/** + * Approximate hosted price for the default NIM model ($/MTok), used only to + * estimate what the analysis call itself cost for the ROI footer. + */ +export const NIM_PRICE_PER_MTOK = { input: 0.3, output: 0.9 }; + /* ─────────────── ANALYSIS RULES (LLM guidance) ─────────────── */ export const ANALYSIS_RULES = ` @@ -355,17 +366,51 @@ function parseFindings(content: string): LlmFinding[] { return Array.isArray(parsed) ? parsed : parsed.findings || []; } +export interface LlmAnalysisResult { + findings: Finding[]; + usage?: LlmUsage; +} + +/** Outcome of an analysis run: findings plus which engine produced them. */ +export interface AnalysisOutcome { + findings: Finding[]; + engine: AnalysisEngine; + notice?: string; + llmUsage?: LlmUsage; +} + +export const LLM_FALLBACK_NOTICE = + "LLM unavailable, showing deterministic analysis"; + +/** + * Run the LLM analysis; on any failure fall back to the deterministic rule + * engine and surface a notice so the report says which engine actually ran. + */ +export async function analyzeWithFallback( + llm: () => Promise, + rules: () => Finding[] +): Promise { + try { + const res = await llm(); + return { findings: res.findings, engine: "llm", llmUsage: res.usage }; + } catch (e) { + console.warn("LLM analysis failed, falling back to rule engine:", e); + return { findings: rules(), engine: "rules", notice: LLM_FALLBACK_NOTICE }; + } +} + /** * Run LLM-guided analysis via NVIDIA NIM. Returns Findings in the same shape as - * the hardcoded engines. Throws on transport/parse failure so callers can fall - * back to the rule engine. + * the hardcoded engines, plus the NIM call's own token usage when the response + * reports it. Throws on transport/parse failure so callers can fall back to + * the rule engine (see analyzeWithFallback). */ export async function findIssuesLLM( rows: UsageSummary[], ctx: AnalysisContext -): Promise { +): Promise { const payload = summarize(rows); - if (payload.length === 0) return []; + if (payload.length === 0) return { findings: [] }; const body = { model: ctx.model || NIM_DEFAULT_MODEL, @@ -414,5 +459,21 @@ export async function findIssuesLLM( const content: string = data?.choices?.[0]?.message?.content ?? ""; if (!content) throw new Error("NIM returned an empty response"); - return mergeLlmFindings(parseFindings(content), rows, ctx); + const u = data?.usage; + const usage: LlmUsage | undefined = + u && (u.prompt_tokens || u.completion_tokens) + ? { + promptTokens: u.prompt_tokens ?? 0, + completionTokens: u.completion_tokens ?? 0, + costUsd: + ((u.prompt_tokens ?? 0) * NIM_PRICE_PER_MTOK.input + + (u.completion_tokens ?? 0) * NIM_PRICE_PER_MTOK.output) / + 1_000_000, + } + : undefined; + + return { + findings: mergeLlmFindings(parseFindings(content), rows, ctx), + usage, + }; } diff --git a/src/types/analysis.ts b/src/types/analysis.ts index 362beda..f25fd37 100644 --- a/src/types/analysis.ts +++ b/src/types/analysis.ts @@ -114,6 +114,18 @@ export interface WorkspaceSpend { spend: number; } +// Which engine produced a report's findings. "rules" = deterministic rule +// engine; "llm" = NVIDIA NIM LLM-guided analysis. +export type AnalysisEngine = "rules" | "llm"; + +// Token usage of the NIM analysis call itself, for the ROI footer on +// LLM-engine reports. +export interface LlmUsage { + promptTokens: number; + completionTokens: number; + costUsd: number; +} + export interface Report { org: Organization; spend: number; @@ -127,4 +139,8 @@ export interface Report { warnCount: number; infoCount: number; highConfSavings: number; + // Optional so reports stored before these fields existed still parse. + engine?: AnalysisEngine; + notice?: string; // e.g. LLM-fallback notice, shown in the report header + llmUsage?: LlmUsage; // only set on engine === "llm" reports } From defb87a383d5be133f47202dd7410e0b46227b57 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 20:32:31 +0000 Subject: [PATCH 3/3] feat: show engine provenance, pricing date, and analysis roi in report Recommendations page: engine badge in the report header ('Rule engine' / 'LLM analysis'), a banner when the report carries a fallback notice, an always-visible 'Prices as of ' footer line (the 90-day staleness warning stays), and an ROI footer line on LLM-engine reports comparing the analysis token cost to the identified monthly savings. Month-fetch paths on the recommendations and analytics pages stamp engine: 'rules'. --- src/app/history/[id]/analytics/page.tsx | 2 + src/app/history/[id]/recommendations/page.tsx | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/app/history/[id]/analytics/page.tsx b/src/app/history/[id]/analytics/page.tsx index 7a2d010..c564f0e 100644 --- a/src/app/history/[id]/analytics/page.tsx +++ b/src/app/history/[id]/analytics/page.tsx @@ -239,6 +239,7 @@ export default function AnalyticsPage() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: "rules" as const, }; storage.saveAnalysis( @@ -462,6 +463,7 @@ export default function AnalyticsPage() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: "rules" as const, }; storage.saveAnalysis( diff --git a/src/app/history/[id]/recommendations/page.tsx b/src/app/history/[id]/recommendations/page.tsx index cc782fb..86a75f7 100644 --- a/src/app/history/[id]/recommendations/page.tsx +++ b/src/app/history/[id]/recommendations/page.tsx @@ -219,6 +219,7 @@ function RecommendationsPageContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: "rules" as const, }; storage.saveAnalysis( @@ -385,6 +386,7 @@ function RecommendationsPageContent() { .length, infoCount: findings.filter((f) => f.sev === Severity.INFO).length, highConfSavings: totalHighConfSavings, + engine: "rules" as const, }; storage.saveAnalysis( @@ -580,6 +582,12 @@ function RecommendationsPageContent() { {analysisRecord.orgName} + + {(r.engine ?? "rules") === "llm" ? "LLM analysis" : "Rule engine"} + {r && r.savings > 0 && ( Save {$(r.savings)}/mo @@ -721,6 +729,26 @@ function RecommendationsPageContent() { + {/* Engine notice (e.g. LLM unavailable → deterministic fallback) */} + {r.notice && ( +
+ + + +

{r.notice}

+
+ )} + {/* Stale-pricing warning */} {pricingStale && (
@@ -1162,6 +1190,22 @@ function RecommendationsPageContent() {
)} + {/* Report footer: pricing provenance + analysis ROI */} +
+

+ Prices as of {pricingDate} +

+ {r.engine === "llm" && r.llmUsage && ( +

+ This analysis cost ~$ + {r.llmUsage.costUsd < 0.01 + ? r.llmUsage.costUsd.toFixed(3) + : r.llmUsage.costUsd.toFixed(2)}{" "} + in LLM tokens and found {$(r.savings)}/mo in savings. +

+ )} +
+ {/* Month Picker Modal */} {showMonthPicker && (