diff --git a/src/__tests__/demo-determinism.test.ts b/src/__tests__/demo-determinism.test.ts new file mode 100644 index 0000000..6495931 --- /dev/null +++ b/src/__tests__/demo-determinism.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect } from "vitest"; +import { demoAnthropic, demoOpenAI, DEMO_SEED } from "@/lib/demo"; +import { agg, findIssues } from "@/lib/anthropic/analysis"; +import { aggOpenAI, findIssuesOpenAI } from "@/lib/openai/analysis"; +import { tc } from "@/lib/anthropic/pricing"; +import { tcOpenAI } from "@/lib/openai/pricing"; + +// Fixed base period so the test itself is clock-independent. Mirrors +// startDemo's loop: the base month plus the 5 months before it. +const BASE_YEAR = 2026; +const BASE_MONTH = 6; // July (0-indexed) + +function demoMonths(): { y: number; m: number }[] { + const out: { y: number; m: number }[] = []; + for (let i = 5; i >= 0; i--) { + let m = BASE_MONTH - i; + let y = BASE_YEAR; + while (m < 0) { + m += 12; + y--; + } + out.push({ y, m }); + } + return out; +} + +// Same report-building pipeline startDemo runs per month (rule engine path). +function runAnthropicDemo() { + return demoMonths().map(({ y, m }) => { + const d = demoAnthropic(y, m, DEMO_SEED); + const bk = agg(d.bk); + const bm = agg(d.bm); + const src = bk.length ? bk : bm; + const buckets = d.rawBk.length ? d.rawBk : d.rawBm; + + let spend = 0, + ti = 0, + to = 0; + for (const a of bm.length ? bm : src) { + spend += tc(a.model, a.inp, a.out); + ti += a.inp; + to += a.out; + } + + const findings = findIssues(src, d.ws, buckets); + return { + org: d.org, + spend, + savings: findings.reduce((s, f) => s + f.sav, 0), + tokens: ti + to, + findings, + }; + }); +} + +function runOpenAIDemo() { + return demoMonths().map(({ y, m }) => { + const d = demoOpenAI(y, m, DEMO_SEED); + const rows = aggOpenAI(d.usage); + + let spend = 0; + if (d.costs && d.costs.data.length > 0) { + for (const bucket of d.costs.data) { + for (const result of bucket.results) { + spend += result.amount.value; + } + } + } else { + for (const row of rows) { + spend += tcOpenAI(row.model, row.inp, row.out); + } + } + + const findings = findIssuesOpenAI(rows, d.projects); + return { + org: d.org, + spend, + savings: findings.reduce((s, f) => s + f.sav, 0), + findings, + }; + }); +} + +describe("demo mode determinism", () => { + it("two runs of the full 6-month Anthropic demo produce byte-identical reports", () => { + expect(JSON.stringify(runAnthropicDemo())).toBe( + JSON.stringify(runAnthropicDemo()) + ); + }); + + it("two runs of the full 6-month OpenAI demo produce byte-identical reports", () => { + expect(JSON.stringify(runOpenAIDemo())).toBe( + JSON.stringify(runOpenAIDemo()) + ); + }); + + it("all 6 months of a run come from the same org", () => { + const anth = runAnthropicDemo(); + expect(new Set(anth.map((r) => r.org.name)).size).toBe(1); + const oai = runOpenAIDemo(); + expect(new Set(oai.map((r) => r.org.name)).size).toBe(1); + }); + + it("a different seed produces different data", () => { + const a = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED); + const b = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED + 1); + expect(JSON.stringify(a.bm)).not.toBe(JSON.stringify(b.bm)); + }); +}); diff --git a/src/__tests__/nim-fallback.test.ts b/src/__tests__/nim-fallback.test.ts new file mode 100644 index 0000000..bfaf219 --- /dev/null +++ b/src/__tests__/nim-fallback.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect, vi } from "vitest"; +import { analyzeWithFallback, LLM_FALLBACK_NOTICE } from "@/lib/nim/analysis"; +import type { Finding } from "@/types"; +import { Severity, AnthropicCategory } from "@/types/analysis"; + +const ruleFinding: Finding = { + id: "key1-ws1-caching", + name: "key1", + ws: "production", + model: "claude-opus-4-6", + ml: "Opus 4.6", + inp: 10_000_000, + out: 100_000, + cached: 0, + reqs: 1000, + ao: 100, + ai: 10_000, + ratio: 100, + cr: 0, + cur: 200, + opt: 120, + sav: 80, + reason: "high input volume with no cache reads", + action: "add cache_control breakpoints", + sev: Severity.WARNING, + cat: AnthropicCategory.PROMPT_CACHING, + conf: 0.8, + impact: "$80.00/mo (40%)", + activeDays: 25, + temporal: { + burstiness: 0.2, + consistency: 0.8, + batchCandidate: false, + meanDaily: 300_000, + }, +}; + +describe("analyzeWithFallback", () => { + it("falls back to the rule engine and sets the notice when the LLM throws", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + let rulesRan = false; + + const out = await analyzeWithFallback( + () => Promise.reject(new Error("NIM analysis failed (503): down")), + () => { + rulesRan = true; + return [ruleFinding]; + } + ); + + expect(rulesRan).toBe(true); + expect(out.findings).toEqual([ruleFinding]); + expect(out.engine).toBe("rules"); + expect(out.notice).toBe(LLM_FALLBACK_NOTICE); + expect(out.llmUsage).toBeUndefined(); + vi.restoreAllMocks(); + }); + + it("returns the LLM findings and usage with engine 'llm' on success", async () => { + const usage = { + promptTokens: 4000, + completionTokens: 800, + costUsd: 0.00192, + }; + let rulesRan = false; + + const out = await analyzeWithFallback( + async () => ({ findings: [ruleFinding], usage }), + () => { + rulesRan = true; + return []; + } + ); + + expect(rulesRan).toBe(false); + expect(out.findings).toEqual([ruleFinding]); + expect(out.engine).toBe("llm"); + expect(out.notice).toBeUndefined(); + expect(out.llmUsage).toEqual(usage); + }); +}); diff --git a/src/app/history/[id]/analytics/page.tsx b/src/app/history/[id]/analytics/page.tsx index 7a2d010..c564f0e 100644 --- a/src/app/history/[id]/analytics/page.tsx +++ b/src/app/history/[id]/analytics/page.tsx @@ -239,6 +239,7 @@ export default function AnalyticsPage() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: "rules" as const, }; storage.saveAnalysis( @@ -462,6 +463,7 @@ export default function AnalyticsPage() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: "rules" as const, }; storage.saveAnalysis( diff --git a/src/app/history/[id]/recommendations/page.tsx b/src/app/history/[id]/recommendations/page.tsx index cc782fb..86a75f7 100644 --- a/src/app/history/[id]/recommendations/page.tsx +++ b/src/app/history/[id]/recommendations/page.tsx @@ -219,6 +219,7 @@ function RecommendationsPageContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: "rules" as const, }; storage.saveAnalysis( @@ -385,6 +386,7 @@ function RecommendationsPageContent() { .length, infoCount: findings.filter((f) => f.sev === Severity.INFO).length, highConfSavings: totalHighConfSavings, + engine: "rules" as const, }; storage.saveAnalysis( @@ -580,6 +582,12 @@ function RecommendationsPageContent() { {analysisRecord.orgName} + + {(r.engine ?? "rules") === "llm" ? "LLM analysis" : "Rule engine"} + {r && r.savings > 0 && ( Save {$(r.savings)}/mo @@ -721,6 +729,26 @@ function RecommendationsPageContent() { + {/* Engine notice (e.g. LLM unavailable → deterministic fallback) */} + {r.notice && ( +
+ + + +

{r.notice}

+
+ )} + {/* Stale-pricing warning */} {pricingStale && (
@@ -1162,6 +1190,22 @@ function RecommendationsPageContent() {
)} + {/* Report footer: pricing provenance + analysis ROI */} +
+

+ Prices as of {pricingDate} +

+ {r.engine === "llm" && r.llmUsage && ( +

+ This analysis cost ~$ + {r.llmUsage.costUsd < 0.01 + ? r.llmUsage.costUsd.toFixed(3) + : r.llmUsage.costUsd.toFixed(2)}{" "} + in LLM tokens and found {$(r.savings)}/mo in savings. +

+ )} +
+ {/* Month Picker Modal */} {showMonthPicker && ( useNim && nimAvailable; const analyzeAnthropic = async ( @@ -263,32 +267,40 @@ function HomeContent() { ws: Parameters[1], buckets: Parameters[2], spend: number - ) => { + ): Promise => { if (nimOn()) { setStep("Analyzing usage with NVIDIA NIM..."); - return findIssuesLLM(toSummariesAnthropic(src, ws, buckets), { - vendor: "anthropic", - totalSpend: spend, - workspaceCount: ws?.length || 0, - }); + return analyzeWithFallback( + () => + findIssuesLLM(toSummariesAnthropic(src, ws, buckets), { + vendor: "anthropic", + totalSpend: spend, + workspaceCount: ws?.length || 0, + }), + () => findIssues(src, ws, buckets) + ); } - return findIssues(src, ws, buckets); + return { findings: findIssues(src, ws, buckets), engine: "rules" }; }; const analyzeOpenAI = async ( rows: Parameters[0], projects: Parameters[1], spend: number - ) => { + ): Promise => { if (nimOn()) { setStep("Analyzing usage with NVIDIA NIM..."); - return findIssuesLLM(toSummariesOpenAI(rows, projects), { - vendor: "openai", - totalSpend: spend, - workspaceCount: projects?.length || 0, - }); + return analyzeWithFallback( + () => + findIssuesLLM(toSummariesOpenAI(rows, projects), { + vendor: "openai", + totalSpend: spend, + workspaceCount: projects?.length || 0, + }), + () => findIssuesOpenAI(rows, projects) + ); } - return findIssuesOpenAI(rows, projects); + return { findings: findIssuesOpenAI(rows, projects), engine: "rules" }; }; const startAnalysis = async () => { @@ -353,7 +365,8 @@ function HomeContent() { } } - const findings = await analyzeOpenAI(rows, d.projects, spend); + const analysis = await analyzeOpenAI(rows, d.projects, spend); + const findings = analysis.findings; let ti = 0; let to = 0; @@ -428,6 +441,9 @@ function HomeContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: analysis.engine, + notice: analysis.notice, + llmUsage: analysis.llmUsage, }; storage.saveAnalysis( @@ -470,7 +486,8 @@ function HomeContent() { to += a.out; } - const findings = await analyzeAnthropic(src, d.ws, buckets, spend); + const analysis = await analyzeAnthropic(src, d.ws, buckets, spend); + const findings = analysis.findings; const wb = agg(d.bw); @@ -518,6 +535,9 @@ function HomeContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: analysis.engine, + notice: analysis.notice, + llmUsage: analysis.llmUsage, }; // Save to localStorage @@ -569,7 +589,7 @@ function HomeContent() { y--; } - const d = demoOpenAI(y, m); + const d = demoOpenAI(y, m, DEMO_SEED); const rows = aggOpenAI(d.usage); let spend = 0; @@ -585,7 +605,8 @@ function HomeContent() { } } - const findings = await analyzeOpenAI(rows, d.projects, spend); + const analysis = await analyzeOpenAI(rows, d.projects, spend); + const findings = analysis.findings; let ti = 0, to = 0; @@ -635,6 +656,9 @@ function HomeContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: analysis.engine, + notice: analysis.notice, + llmUsage: analysis.llmUsage, }; storage.saveAnalysis( @@ -664,7 +688,7 @@ function HomeContent() { y--; } - const d = demoAnthropic(y, m); + const d = demoAnthropic(y, m, DEMO_SEED); const bk = agg(d.bk); const bm = agg(d.bm); const src = bk.length ? bk : bm; @@ -679,7 +703,8 @@ function HomeContent() { to += a.out; } - const findings = await analyzeAnthropic(src, d.ws, buckets, spend); + const analysis = await analyzeAnthropic(src, d.ws, buckets, spend); + const findings = analysis.findings; const wb = agg(d.bw); const wa: Record = {}; @@ -719,6 +744,9 @@ function HomeContent() { highConfSavings: findings .filter((f) => f.conf >= 0.65) .reduce((s, f) => s + f.sav, 0), + engine: analysis.engine, + notice: analysis.notice, + llmUsage: analysis.llmUsage, }; storage.saveAnalysis( diff --git a/src/lib/demo.ts b/src/lib/demo.ts index ebd981f..914452c 100644 --- a/src/lib/demo.ts +++ b/src/lib/demo.ts @@ -5,6 +5,12 @@ import type { OpenAICostsData, } from "@/lib/openai/api"; +// Fixed default seed so every demo run generates the same org, the same +// 6-month trend, and the same hero savings number. Thread one seed through +// all monthly calls of a run — re-seeding per month would produce a different +// fake org each month. +export const DEMO_SEED = 20260717; + // ─── PRNG ──────────────────────────────────────────────────────────────────── function makeRand(seed: number): () => number { @@ -175,8 +181,11 @@ function genAnthropicEntries( return entries; } -export function demoAnthropic(year: number, month: number): PullResult { - const seed = Date.now() & 0x7fffffff; +export function demoAnthropic( + year: number, + month: number, + seed: number = DEMO_SEED +): PullResult { const profile = newProfile(seed); const org: Organization = { id: "demo_org_01", name: profile.orgName }; @@ -319,8 +328,11 @@ const OAI_SERVICES: ServiceConfig[] = [ }, ]; -export function demoOpenAI(year: number, month: number): OpenAIPullResult { - const seed = Date.now() & 0x7fffffff; +export function demoOpenAI( + year: number, + month: number, + seed: number = DEMO_SEED +): OpenAIPullResult { const profile = newProfile(seed); const rand = makeRand(seed ^ (year * 12 + month) ^ hashStr("oai")); const { scale } = profile; diff --git a/src/lib/nim/analysis.ts b/src/lib/nim/analysis.ts index 216b2d1..f0f2a40 100644 --- a/src/lib/nim/analysis.ts +++ b/src/lib/nim/analysis.ts @@ -9,12 +9,23 @@ * here so dollar figures stay grounded and the model can't hallucinate numbers. */ -import type { Finding, TemporalPattern } from "@/types"; +import type { + AnalysisEngine, + Finding, + LlmUsage, + TemporalPattern, +} from "@/types"; import { Severity } from "@/types/analysis"; /** Default NIM-hosted model. OpenAI-compatible chat completions. */ export const NIM_DEFAULT_MODEL = "meta/llama-3.3-70b-instruct"; +/** + * Approximate hosted price for the default NIM model ($/MTok), used only to + * estimate what the analysis call itself cost for the ROI footer. + */ +export const NIM_PRICE_PER_MTOK = { input: 0.3, output: 0.9 }; + /* ─────────────── ANALYSIS RULES (LLM guidance) ─────────────── */ export const ANALYSIS_RULES = ` @@ -355,17 +366,51 @@ function parseFindings(content: string): LlmFinding[] { return Array.isArray(parsed) ? parsed : parsed.findings || []; } +export interface LlmAnalysisResult { + findings: Finding[]; + usage?: LlmUsage; +} + +/** Outcome of an analysis run: findings plus which engine produced them. */ +export interface AnalysisOutcome { + findings: Finding[]; + engine: AnalysisEngine; + notice?: string; + llmUsage?: LlmUsage; +} + +export const LLM_FALLBACK_NOTICE = + "LLM unavailable, showing deterministic analysis"; + +/** + * Run the LLM analysis; on any failure fall back to the deterministic rule + * engine and surface a notice so the report says which engine actually ran. + */ +export async function analyzeWithFallback( + llm: () => Promise, + rules: () => Finding[] +): Promise { + try { + const res = await llm(); + return { findings: res.findings, engine: "llm", llmUsage: res.usage }; + } catch (e) { + console.warn("LLM analysis failed, falling back to rule engine:", e); + return { findings: rules(), engine: "rules", notice: LLM_FALLBACK_NOTICE }; + } +} + /** * Run LLM-guided analysis via NVIDIA NIM. Returns Findings in the same shape as - * the hardcoded engines. Throws on transport/parse failure so callers can fall - * back to the rule engine. + * the hardcoded engines, plus the NIM call's own token usage when the response + * reports it. Throws on transport/parse failure so callers can fall back to + * the rule engine (see analyzeWithFallback). */ export async function findIssuesLLM( rows: UsageSummary[], ctx: AnalysisContext -): Promise { +): Promise { const payload = summarize(rows); - if (payload.length === 0) return []; + if (payload.length === 0) return { findings: [] }; const body = { model: ctx.model || NIM_DEFAULT_MODEL, @@ -414,5 +459,21 @@ export async function findIssuesLLM( const content: string = data?.choices?.[0]?.message?.content ?? ""; if (!content) throw new Error("NIM returned an empty response"); - return mergeLlmFindings(parseFindings(content), rows, ctx); + const u = data?.usage; + const usage: LlmUsage | undefined = + u && (u.prompt_tokens || u.completion_tokens) + ? { + promptTokens: u.prompt_tokens ?? 0, + completionTokens: u.completion_tokens ?? 0, + costUsd: + ((u.prompt_tokens ?? 0) * NIM_PRICE_PER_MTOK.input + + (u.completion_tokens ?? 0) * NIM_PRICE_PER_MTOK.output) / + 1_000_000, + } + : undefined; + + return { + findings: mergeLlmFindings(parseFindings(content), rows, ctx), + usage, + }; } diff --git a/src/types/analysis.ts b/src/types/analysis.ts index 362beda..f25fd37 100644 --- a/src/types/analysis.ts +++ b/src/types/analysis.ts @@ -114,6 +114,18 @@ export interface WorkspaceSpend { spend: number; } +// Which engine produced a report's findings. "rules" = deterministic rule +// engine; "llm" = NVIDIA NIM LLM-guided analysis. +export type AnalysisEngine = "rules" | "llm"; + +// Token usage of the NIM analysis call itself, for the ROI footer on +// LLM-engine reports. +export interface LlmUsage { + promptTokens: number; + completionTokens: number; + costUsd: number; +} + export interface Report { org: Organization; spend: number; @@ -127,4 +139,8 @@ export interface Report { warnCount: number; infoCount: number; highConfSavings: number; + // Optional so reports stored before these fields existed still parse. + engine?: AnalysisEngine; + notice?: string; // e.g. LLM-fallback notice, shown in the report header + llmUsage?: LlmUsage; // only set on engine === "llm" reports }