fix: deterministic demo, LLM fallback, engine provenance, pricing/ROI footer

src/__tests__/demo-determinism.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, expect } from "vitest";
+import { demoAnthropic, demoOpenAI, DEMO_SEED } from "@/lib/demo";
+import { agg, findIssues } from "@/lib/anthropic/analysis";
+import { aggOpenAI, findIssuesOpenAI } from "@/lib/openai/analysis";
+import { tc } from "@/lib/anthropic/pricing";
+import { tcOpenAI } from "@/lib/openai/pricing";
+
+// Fixed base period so the test itself is clock-independent. Mirrors
+// startDemo's loop: the base month plus the 5 months before it.
+const BASE_YEAR = 2026;
+const BASE_MONTH = 6; // July (0-indexed)
+
+function demoMonths(): { y: number; m: number }[] {
+  const out: { y: number; m: number }[] = [];
+  for (let i = 5; i >= 0; i--) {
+    let m = BASE_MONTH - i;
+    let y = BASE_YEAR;
+    while (m < 0) {
+      m += 12;
+      y--;
+    }
+    out.push({ y, m });
+  }
+  return out;
+}
+
+// Same report-building pipeline startDemo runs per month (rule engine path).
+function runAnthropicDemo() {
+  return demoMonths().map(({ y, m }) => {
+    const d = demoAnthropic(y, m, DEMO_SEED);
+    const bk = agg(d.bk);
+    const bm = agg(d.bm);
+    const src = bk.length ? bk : bm;
+    const buckets = d.rawBk.length ? d.rawBk : d.rawBm;
+
+    let spend = 0,
+      ti = 0,
+      to = 0;
+    for (const a of bm.length ? bm : src) {
+      spend += tc(a.model, a.inp, a.out);
+      ti += a.inp;
+      to += a.out;
+    }
+
+    const findings = findIssues(src, d.ws, buckets);
+    return {
+      org: d.org,
+      spend,
+      savings: findings.reduce((s, f) => s + f.sav, 0),
+      tokens: ti + to,
+      findings,
+    };
+  });
+}
+
+function runOpenAIDemo() {
+  return demoMonths().map(({ y, m }) => {
+    const d = demoOpenAI(y, m, DEMO_SEED);
+    const rows = aggOpenAI(d.usage);
+
+    let spend = 0;
+    if (d.costs && d.costs.data.length > 0) {
+      for (const bucket of d.costs.data) {
+        for (const result of bucket.results) {
+          spend += result.amount.value;
+        }
+      }
+    } else {
+      for (const row of rows) {
+        spend += tcOpenAI(row.model, row.inp, row.out);
+      }
+    }
+
+    const findings = findIssuesOpenAI(rows, d.projects);
+    return {
+      org: d.org,
+      spend,
+      savings: findings.reduce((s, f) => s + f.sav, 0),
+      findings,
+    };
+  });
+}
+
+describe("demo mode determinism", () => {
+  it("two runs of the full 6-month Anthropic demo produce byte-identical reports", () => {
+    expect(JSON.stringify(runAnthropicDemo())).toBe(
+      JSON.stringify(runAnthropicDemo())
+    );
+  });
+
+  it("two runs of the full 6-month OpenAI demo produce byte-identical reports", () => {
+    expect(JSON.stringify(runOpenAIDemo())).toBe(
+      JSON.stringify(runOpenAIDemo())
+    );
+  });
+
+  it("all 6 months of a run come from the same org", () => {
+    const anth = runAnthropicDemo();
+    expect(new Set(anth.map((r) => r.org.name)).size).toBe(1);
+    const oai = runOpenAIDemo();
+    expect(new Set(oai.map((r) => r.org.name)).size).toBe(1);
+  });
+
+  it("a different seed produces different data", () => {
+    const a = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED);
+    const b = demoAnthropic(BASE_YEAR, BASE_MONTH, DEMO_SEED + 1);
+    expect(JSON.stringify(a.bm)).not.toBe(JSON.stringify(b.bm));
+  });
+});

src/__tests__/nim-fallback.test.ts

@@ -0,0 +1,81 @@
+import { describe, it, expect, vi } from "vitest";
+import { analyzeWithFallback, LLM_FALLBACK_NOTICE } from "@/lib/nim/analysis";
+import type { Finding } from "@/types";
+import { Severity, AnthropicCategory } from "@/types/analysis";
+
+const ruleFinding: Finding = {
+  id: "key1-ws1-caching",
+  name: "key1",
+  ws: "production",
+  model: "claude-opus-4-6",
+  ml: "Opus 4.6",
+  inp: 10_000_000,
+  out: 100_000,
+  cached: 0,
+  reqs: 1000,
+  ao: 100,
+  ai: 10_000,
+  ratio: 100,
+  cr: 0,
+  cur: 200,
+  opt: 120,
+  sav: 80,
+  reason: "high input volume with no cache reads",
+  action: "add cache_control breakpoints",
+  sev: Severity.WARNING,
+  cat: AnthropicCategory.PROMPT_CACHING,
+  conf: 0.8,
+  impact: "$80.00/mo (40%)",
+  activeDays: 25,
+  temporal: {
+    burstiness: 0.2,
+    consistency: 0.8,
+    batchCandidate: false,
+    meanDaily: 300_000,
+  },
+};
+
+describe("analyzeWithFallback", () => {
+  it("falls back to the rule engine and sets the notice when the LLM throws", async () => {
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+    let rulesRan = false;
+
+    const out = await analyzeWithFallback(
+      () => Promise.reject(new Error("NIM analysis failed (503): down")),
+      () => {
+        rulesRan = true;
+        return [ruleFinding];
+      }
+    );
+
+    expect(rulesRan).toBe(true);
+    expect(out.findings).toEqual([ruleFinding]);
+    expect(out.engine).toBe("rules");
+    expect(out.notice).toBe(LLM_FALLBACK_NOTICE);
+    expect(out.llmUsage).toBeUndefined();
+    vi.restoreAllMocks();
+  });
+
+  it("returns the LLM findings and usage with engine 'llm' on success", async () => {
+    const usage = {
+      promptTokens: 4000,
+      completionTokens: 800,
+      costUsd: 0.00192,
+    };
+    let rulesRan = false;
+
+    const out = await analyzeWithFallback(
+      async () => ({ findings: [ruleFinding], usage }),
+      () => {
+        rulesRan = true;
+        return [];
+      }
+    );
+
+    expect(rulesRan).toBe(false);
+    expect(out.findings).toEqual([ruleFinding]);
+    expect(out.engine).toBe("llm");
+    expect(out.notice).toBeUndefined();
+    expect(out.llmUsage).toEqual(usage);
+  });
+});

src/app/history/[id]/analytics/page.tsx

@@ -239,6 +239,7 @@ export default function AnalyticsPage() {
               highConfSavings: findings
                 .filter((f) => f.conf >= 0.65)
                 .reduce((s, f) => s + f.sav, 0),
+              engine: "rules" as const,
             };
 
             storage.saveAnalysis(
@@ -462,6 +463,7 @@ export default function AnalyticsPage() {
               highConfSavings: findings
                 .filter((f) => f.conf >= 0.65)
                 .reduce((s, f) => s + f.sav, 0),
+              engine: "rules" as const,
             };
 
             storage.saveAnalysis(

src/app/history/[id]/recommendations/page.tsx

@@ -219,6 +219,7 @@ function RecommendationsPageContent() {
               highConfSavings: findings
                 .filter((f) => f.conf >= 0.65)
                 .reduce((s, f) => s + f.sav, 0),
+              engine: "rules" as const,
             };
 
             storage.saveAnalysis(
@@ -385,6 +386,7 @@ function RecommendationsPageContent() {
                 .length,
               infoCount: findings.filter((f) => f.sev === Severity.INFO).length,
               highConfSavings: totalHighConfSavings,
+              engine: "rules" as const,
             };
 
             storage.saveAnalysis(
@@ -580,6 +582,12 @@ function RecommendationsPageContent() {
           <span className="text-sm text-bone font-medium">
             {analysisRecord.orgName}
           </span>
+          <span
+            className="px-2 py-0.5 rounded-full border border-ink-border bg-ink-elevated text-[10px] font-mono text-bone-subtle"
+            title="Which analysis engine produced this report"
+          >
+            {(r.engine ?? "rules") === "llm" ? "LLM analysis" : "Rule engine"}
+          </span>
           {r && r.savings > 0 && (
             <span className="px-2 py-0.5 rounded-full bg-moss/10 border border-moss/20 text-xs font-mono font-medium text-moss-light">
               Save {$(r.savings)}/mo
@@ -721,6 +729,26 @@ function RecommendationsPageContent() {
         </Link>
       </div>
 
+      {/* Engine notice (e.g. LLM unavailable → deterministic fallback) */}
+      {r.notice && (
+        <div className="flex items-start gap-3 rounded-sm border border-warning/30 bg-warning/5 px-4 py-3">
+          <svg
+            className="w-4 h-4 text-warning shrink-0 mt-0.5"
+            fill="none"
+            viewBox="0 0 24 24"
+            stroke="currentColor"
+            strokeWidth={2}
+          >
+            <path
+              strokeLinecap="round"
+              strokeLinejoin="round"
+              d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"
+            />
+          </svg>
+          <p className="text-xs text-warning/90">{r.notice}</p>
+        </div>
+      )}
+
       {/* Stale-pricing warning */}
       {pricingStale && (
         <div className="flex items-start gap-3 rounded-sm border border-warning/30 bg-warning/5 px-4 py-3 no-print">
@@ -1162,6 +1190,22 @@ function RecommendationsPageContent() {
         </div>
       )}
 
+      {/* Report footer: pricing provenance + analysis ROI */}
+      <div className="border-t border-ink-border pt-4 space-y-1">
+        <p className="text-[11px] text-bone-subtle font-mono">
+          Prices as of {pricingDate}
+        </p>
+        {r.engine === "llm" && r.llmUsage && (
+          <p className="text-[11px] text-bone-subtle font-mono">
+            This analysis cost ~$
+            {r.llmUsage.costUsd < 0.01
+              ? r.llmUsage.costUsd.toFixed(3)
+              : r.llmUsage.costUsd.toFixed(2)}{" "}
+            in LLM tokens and found {$(r.savings)}/mo in savings.
+          </p>
+        )}
+      </div>
+
       {/* Month Picker Modal */}
       {showMonthPicker && (
         <MonthPicker