diff --git a/.env.local.example b/.env.local.example index feae4b5..517eb41 100644 --- a/.env.local.example +++ b/.env.local.example @@ -1,4 +1,23 @@ # Mock API Server Configuration # When set, the app will use the mock server instead of real vendor APIs # Run the mock server first: bun run mock-server/server.ts -MOCK_API_URL=http://localhost:3456 \ No newline at end of file +MOCK_API_URL=http://localhost:3456 + +# NIM_API_KEY — NVIDIA NIM key (nvapi-...). Set this in your deployment to enable +# the in-app "Use AI analysis (NVIDIA NIM)" toggle. Server-side only; never sent +# to the browser. When unset (and NIM_MOCK off), the toggle is hidden. +# NIM_API_KEY=nvapi-... + +# NVIDIA NIM base URL (optional) — override for a self-hosted NIM endpoint. +# Defaults to https://integrate.api.nvidia.com/v1 when unset. +# NIM_BASE_URL=https://integrate.api.nvidia.com/v1 + +# NIM_MODEL (optional) — override the NIM model id (default meta/llama-3.3-70b-instruct). +# Use a smaller/faster model if the 70B endpoint is slow, e.g.: +# NIM_MODEL=meta/llama-3.1-8b-instruct + +# NIM_MOCK (optional) — when "1", /api/nim returns synthesized findings instead +# of calling NVIDIA, so you can test the LLM-analysis flow offline without a real +# nvapi key. This is INDEPENDENT of MOCK_API_URL: with a real nvapi key and +# NIM_MOCK unset, NIM runs for real even while vendor usage data is mocked. +# NIM_MOCK=1 \ No newline at end of file diff --git a/src/__tests__/nim-analysis.test.ts b/src/__tests__/nim-analysis.test.ts new file mode 100644 index 0000000..94fa32b --- /dev/null +++ b/src/__tests__/nim-analysis.test.ts @@ -0,0 +1,239 @@ +import { describe, it, expect } from "vitest"; +import { + mergeLlmFindings, + buildSystemPrompt, + type UsageSummary, + type AnalysisContext, +} from "@/lib/nim/analysis"; +import { Severity } from "@/types/analysis"; + +const row: UsageSummary = { + id: "key1-ws1", + name: "key1", + ws: "production", + model: "claude-opus-4-6", + ml: "Opus 4.6", + inp: 10_000_000, + out: 100_000, + cached: 0, + cacheCreated: 0, + reqs: 1000, + activeDays: 25, + cur: 200, +}; + +const ctx: AnalysisContext = { + vendor: "anthropic", + totalSpend: 200, + workspaceCount: 1, +}; + +describe("mergeLlmFindings", () => { + it("clamps savings to [0, currentCost] and grounds dollar figures", () => { + const [f] = mergeLlmFindings( + [ + { + rowId: "key1-ws1", + category: "Model Downgrade → Haiku", + severity: "critical", + confidence: 0.9, + savingsMonthly: 9999, // absurd; must clamp to cur + reason: "low output", + action: "switch model", + }, + ], + [row], + ctx + ); + expect(f.sav).toBe(200); + expect(f.opt).toBe(0); + expect(f.cur).toBe(200); + expect(f.sev).toBe(Severity.CRITICAL); + expect(f.model).toBe("claude-opus-4-6"); + }); + + it("clamps negative savings to 0 and confidence to [0,1]", () => { + const [f] = mergeLlmFindings( + [ + { + rowId: "key1-ws1", + category: "Model Upgrade", // quality category survives zero savings + severity: "info", + confidence: 5, + savingsMonthly: -50, + reason: "r", + action: "a", + }, + ], + [row], + ctx + ); + expect(f.sav).toBe(0); + expect(f.conf).toBe(1); + expect(f.impact).toBe("Quality improvement"); + }); + + it("handles org-level findings with no matching row", () => { + const [f] = mergeLlmFindings( + [ + { + rowId: "org", + category: "Workspace Organization", + severity: "info", + confidence: 0.8, + savingsMonthly: 0, + reason: "all spend in default", + action: "split workspaces", + }, + ], + [row], + ctx + ); + expect(f.cur).toBe(200); // falls back to totalSpend + expect(f.name).toBe("Organization"); + }); + + it("drops findings without a reason and sorts by severity then savings", () => { + const out = mergeLlmFindings( + [ + { + rowId: "key1-ws1", + category: "A", + severity: "info", + confidence: 0.5, + savingsMonthly: 10, + reason: "r", + action: "a", + }, + { + rowId: "bad", + category: "B", + severity: "critical", + confidence: 0.5, + savingsMonthly: 10, + reason: "", + action: "a", + }, + { + rowId: "key1-ws1", + category: "C", + severity: "critical", + confidence: 0.5, + savingsMonthly: 50, + reason: "r", + action: "a", + }, + ], + [row], + ctx + ); + expect(out).toHaveLength(2); // empty-reason dropped + expect(out[0].sev).toBe(Severity.CRITICAL); + expect(out[1].sev).toBe(Severity.INFO); + }); +}); + +describe("mergeLlmFindings guardrails", () => { + // A cheap Haiku row — downgrades to pricier tiers must be rejected. + const haikuRow: UsageSummary = { + ...row, + id: "key2-ws1", + name: "key2", + model: "claude-haiku-3", + ml: "Haiku 3", + cur: 0.03, + }; + const f = (over: Partial>) => ({ + rowId: "key1-ws1", + category: "X", + severity: "warning", + confidence: 0.7, + savingsMonthly: 10, + reason: "r", + action: "a", + ...over, + }); + + it("drops a 'downgrade' whose target is not cheaper than the row's model", () => { + const out = mergeLlmFindings( + [ + f({ + rowId: "key2-ws1", + category: "Model Downgrade → Sonnet", + savingsMonthly: 0.01, + }), + f({ + rowId: "key2-ws1", + category: "Model Downgrade → Haiku", + savingsMonthly: 0.01, + }), + ], + [haikuRow], + ctx + ); + // Haiku→Sonnet (upgrade) and Haiku→Haiku (no-op) both dropped. + expect(out).toHaveLength(0); + }); + + it("keeps only the single best downgrade per row", () => { + const out = mergeLlmFindings( + [ + f({ category: "Model Downgrade → Sonnet", savingsMonthly: 80 }), + f({ category: "Model Downgrade → Haiku", savingsMonthly: 90 }), + ], + [row], + ctx + ); + expect(out).toHaveLength(1); + expect(out[0].cat).toContain("Haiku"); + expect(out[0].sav).toBe(90); + }); + + it("caps cumulative savings per row at the row's spend", () => { + const out = mergeLlmFindings( + [ + f({ category: "Prompt Caching", savingsMonthly: 150 }), + f({ category: "RAG Optimization", savingsMonthly: 150 }), + ], + [row], // cur = 200 + ctx + ); + const total = out.reduce((s, x) => s + x.sav, 0); + expect(total).toBeLessThanOrEqual(200); + expect(total).toBeCloseTo(200); + }); + + it("drops zero-savings cost findings as noise", () => { + const out = mergeLlmFindings( + [f({ category: "Batch API Migration", savingsMonthly: 0 })], + [row], + ctx + ); + expect(out).toHaveLength(0); + }); + + it("drops cost findings the cumulative cap zeroed out (no $0 'quality' noise)", () => { + // First finding claims the row's whole cost; later ones get capped to $0 + // and must not survive as junk "Quality improvement" rows. + const out = mergeLlmFindings( + [ + f({ category: "Prompt Caching", savingsMonthly: 200 }), // = full cur + f({ category: "RAG Optimization", savingsMonthly: 50 }), + f({ category: "Batch API Migration", savingsMonthly: 50 }), + ], + [row], // cur = 200 + ctx + ); + expect(out).toHaveLength(1); + expect(out[0].cat).toBe("Prompt Caching"); + expect(out.every((x) => x.sav > 0)).toBe(true); + }); +}); + +describe("buildSystemPrompt", () => { + it("includes the analysis rules and vendor-appropriate models", () => { + expect(buildSystemPrompt("anthropic")).toContain("claude-haiku-4-5"); + expect(buildSystemPrompt("openai")).toContain("gpt-4o-mini"); + expect(buildSystemPrompt("anthropic")).toContain("RAG CONTEXT BLOAT"); + }); +}); diff --git a/src/__tests__/storage-eviction.test.ts b/src/__tests__/storage-eviction.test.ts new file mode 100644 index 0000000..464e334 --- /dev/null +++ b/src/__tests__/storage-eviction.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect, beforeEach } from "vitest"; + +// localStorage isn't in the node test env; mock one with a byte cap so setItem +// throws QuotaExceededError once the serialized history grows too large. +class MockStorage { + store = new Map(); + constructor(private cap: number) {} + getItem(k: string) { + return this.store.has(k) ? this.store.get(k)! : null; + } + setItem(k: string, v: string) { + let total = v.length; + for (const [kk, vv] of this.store) if (kk !== k) total += vv.length; + if (total > this.cap) { + const e = new Error("quota") as Error & { name: string }; + e.name = "QuotaExceededError"; + throw e; + } + this.store.set(k, v); + } + removeItem(k: string) { + this.store.delete(k); + } + clear() { + this.store.clear(); + } +} + +const g = globalThis as unknown as { + window?: object; + localStorage?: MockStorage; +}; + +const N = 10_000; +const bigRaw = { blob: "x".repeat(N) } as never; + +async function freshStorage(cap: number) { + g.window = {}; + g.localStorage = new MockStorage(cap); + const mod = await import("@/lib/storage"); + return mod; +} + +describe("saveHistory quota eviction", () => { + beforeEach(() => { + g.localStorage?.clear(); + }); + + it("evicts the oldest analysis when storage is full, keeping the newest", async () => { + const { storage, Vendor } = await freshStorage(2.5 * N); + + const rep = {} as never; + storage.saveAnalysis( + "id1", + Vendor.ANTHROPIC, + 2026, + 0, + "Org1", + "o1", + rep, + bigRaw + ); + storage.saveAnalysis( + "id2", + Vendor.ANTHROPIC, + 2026, + 1, + "Org2", + "o2", + rep, + bigRaw + ); + // Third write overflows → oldest (id1) must be evicted, id2 + id3 survive. + storage.saveAnalysis( + "id3", + Vendor.ANTHROPIC, + 2026, + 2, + "Org3", + "o3", + rep, + bigRaw + ); + + expect(storage.getAnalysis("id1")).toBeNull(); + expect(storage.getAnalysis("id2")).not.toBeNull(); + expect(storage.getAnalysis("id3")).not.toBeNull(); + }); +}); diff --git a/src/app/api/nim/route.ts b/src/app/api/nim/route.ts new file mode 100644 index 0000000..6796978 --- /dev/null +++ b/src/app/api/nim/route.ts @@ -0,0 +1,139 @@ +import { NextResponse } from "next/server"; + +// Proxies chat-completion requests to NVIDIA NIM (OpenAI-compatible). The key is +// a server-side deployment env var (NIM_API_KEY) — never sent from the browser. +const NIM_URL = + process.env.NIM_BASE_URL || "https://integrate.api.nvidia.com/v1"; +// Independent of MOCK_API_URL: mocking vendor usage data must NOT also mock the +// LLM. Set NIM_MOCK=1 only to exercise the analysis flow without a real nvapi key. +const NIM_MOCK = + process.env.NIM_MOCK === "1" || process.env.NIM_MOCK === "true"; + +// True when the LLM-analysis feature is usable: a key is configured (or mock). +const nimEnabled = () => NIM_MOCK || !!process.env.NIM_API_KEY; + +// Lets the UI show the "AI analysis" toggle only when NIM is actually configured. +export async function GET() { + return NextResponse.json({ enabled: nimEnabled() }); +} + +// When NIM_MOCK is set, synthesize findings from the rows in the request instead +// of calling NVIDIA — so the LLM path is testable offline. +function mockCompletion(body: string) { + let rows: Array<{ id: string; monthlyCostUsd: number; model: string }> = []; + let workspaceCount = 0; + try { + const parsed = JSON.parse(body); + const user = parsed.messages?.find( + (m: { role: string }) => m.role === "user" + ); + const payload = JSON.parse(user.content); + rows = payload.rows || []; + workspaceCount = payload.orgContext?.workspaceCount ?? 0; + } catch { + /* fall through with empty rows */ + } + + const findings: unknown[] = []; + const top = [...rows] + .sort((a, b) => b.monthlyCostUsd - a.monthlyCostUsd) + .slice(0, 2); + for (const r of top) { + const sav = +(r.monthlyCostUsd * 0.4).toFixed(2); + findings.push({ + rowId: r.id, + category: "Model Downgrade → smaller model", + severity: + r.monthlyCostUsd > 100 + ? "critical" + : r.monthlyCostUsd > 10 + ? "warning" + : "info", + confidence: 0.7, + savingsMonthly: sav, + reason: `[MOCK] ${r.model} costs $${r.monthlyCostUsd}/mo; a smaller model likely handles this workload.`, + action: "Mock finding — set a real NIM key and unset MOCK_API_URL.", + }); + } + if (rows.length > 0 && workspaceCount <= 1) { + findings.push({ + rowId: "org", + category: "Workspace Organization", + severity: "info", + confidence: 0.9, + savingsMonthly: 0, + reason: "[MOCK] All spend sits in one workspace/project.", + action: "Segment by environment/team/product for cost attribution.", + }); + } + + return { + choices: [{ message: { content: JSON.stringify({ findings }) } }], + }; +} + +export async function POST(request: Request) { + try { + let body = await request.text(); + + if (NIM_MOCK) { + return NextResponse.json(mockCompletion(body), { status: 200 }); + } + + const nimKey = process.env.NIM_API_KEY; + if (!nimKey) { + return NextResponse.json( + { + error: + "NVIDIA NIM is not configured. Set NIM_API_KEY in the deployment environment.", + }, + { status: 503 } + ); + } + + // Optional deployment-side model override (e.g. a faster NIM model). + if (process.env.NIM_MODEL) { + try { + body = JSON.stringify({ + ...JSON.parse(body), + model: process.env.NIM_MODEL, + }); + } catch { + /* leave body as-is if it isn't JSON */ + } + } + + let response: Response; + try { + response = await fetch(`${NIM_URL}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${nimKey}`, + }, + body, + // Bound the upstream wait so we don't hang to the 300s stream timeout. + signal: AbortSignal.timeout(115_000), + }); + } catch (e) { + if (e instanceof DOMException && e.name === "TimeoutError") { + return NextResponse.json( + { + error: "NVIDIA NIM timed out — the model took too long to respond.", + }, + { status: 504 } + ); + } + throw e; + } + + const data = await response.json(); + return NextResponse.json(data, { status: response.status }); + } catch (error) { + console.error("NIM proxy error:", error); + return NextResponse.json( + { error: "Failed to proxy request to NVIDIA NIM" }, + { status: 500 } + ); + } +} diff --git a/src/app/page.tsx b/src/app/page.tsx index d23fce0..3770ccf 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -10,6 +10,8 @@ import { pull as pullAnthropic } from "@/lib/anthropic/api"; import { pull as pullOpenAI } from "@/lib/openai/api"; import { agg, findIssues } from "@/lib/anthropic/analysis"; import { aggOpenAI, findIssuesOpenAI } from "@/lib/openai/analysis"; +import { findIssuesLLM } from "@/lib/nim/analysis"; +import { toSummariesAnthropic, toSummariesOpenAI } from "@/lib/nim/adapters"; import { tc } from "@/lib/anthropic/pricing"; import { tcOpenAI } from "@/lib/openai/pricing"; import { demoAnthropic, demoOpenAI } from "@/lib/demo"; @@ -166,6 +168,25 @@ function HomeContent() { const [step, setStep] = useState(""); const [tickerItems, setTickerItems] = useState(DEFAULT_TICKER); + // NVIDIA NIM LLM analysis: a persisted on/off setting. The key lives in a + // server env var (NIM_API_KEY); the toggle only shows when the server reports + // NIM is configured. + const [nimAvailable, setNimAvailable] = useState(false); + const [useNim, setUseNim] = useState(false); + + useEffect(() => { + fetch("/api/nim") + .then((r) => r.json()) + .then((d) => setNimAvailable(!!d?.enabled)) + .catch(() => setNimAvailable(false)); + setUseNim(localStorage.getItem("tokenpilot_use_nim") === "1"); + }, []); + + const toggleNim = (on: boolean) => { + setUseNim(on); + localStorage.setItem("tokenpilot_use_nim", on ? "1" : "0"); + }; + const buildTicker = useCallback( ( data: Record< @@ -232,6 +253,44 @@ function HomeContent() { router.push(`/?${params.toString()}`); }; + // Choose NIM-guided LLM analysis when the setting is on, else the rule engine. + // NIM errors propagate to the caller so they surface in the UI rather than + // silently falling back. Used by both the real analysis and the demo. + const nimOn = () => useNim && nimAvailable; + + const analyzeAnthropic = async ( + src: Parameters[0], + ws: Parameters[1], + buckets: Parameters[2], + spend: number + ) => { + if (nimOn()) { + setStep("Analyzing usage with NVIDIA NIM..."); + return findIssuesLLM(toSummariesAnthropic(src, ws, buckets), { + vendor: "anthropic", + totalSpend: spend, + workspaceCount: ws?.length || 0, + }); + } + return findIssues(src, ws, buckets); + }; + + const analyzeOpenAI = async ( + rows: Parameters[0], + projects: Parameters[1], + spend: number + ) => { + if (nimOn()) { + setStep("Analyzing usage with NVIDIA NIM..."); + return findIssuesLLM(toSummariesOpenAI(rows, projects), { + vendor: "openai", + totalSpend: spend, + workspaceCount: projects?.length || 0, + }); + } + return findIssuesOpenAI(rows, projects); + }; + const startAnalysis = async () => { if (!key.trim()) { setErr("Enter your Admin API key"); @@ -276,7 +335,6 @@ function HomeContent() { // Process data into report const rows = aggOpenAI(d.usage); - const findings = findIssuesOpenAI(rows, d.projects); // Calculate total spend and tokens // Use actual costs from Costs API if available, otherwise calculate from usage @@ -295,6 +353,8 @@ function HomeContent() { } } + const findings = await analyzeOpenAI(rows, d.projects, spend); + let ti = 0; let to = 0; for (const row of rows) { @@ -399,11 +459,7 @@ function HomeContent() { const bk = agg(d.bk); const bm = agg(d.bm); const src = bk.length ? bk : bm; - const findings = findIssues( - src, - d.ws, - d.rawBk.length ? d.rawBk : d.rawBm - ); + const buckets = d.rawBk.length ? d.rawBk : d.rawBm; let spend = 0; let ti = 0; @@ -414,6 +470,8 @@ function HomeContent() { to += a.out; } + const findings = await analyzeAnthropic(src, d.ws, buckets, spend); + const wb = agg(d.bw); // Initialize all workspaces with $0 spend @@ -513,7 +571,6 @@ function HomeContent() { const d = demoOpenAI(y, m); const rows = aggOpenAI(d.usage); - const findings = findIssuesOpenAI(rows, d.projects); let spend = 0; if (d.costs && d.costs.data.length > 0) { @@ -528,6 +585,8 @@ function HomeContent() { } } + const findings = await analyzeOpenAI(rows, d.projects, spend); + let ti = 0, to = 0; for (const row of rows) { @@ -609,11 +668,7 @@ function HomeContent() { const bk = agg(d.bk); const bm = agg(d.bm); const src = bk.length ? bk : bm; - const findings = findIssues( - src, - d.ws, - d.rawBk.length ? d.rawBk : d.rawBm - ); + const buckets = d.rawBk.length ? d.rawBk : d.rawBm; let spend = 0, ti = 0, @@ -624,6 +679,8 @@ function HomeContent() { to += a.out; } + const findings = await analyzeAnthropic(src, d.ws, buckets, spend); + const wb = agg(d.bw); const wa: Record = {}; const wn: Record = {}; @@ -790,6 +847,27 @@ function HomeContent() { : "Console → API Keys → Admin Keys → Create admin key (read-only)"}

+ {/* AI analysis setting — only shown when NIM is configured server-side */} + {nimAvailable && ( + + )} + {/* Demo mode */}
diff --git a/src/lib/nim/adapters.ts b/src/lib/nim/adapters.ts new file mode 100644 index 0000000..46175b3 --- /dev/null +++ b/src/lib/nim/adapters.ts @@ -0,0 +1,67 @@ +/* Vendor row → neutral UsageSummary for LLM-guided analysis. */ + +import type { AggregatedRow, UsageBucket, Workspace } from "@/types"; +import { pr, tc } from "@/lib/anthropic/pricing"; +import { analyzeTemporalPattern } from "@/lib/anthropic/analysis"; +import { prOpenAI, tcOpenAI } from "@/lib/openai/pricing"; +import type { OpenAIAggregatedRow } from "@/lib/openai/analysis"; +import type { UsageSummary } from "./analysis"; + +export function toSummariesAnthropic( + rows: AggregatedRow[], + workspaces: Workspace[], + buckets: UsageBucket[] +): UsageSummary[] { + const wn: Record = {}; + (workspaces || []).forEach((w) => { + wn[w.id] = w.display_name || w.name || w.id; + }); + + return rows.map((r) => { + const p = pr(r.model); + return { + id: `${r.kid || r.model}-${r.wid || "x"}`, + name: r.kid || r.model, + ws: r.wid ? wn[r.wid] || r.wid : "Default workspace", + model: r.model, + ml: p.l, + inp: r.inp, + out: r.out, + cached: r.cached, + cacheCreated: r.cacheCreated, + reqs: r.reqs, + activeDays: r.activeDays, + cur: tc(r.model, r.inp, r.out), + temporal: analyzeTemporalPattern(buckets || [], r.kid, r.model), + }; + }); +} + +export function toSummariesOpenAI( + rows: OpenAIAggregatedRow[], + projects: { id: string; name?: string }[] +): UsageSummary[] { + const pn: Record = {}; + (projects || []).forEach((p) => { + pn[p.id] = p.name || p.id; + }); + + return rows.map((r) => { + const p = prOpenAI(r.model || r.line_item || ""); + const cur = r.cost > 0 ? r.cost : tcOpenAI(r.model, r.inp, r.out); + return { + id: `${r.project_id || "default"}-${r.model || r.line_item || "?"}`, + name: r.model || r.line_item || "?", + ws: r.project_id ? pn[r.project_id] || r.project_id : "Default project", + model: r.model || r.line_item || "?", + ml: p.l, + inp: r.inp, + out: r.out, + cached: 0, + cacheCreated: 0, + reqs: r.reqs, + activeDays: r.activeDays, + cur, + }; + }); +} diff --git a/src/lib/nim/analysis.ts b/src/lib/nim/analysis.ts new file mode 100644 index 0000000..216b2d1 --- /dev/null +++ b/src/lib/nim/analysis.ts @@ -0,0 +1,418 @@ +/* ═══════════════════ NVIDIA NIM — LLM-GUIDED ANALYSIS ═══════════════════ */ +/* + * Drop-in alternative to the hardcoded rule engines (findIssues / + * findIssuesOpenAI). Instead of fixed thresholds, we hand a structured usage + * summary to an LLM hosted on NVIDIA NIM and let it reason about which + * optimizations apply. The rules below are *guidance* for the model, not code. + * + * The deterministic metrics (cost, savings clamping, ids) are still computed + * here so dollar figures stay grounded and the model can't hallucinate numbers. + */ + +import type { Finding, TemporalPattern } from "@/types"; +import { Severity } from "@/types/analysis"; + +/** Default NIM-hosted model. OpenAI-compatible chat completions. */ +export const NIM_DEFAULT_MODEL = "meta/llama-3.3-70b-instruct"; + +/* ─────────────── ANALYSIS RULES (LLM guidance) ─────────────── */ + +export const ANALYSIS_RULES = ` +1. MODEL DOWNGRADE → small model (Haiku / GPT-4o-mini) + When: avg output tokens per request is low (<~150) over many requests, and + input isn't huge. Pattern of classification / routing / extraction. + Action: A/B test the small model on ~100 requests; ship if accuracy delta <2%. + +2. RAG CONTEXT BLOAT + When: input:output ratio is high (>~12:1) with large average input + (>~5000 tok/req) and millions of input tokens/mo. Retrieval is over-fetching. + Action: reduce top-k, add reranking, tighten chunk size. Downgrade Opus→Sonnet + for RAG (quality is retrieval-bound, not model-bound). + +3. PROMPT CACHING MISS + When: high input volume (>~20M tok/mo) with a very low cache-read rate (<5%). + Static prefixes (system prompt, tool defs) re-sent uncached every request. + Action: add cache_control breakpoints on the stable prefix. ~90% off cached part. + +4. CACHE WRITE INEFFICIENCY + When: large cache-creation tokens but low reuse (reads/writes < ~1). Cache is + invalidating before it pays back (writes cost 25% more; reads 90% less). + Action: extend cache TTL, keep prefix stable, avoid dynamic content before the + breakpoint. + +5. BATCH API MIGRATION + When: bursty / spiky daily traffic (high coefficient of variation) or many + zero-usage days, and the work isn't latency-sensitive. + Action: move async work to the Batch API for ~50% input discount (≤24h turnaround). + +6. MODEL DOWNGRADE Opus→Sonnet (or premium→mid reasoning model) + When: a premium model handles moderate-complexity work where the mid tier is + within ~5% quality. Action: A/B on 10% traffic, migrate if quality holds. + +7. LEGACY / OLD-GENERATION MODEL + When: an older model generation is still in use and a newer same-tier model is + cheaper or better. Action: update the model string (usually drop-in). + +8. ORG STRUCTURE (workspaces / projects) + When: all spend is in one default workspace/project with no segmentation. + Action: split by environment/team/product for cost attribution. (Quality/visibility + win, savings = 0.) + +Only emit a finding when the data actually supports it. Be conservative with +savings — prefer underestimating. Skip rows costing under ~$0.50/mo. +`.trim(); + +export function buildSystemPrompt(vendor: "anthropic" | "openai"): string { + const small = vendor === "openai" ? "gpt-4o-mini" : "claude-haiku-4-5"; + const mid = vendor === "openai" ? "gpt-4o" : "claude-sonnet-4-6"; + return `You are TokenPilot's LLM cost-optimization analyst for ${vendor === "openai" ? "OpenAI" : "Anthropic"} API usage. + +You receive a JSON array of per-row usage summaries (one row = a model used by an +API key inside a workspace/project) plus org-level context. Apply the rules below +and return concrete, actionable cost-savings findings. + +ANALYSIS RULES: +${ANALYSIS_RULES} + +Suggested small model: ${small}. Suggested mid model: ${mid}. + +Respond with ONLY a JSON object, no prose, of the form: +{ + "findings": [ + { + "rowId": "", + "category": "short label, e.g. 'Model Downgrade → Haiku', 'RAG Optimization', 'Prompt Caching', 'Batch API Migration', 'Model Upgrade', 'Workspace Organization'", + "severity": "critical | warning | info", + "confidence": 0.0, + "savingsMonthly": 0.0, + "reason": "1-3 sentences. Cite ONLY the actual values from THIS row.", + "action": "concrete next step the engineer can take." + } + ] +} + +GROUNDING (critical): In "reason", quote only the real numbers from the row you are +analyzing — its monthlyCostUsd, inputTokens, cacheReadRate, avgOutputPerReq, requests, +and its own model id. NEVER repeat the threshold numbers written in the rules above +(e.g. ">20M tok/mo", "<5%", ">5000 tok/req") — those are triggers, not this row's data. +NEVER mention another row's model or numbers. If a row's real numbers don't clear a +rule's threshold, do not emit that finding for it. + +Severity guide: critical = >$100/mo or >20% of this row's cost; warning = meaningful +savings; info = small or quality-only. savingsMonthly is the estimated monthly USD +saved (0 for quality-only or org-structure findings) and must not exceed the row's +current monthly cost. confidence is 0-1. Emit at most one finding per category per row.`; +} + +/* ─────────────── INPUT SUMMARY (vendor-agnostic) ─────────────── */ + +export interface UsageSummary { + id: string; // stable id base (e.g. apiKeyId|model|wid) + name: string; // api key id or model + ws: string; // workspace / project display name + model: string; + ml: string; // model label + inp: number; + out: number; + cached: number; + cacheCreated: number; + reqs: number; + activeDays: number; + cur: number; // current monthly cost (USD), computed by caller + temporal?: TemporalPattern; +} + +export interface AnalysisContext { + vendor: "anthropic" | "openai"; + totalSpend: number; + workspaceCount: number; + model?: string; +} + +const EMPTY_TEMPORAL: TemporalPattern = { + burstiness: 0, + consistency: 0, + batchCandidate: false, + meanDaily: 0, +}; + +interface LlmFinding { + rowId: string; + category: string; + severity: string; + confidence: number; + savingsMonthly: number; + reason: string; + action: string; +} + +// Cap rows sent to the LLM. A large org has hundreds of model/key/workspace +// combos; sending all of them bloats the prompt and the model's runtime (the +// free NIM endpoint times out). Savings concentrate in the priciest rows, so +// keep the top N by cost. ponytail: raise if the long tail ever matters. +const MAX_ROWS = 30; + +/** Build the compact, number-rich payload the model reasons over. */ +function summarize(rows: UsageSummary[]) { + return rows + .filter((r) => r.cur >= 0.5 && (r.inp > 0 || r.out > 0)) + .sort((a, b) => b.cur - a.cur) + .slice(0, MAX_ROWS) + .map((r) => { + const ratio = r.out > 0 ? +(r.inp / r.out).toFixed(1) : 0; + const cacheRate = + r.inp + r.cached > 0 ? +(r.cached / (r.inp + r.cached)).toFixed(3) : 0; + return { + id: r.id, + model: r.model, + workspace: r.ws, + monthlyCostUsd: +r.cur.toFixed(2), + inputTokens: r.inp, + outputTokens: r.out, + cacheReadTokens: r.cached, + cacheWriteTokens: r.cacheCreated, + requests: r.reqs, + avgInputPerReq: r.reqs > 0 ? Math.round(r.inp / r.reqs) : 0, + avgOutputPerReq: r.reqs > 0 ? Math.round(r.out / r.reqs) : 0, + inputOutputRatio: ratio, + cacheReadRate: cacheRate, + activeDays: r.activeDays, + burstiness: r.temporal ? +r.temporal.burstiness.toFixed(2) : undefined, + batchCandidate: r.temporal?.batchCandidate, + }; + }); +} + +function toSeverity(s: string): Severity { + switch ((s || "").toLowerCase()) { + case "critical": + return Severity.CRITICAL; + case "warning": + return Severity.WARNING; + default: + return Severity.INFO; + } +} + +// Cost tier rank from a model id/label — lower = cheaper. Heuristic, but enough +// to catch the LLM mislabelling an upgrade as a "downgrade". ponytail: extend +// the regexes if a new tier shows up. +function tierRank(s: string): number { + const t = (s || "").toLowerCase(); + if (/haiku|mini|nano|small|flash|lite|8b/.test(t)) return 0; + if (/sonnet|gpt-4o|[^a-z]4o|medium|70b/.test(t)) return 1; + if (/opus|gpt-4(?!o)|gpt-5|ultra|large|405b/.test(t)) return 2; + return 1; // unknown → neutral mid tier +} + +// If a category describes a model downgrade, return the target tier rank, else +// null. e.g. "Model Downgrade → Sonnet" → 1. +function downgradeTargetRank(category: string): number | null { + if (!/downgrad/i.test(category || "")) return null; + const target = (category.split(/→|->/).pop() || category).trim(); + return tierRank(target); +} + +const KEEP_ZERO_SAVINGS = /upgrade|organization|workspace|project|quality/i; + +interface Candidate { + finding: Finding; + rowId: string; + cur: number; + sav: number; + conf: number; + isDowngrade: boolean; +} + +/** + * Merge the LLM's reasoning back onto deterministic per-row metrics, then apply + * guardrails so a weak model can't produce contradictory or impossible advice: + * 1. Per-finding savings clamped to [0, row cost]. + * 2. Drop "downgrades" whose target isn't actually cheaper than the row's model. + * 3. Keep only the single best downgrade per row (no Sonnet AND Haiku at once). + * 4. Drop zero-savings cost findings (noise); keep zero-savings quality/org ones. + * 5. Cap cumulative savings per row at the row's spend (no >100% savings). + * Exported for testing. + */ +export function mergeLlmFindings( + llm: LlmFinding[], + rows: UsageSummary[], + ctx: AnalysisContext +): Finding[] { + const byId = new Map(rows.map((r) => [r.id, r])); + const candidates: Candidate[] = []; + + for (const f of llm || []) { + if (!f || !f.reason) continue; + const r = byId.get(f.rowId); + const cur = r ? r.cur : ctx.totalSpend; + const sav = Math.max(0, Math.min(f.savingsMonthly || 0, cur)); + const conf = Math.max(0, Math.min(f.confidence ?? 0.5, 1)); + const category = f.category || "Optimization"; + + // Guardrail 2: a "downgrade" to an equal/pricier tier isn't a saving. + const targetRank = downgradeTargetRank(category); + const isDowngrade = targetRank !== null; + if (r && isDowngrade && targetRank! >= tierRank(r.model || r.ml)) continue; + + // Guardrail 4: drop zero-savings cost findings; keep quality/org ones. + if (sav <= 0 && !KEEP_ZERO_SAVINGS.test(category)) continue; + + const ratio = r && r.out > 0 ? r.inp / r.out : 0; + const cr = r && r.inp + r.cached > 0 ? r.cached / (r.inp + r.cached) : 0; + const slug = category.replace(/[^a-z0-9]/gi, "-").toLowerCase(); + + candidates.push({ + rowId: f.rowId, + cur, + sav, + conf, + isDowngrade, + finding: { + id: `${f.rowId}-${slug}`, + name: r ? r.name : "Organization", + ws: r ? r.ws : "All workspaces", + model: r ? r.model : "N/A", + ml: r ? r.ml : category, + inp: r?.inp ?? 0, + out: r?.out ?? 0, + cached: r?.cached ?? 0, + reqs: r?.reqs ?? 0, + ao: r && r.reqs > 0 ? Math.round(r.out / r.reqs) : 0, + ai: r && r.reqs > 0 ? Math.round(r.inp / r.reqs) : 0, + ratio, + cr, + cur, + opt: cur, // filled after capping + sav, // filled after capping + reason: f.reason, + action: f.action || "", + sev: toSeverity(f.severity), + cat: category as Finding["cat"], + conf, + impact: "", + activeDays: r?.activeDays ?? 0, + temporal: r?.temporal ?? EMPTY_TEMPORAL, + }, + }); + } + + // Guardrail 3: at most one downgrade per row — keep the highest-savings one. + const bestDowngrade = new Map(); + for (const c of candidates) { + if (!c.isDowngrade) continue; + const cur = bestDowngrade.get(c.rowId); + if (!cur || c.sav > cur.sav || (c.sav === cur.sav && c.conf > cur.conf)) { + bestDowngrade.set(c.rowId, c); + } + } + const kept = candidates.filter( + (c) => !c.isDowngrade || bestDowngrade.get(c.rowId) === c + ); + + // Guardrail 5: cap cumulative savings per row at its spend (highest first). + const headroom = new Map(); + for (const c of [...kept].sort((a, b) => b.sav - a.sav)) { + const left = headroom.get(c.rowId) ?? c.cur; + const sav = Math.min(c.sav, Math.max(0, left)); + headroom.set(c.rowId, left - sav); + c.finding.sav = sav; + c.finding.opt = Math.max(0, c.cur - sav); + const pct = c.cur > 0 ? Math.round((sav / c.cur) * 100) : 0; + c.finding.impact = + sav > 0 ? `$${sav.toFixed(2)}/mo (${pct}%)` : "Quality improvement"; + } + + // Guardrail 4 (final pass): a cost finding the cumulative cap zeroed out is + // noise, not a "quality improvement" — drop it. Quality/org findings stay. + const final = kept.filter( + (c) => c.finding.sav > 0 || KEEP_ZERO_SAVINGS.test(c.finding.cat as string) + ); + + // Same severity+savings ordering the hardcoded engine uses. + const sv: Record = { + [Severity.CRITICAL]: 0, + [Severity.WARNING]: 1, + [Severity.INFO]: 2, + [Severity.OK]: 3, + }; + return final + .map((c) => c.finding) + .sort((a, b) => + sv[a.sev] !== sv[b.sev] ? sv[a.sev] - sv[b.sev] : b.sav - a.sav + ); +} + +/** Extract a JSON object even if the model wraps it in prose / code fences. */ +function parseFindings(content: string): LlmFinding[] { + let txt = content.trim(); + const fence = txt.match(/```(?:json)?\s*([\s\S]*?)```/); + if (fence) txt = fence[1].trim(); + const start = txt.indexOf("{"); + const end = txt.lastIndexOf("}"); + if (start >= 0 && end > start) txt = txt.slice(start, end + 1); + const parsed = JSON.parse(txt); + return Array.isArray(parsed) ? parsed : parsed.findings || []; +} + +/** + * Run LLM-guided analysis via NVIDIA NIM. Returns Findings in the same shape as + * the hardcoded engines. Throws on transport/parse failure so callers can fall + * back to the rule engine. + */ +export async function findIssuesLLM( + rows: UsageSummary[], + ctx: AnalysisContext +): Promise { + const payload = summarize(rows); + if (payload.length === 0) return []; + + const body = { + model: ctx.model || NIM_DEFAULT_MODEL, + temperature: 0.2, + max_tokens: 2048, + messages: [ + { role: "system", content: buildSystemPrompt(ctx.vendor) }, + { + role: "user", + content: JSON.stringify({ + orgContext: { + vendor: ctx.vendor, + totalMonthlySpendUsd: +ctx.totalSpend.toFixed(2), + workspaceCount: ctx.workspaceCount, + }, + rows: payload, + }), + }, + ], + }; + + let res: Response; + try { + res = await fetch("/api/nim", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + // Fail fast rather than hanging on a slow NIM model. + signal: AbortSignal.timeout(120_000), + }); + } catch (e) { + if (e instanceof DOMException && e.name === "TimeoutError") { + throw new Error( + "NIM request timed out (120s) — the model is taking too long. Retry, or set NIM_BASE_URL/NIM_MODEL to a faster NIM model." + ); + } + throw e; + } + + if (!res.ok) { + const t = await res.text(); + throw new Error(`NIM analysis failed (${res.status}): ${t.slice(0, 200)}`); + } + + const data = await res.json(); + const content: string = data?.choices?.[0]?.message?.content ?? ""; + if (!content) throw new Error("NIM returned an empty response"); + + return mergeLlmFindings(parseFindings(content), rows, ctx); +} diff --git a/src/lib/storage.ts b/src/lib/storage.ts index 9cacd8a..374e03d 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -80,10 +80,26 @@ function getHistory(): StoredHistory { */ function saveHistory(history: StoredHistory): void { if (typeof window === "undefined") return; - try { - localStorage.setItem(HISTORY_STORAGE_KEY, JSON.stringify(history)); - } catch (e) { - console.error("Failed to save history:", e); + // On quota errors, evict the oldest analysis and retry. Each analysis stores + // full raw API responses, so a few accumulate past the ~5MB localStorage cap. + // ponytail: oldest-first eviction; if a single record alone overflows, give up. + let working = history; + for (;;) { + try { + localStorage.setItem(HISTORY_STORAGE_KEY, JSON.stringify(working)); + return; + } catch (e) { + const ids = Object.keys(working); + if (ids.length <= 1) { + console.error("Failed to save history (storage full):", e); + return; + } + const oldest = ids.reduce((a, b) => + (working[a].createdAt || "") <= (working[b].createdAt || "") ? a : b + ); + working = { ...working }; + delete working[oldest]; + } } }