feat: say when the AI budget runs out, on email or any other channel

.changeset/budget-alert.md

@@ -0,0 +1,5 @@
+---
+'@flakemetry/notify': minor
+---
+
+New `ai_budget_spent` notification, sent when a project's daily LLM token budget runs out and root-cause analysis pauses. Subscribe on any channel — Slack, Discord, email or a signed webhook. Deduplicated per project per day, since the budget is re-checked on every run.

apps/web/src/lib/notify-events.ts

@@ -4,4 +4,5 @@ export const NOTIFY_EVENTS = [
   'rca_ready',
   'suite_regressed',
   'suite_slowed',
+  'ai_budget_spent',
 ]

apps/worker/src/__tests__/budget-alert.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { createEventBus } from '../events'
+import { startNotifications } from '../notify'
+
+/**
+ * The budget alert is only useful if it travels: the worker emits it deep inside RCA, and
+ * three separate pieces have to agree — the event bus name, the notification type, and the
+ * channel's subscription list. Testing the formatter alone would pass with any of them
+ * broken.
+ */
+describe('the AI budget alert reaches a channel', () => {
+  it('dispatches to a subscribed channel when the budget is spent', async () => {
+    const delivered: string[] = []
+    const events = createEventBus(() => undefined)
+
+    const enabled = startNotifications(
+      events,
+      {
+        FLAKEMETRY_SLACK_WEBHOOK: 'https://hooks.slack.com/services/probe',
+        FLAKEMETRY_NOTIFY_EVENTS: 'ai_budget_spent',
+      },
+      () => Promise.resolve([]),
+    )
+    expect(enabled, 'notifications did not start — the rest proves nothing').toBe(true)
+
+    const originalFetch = globalThis.fetch
+    globalThis.fetch = (async (_url: string, init: RequestInit) => {
+      delivered.push(String(init.body))
+      return new Response('ok', { status: 200 })
+    }) as unknown as typeof fetch
+
+    try {
+      events.emit('ai.budget.spent', { projectId: 'p1', spent: 200_000, budget: 200_000 })
+      await vi.waitFor(() => expect(delivered.length).toBeGreaterThan(0), { timeout: 5000 })
+    } finally {
+      globalThis.fetch = originalFetch
+    }
+
+    expect(delivered.join()).toContain('paused until tomorrow')
+  })
+})

apps/worker/src/__tests__/rca.test.ts

@@ -158,6 +158,52 @@ describe.skipIf(!hasDb)('processFailures', () => {
     expect(seen).toHaveLength(1)
   })
 
+  it('announces that the budget is spent rather than only counting it', async () => {
+    const ctx = await seed()
+    const events = createEventBus()
+    const announced: DomainEventMap['ai.budget.spent'][] = []
+    events.on('ai.budget.spent', (payload) => announced.push(payload))
+
+    // A budget of one token is already spent by the first report, so the second failure
+    // hits the ceiling.
+    await processFailures(
+      prisma,
+      {
+        orgId: ctx.orgId,
+        projectId: ctx.projectId,
+        now: NOW,
+        provider: fakeProvider(),
+        aiEnabled: true,
+        dailyTokenBudget: 1,
+        events,
+      },
+      [failure(ctx.executionId)],
+    )
+    await processFailures(
+      prisma,
+      {
+        orgId: ctx.orgId,
+        projectId: ctx.projectId,
+        now: NOW,
+        provider: fakeProvider(),
+        aiEnabled: true,
+        dailyTokenBudget: 1,
+        events,
+      },
+      [
+        failure(
+          await seedExecution(ctx, 'AssertionError: expected cart to contain 3 items'),
+          'AssertionError: expected cart to contain 3 items',
+        ),
+      ],
+    )
+
+    // Without this the only trace is a metric counter, and a self-hosted instance whose
+    // explanations stopped at noon has nothing to tell anyone why.
+    expect(announced).toHaveLength(1)
+    expect(announced[0]).toMatchObject({ projectId: ctx.projectId, budget: 1 })
+  })
+
   it('does not call the model when AI is disabled', async () => {
     const ctx = await seed()
     const provider = fakeProvider()

apps/worker/src/events.ts

@@ -53,6 +53,11 @@ export interface DomainEventMap {
     baselineFailRate: number
     total: number
   }
+  'ai.budget.spent': {
+    projectId: string
+    spent: number
+    budget: number
+  }
   'suite.slowed': {
     projectId: string
     suite: string

apps/worker/src/notify.ts

@@ -83,6 +83,22 @@ export const startNotifications = (
     })
   })
 
+  events.on('ai.budget.spent', (payload) => {
+    void dispatcher.dispatch({
+      type: 'ai_budget_spent',
+      projectId: payload.projectId,
+      heading: 'AI budget spent for today',
+      summary: `Root-cause analysis is paused until tomorrow — ${payload.spent.toLocaleString()} of ${payload.budget.toLocaleString()} tokens used today.`,
+      fields: [
+        { label: 'Spent', value: payload.spent.toLocaleString() },
+        { label: 'Budget', value: payload.budget.toLocaleString() },
+      ],
+      // Once per project per day. The budget is checked on every run, and a suite that
+      // keeps failing would otherwise send one of these for each of them.
+      dedupeKey: `ai_budget_spent:${payload.projectId}:${new Date().toISOString().slice(0, 10)}`,
+    })
+  })
+
   events.on('quarantine.changed', (payload) => {
     void dispatcher.dispatch({
       type: 'quarantine_changed',

apps/worker/src/rca.ts

@@ -218,6 +218,10 @@ export const processFailures = async (
     if (!group.isNew) continue
     if (spent >= budget) {
       workerMetrics.rcaBudgetExhausted.add(1)
+      // Said out loud, not only counted. A metric nobody self-hosting is scraping is the
+      // difference between "analysis is off today" and nobody ever finding out why the
+      // explanations stopped.
+      ctx.events?.emit('ai.budget.spent', { projectId: ctx.projectId, spent, budget })
       break
     }
 

docs/configuration.md

@@ -139,6 +139,10 @@ without a provider gets you nothing — the worker has nothing to ask.
 Spend is bounded by `ai.dailyTokenBudget` per project, and only genuinely new error
 signatures reach the model at all; the rest are answered from the cluster's cached analysis.
 
+When the budget runs out, the worker emits `ai_budget_spent` — subscribe to it on any
+notification channel, including email, and the day analysis stops is a day someone hears
+about. It is deduplicated per project per day, since the budget is re-checked on every run.
+
 ### Notifications
 
 The worker pushes intelligence to Slack, Discord and email. Webhook delivery is best-effort and de-duplicated per channel so a flapping test can't spam a channel. Channels come from two places, applied together: **global env channels** (below) and **per-project channels** configured in **Settings → Notifications** (add a Slack/Discord webhook or an email address with an event filter). Events: `flaky_detected`, `quarantine_changed`, `rca_ready`, `suite_regressed` (a suite's fail-rate crossing its trailing baseline), and `suite_slowed` (a suite's average duration rising well above its trailing baseline).

packages/notify/src/__tests__/budget.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from 'vitest'
+
+import { formatDiscord } from '../discord'
+import { formatEmail } from '../email'
+import { NOTIFICATION_TYPES, type NotificationEvent } from '../message'
+import { formatSlack } from '../slack'
+
+const event: NotificationEvent = {
+  type: 'ai_budget_spent',
+  projectId: 'p1',
+  heading: 'AI budget spent for today',
+  summary: 'Root-cause analysis is paused until tomorrow — 200,000 of 200,000 tokens used today.',
+  fields: [
+    { label: 'Spent', value: '200,000' },
+    { label: 'Budget', value: '200,000' },
+  ],
+  dedupeKey: 'ai_budget_spent:p1:2026-08-18',
+}
+
+describe('the budget notification', () => {
+  it('says what happened and what it means, not just a number', () => {
+    const mail = formatEmail(event)
+
+    // Someone reading this in a mail client has no dashboard open. "Budget exceeded" alone
+    // does not tell them analysis has stopped.
+    expect(mail.subject).toContain('AI budget spent')
+    expect(mail.text).toContain('paused until tomorrow')
+    expect(mail.text).toContain('Budget: 200,000')
+  })
+
+  it('renders on every channel, not only the one it was written for', () => {
+    expect(() => formatSlack(event)).not.toThrow()
+    expect(() => formatDiscord(event)).not.toThrow()
+    expect(formatDiscord(event).embeds).toHaveLength(1)
+  })
+
+  it('is a type the routing layer knows about', () => {
+    // A type the dashboard cannot offer as a checkbox is one nobody can subscribe to.
+    expect(NOTIFICATION_TYPES).toContain('ai_budget_spent')
+  })
+
+  it('dedupes per project per day', () => {
+    // The budget is re-checked on every run, so without a key that collapses them a busy
+    // afternoon sends one of these per failing suite.
+    const monday = 'ai_budget_spent:p1:2026-08-18'
+    const alsoMonday = 'ai_budget_spent:p1:2026-08-18'
+    const tuesday = 'ai_budget_spent:p1:2026-08-19'
+    const otherProject = 'ai_budget_spent:p2:2026-08-18'
+
+    expect(monday).toBe(alsoMonday)
+    expect(monday).not.toBe(tuesday)
+    expect(monday).not.toBe(otherProject)
+  })
+})

packages/notify/src/discord.ts

@@ -11,6 +11,7 @@ const COLOR: Record<NotificationType, number> = {
   rca_ready: 0x5319e7,
   suite_regressed: 0xdc2626,
   suite_slowed: 0xd97706,
+  ai_budget_spent: 0x6b7280,
 }
 
 export const formatDiscord = (event: NotificationEvent): DiscordPayload => ({

packages/notify/src/message.ts

@@ -1,12 +1,18 @@
 export type NotificationType =
-  'flaky_detected' | 'quarantine_changed' | 'rca_ready' | 'suite_regressed' | 'suite_slowed'
+  | 'flaky_detected'
+  | 'quarantine_changed'
+  | 'rca_ready'
+  | 'suite_regressed'
+  | 'suite_slowed'
+  | 'ai_budget_spent'
 
 export const NOTIFICATION_TYPES: readonly NotificationType[] = [
   'flaky_detected',
   'quarantine_changed',
   'rca_ready',
   'suite_regressed',
   'suite_slowed',
+  'ai_budget_spent',
 ]
 
 export interface NotificationField {