diff --git a/.changeset/budget-alert.md b/.changeset/budget-alert.md new file mode 100644 index 0000000..83ed887 --- /dev/null +++ b/.changeset/budget-alert.md @@ -0,0 +1,5 @@ +--- +'@flakemetry/notify': minor +--- + +New `ai_budget_spent` notification, sent when a project's daily LLM token budget runs out and root-cause analysis pauses. Subscribe on any channel — Slack, Discord, email or a signed webhook. Deduplicated per project per day, since the budget is re-checked on every run. diff --git a/apps/web/src/lib/notify-events.ts b/apps/web/src/lib/notify-events.ts index 7afe1af..47236be 100644 --- a/apps/web/src/lib/notify-events.ts +++ b/apps/web/src/lib/notify-events.ts @@ -4,4 +4,5 @@ export const NOTIFY_EVENTS = [ 'rca_ready', 'suite_regressed', 'suite_slowed', + 'ai_budget_spent', ] diff --git a/apps/worker/src/__tests__/budget-alert.test.ts b/apps/worker/src/__tests__/budget-alert.test.ts new file mode 100644 index 0000000..e048b2a --- /dev/null +++ b/apps/worker/src/__tests__/budget-alert.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from 'vitest' + +import { createEventBus } from '../events' +import { startNotifications } from '../notify' + +/** + * The budget alert is only useful if it travels: the worker emits it deep inside RCA, and + * three separate pieces have to agree — the event bus name, the notification type, and the + * channel's subscription list. Testing the formatter alone would pass with any of them + * broken. + */ +describe('the AI budget alert reaches a channel', () => { + it('dispatches to a subscribed channel when the budget is spent', async () => { + const delivered: string[] = [] + const events = createEventBus(() => undefined) + + const enabled = startNotifications( + events, + { + FLAKEMETRY_SLACK_WEBHOOK: 'https://hooks.slack.com/services/probe', + FLAKEMETRY_NOTIFY_EVENTS: 'ai_budget_spent', + }, + () => Promise.resolve([]), + ) + expect(enabled, 'notifications did not start — the rest proves nothing').toBe(true) + + const originalFetch = globalThis.fetch + globalThis.fetch = (async (_url: string, init: RequestInit) => { + delivered.push(String(init.body)) + return new Response('ok', { status: 200 }) + }) as unknown as typeof fetch + + try { + events.emit('ai.budget.spent', { projectId: 'p1', spent: 200_000, budget: 200_000 }) + await vi.waitFor(() => expect(delivered.length).toBeGreaterThan(0), { timeout: 5000 }) + } finally { + globalThis.fetch = originalFetch + } + + expect(delivered.join()).toContain('paused until tomorrow') + }) +}) diff --git a/apps/worker/src/__tests__/rca.test.ts b/apps/worker/src/__tests__/rca.test.ts index 022d514..08769ff 100644 --- a/apps/worker/src/__tests__/rca.test.ts +++ b/apps/worker/src/__tests__/rca.test.ts @@ -158,6 +158,52 @@ describe.skipIf(!hasDb)('processFailures', () => { expect(seen).toHaveLength(1) }) + it('announces that the budget is spent rather than only counting it', async () => { + const ctx = await seed() + const events = createEventBus() + const announced: DomainEventMap['ai.budget.spent'][] = [] + events.on('ai.budget.spent', (payload) => announced.push(payload)) + + // A budget of one token is already spent by the first report, so the second failure + // hits the ceiling. + await processFailures( + prisma, + { + orgId: ctx.orgId, + projectId: ctx.projectId, + now: NOW, + provider: fakeProvider(), + aiEnabled: true, + dailyTokenBudget: 1, + events, + }, + [failure(ctx.executionId)], + ) + await processFailures( + prisma, + { + orgId: ctx.orgId, + projectId: ctx.projectId, + now: NOW, + provider: fakeProvider(), + aiEnabled: true, + dailyTokenBudget: 1, + events, + }, + [ + failure( + await seedExecution(ctx, 'AssertionError: expected cart to contain 3 items'), + 'AssertionError: expected cart to contain 3 items', + ), + ], + ) + + // Without this the only trace is a metric counter, and a self-hosted instance whose + // explanations stopped at noon has nothing to tell anyone why. + expect(announced).toHaveLength(1) + expect(announced[0]).toMatchObject({ projectId: ctx.projectId, budget: 1 }) + }) + it('does not call the model when AI is disabled', async () => { const ctx = await seed() const provider = fakeProvider() diff --git a/apps/worker/src/events.ts b/apps/worker/src/events.ts index 38c40dc..2c9c110 100644 --- a/apps/worker/src/events.ts +++ b/apps/worker/src/events.ts @@ -53,6 +53,11 @@ export interface DomainEventMap { baselineFailRate: number total: number } + 'ai.budget.spent': { + projectId: string + spent: number + budget: number + } 'suite.slowed': { projectId: string suite: string diff --git a/apps/worker/src/notify.ts b/apps/worker/src/notify.ts index a81a7f9..81aff2c 100644 --- a/apps/worker/src/notify.ts +++ b/apps/worker/src/notify.ts @@ -83,6 +83,22 @@ export const startNotifications = ( }) }) + events.on('ai.budget.spent', (payload) => { + void dispatcher.dispatch({ + type: 'ai_budget_spent', + projectId: payload.projectId, + heading: 'AI budget spent for today', + summary: `Root-cause analysis is paused until tomorrow — ${payload.spent.toLocaleString()} of ${payload.budget.toLocaleString()} tokens used today.`, + fields: [ + { label: 'Spent', value: payload.spent.toLocaleString() }, + { label: 'Budget', value: payload.budget.toLocaleString() }, + ], + // Once per project per day. The budget is checked on every run, and a suite that + // keeps failing would otherwise send one of these for each of them. + dedupeKey: `ai_budget_spent:${payload.projectId}:${new Date().toISOString().slice(0, 10)}`, + }) + }) + events.on('quarantine.changed', (payload) => { void dispatcher.dispatch({ type: 'quarantine_changed', diff --git a/apps/worker/src/rca.ts b/apps/worker/src/rca.ts index a6067a1..33138a3 100644 --- a/apps/worker/src/rca.ts +++ b/apps/worker/src/rca.ts @@ -218,6 +218,10 @@ export const processFailures = async ( if (!group.isNew) continue if (spent >= budget) { workerMetrics.rcaBudgetExhausted.add(1) + // Said out loud, not only counted. A metric nobody self-hosting is scraping is the + // difference between "analysis is off today" and nobody ever finding out why the + // explanations stopped. + ctx.events?.emit('ai.budget.spent', { projectId: ctx.projectId, spent, budget }) break } diff --git a/docs/configuration.md b/docs/configuration.md index 12ba1e8..b59e352 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -139,6 +139,10 @@ without a provider gets you nothing — the worker has nothing to ask. Spend is bounded by `ai.dailyTokenBudget` per project, and only genuinely new error signatures reach the model at all; the rest are answered from the cluster's cached analysis. +When the budget runs out, the worker emits `ai_budget_spent` — subscribe to it on any +notification channel, including email, and the day analysis stops is a day someone hears +about. It is deduplicated per project per day, since the budget is re-checked on every run. + ### Notifications The worker pushes intelligence to Slack, Discord and email. Webhook delivery is best-effort and de-duplicated per channel so a flapping test can't spam a channel. Channels come from two places, applied together: **global env channels** (below) and **per-project channels** configured in **Settings → Notifications** (add a Slack/Discord webhook or an email address with an event filter). Events: `flaky_detected`, `quarantine_changed`, `rca_ready`, `suite_regressed` (a suite's fail-rate crossing its trailing baseline), and `suite_slowed` (a suite's average duration rising well above its trailing baseline). diff --git a/packages/notify/src/__tests__/budget.test.ts b/packages/notify/src/__tests__/budget.test.ts new file mode 100644 index 0000000..3d86761 --- /dev/null +++ b/packages/notify/src/__tests__/budget.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest' + +import { formatDiscord } from '../discord' +import { formatEmail } from '../email' +import { NOTIFICATION_TYPES, type NotificationEvent } from '../message' +import { formatSlack } from '../slack' + +const event: NotificationEvent = { + type: 'ai_budget_spent', + projectId: 'p1', + heading: 'AI budget spent for today', + summary: 'Root-cause analysis is paused until tomorrow — 200,000 of 200,000 tokens used today.', + fields: [ + { label: 'Spent', value: '200,000' }, + { label: 'Budget', value: '200,000' }, + ], + dedupeKey: 'ai_budget_spent:p1:2026-08-18', +} + +describe('the budget notification', () => { + it('says what happened and what it means, not just a number', () => { + const mail = formatEmail(event) + + // Someone reading this in a mail client has no dashboard open. "Budget exceeded" alone + // does not tell them analysis has stopped. + expect(mail.subject).toContain('AI budget spent') + expect(mail.text).toContain('paused until tomorrow') + expect(mail.text).toContain('Budget: 200,000') + }) + + it('renders on every channel, not only the one it was written for', () => { + expect(() => formatSlack(event)).not.toThrow() + expect(() => formatDiscord(event)).not.toThrow() + expect(formatDiscord(event).embeds).toHaveLength(1) + }) + + it('is a type the routing layer knows about', () => { + // A type the dashboard cannot offer as a checkbox is one nobody can subscribe to. + expect(NOTIFICATION_TYPES).toContain('ai_budget_spent') + }) + + it('dedupes per project per day', () => { + // The budget is re-checked on every run, so without a key that collapses them a busy + // afternoon sends one of these per failing suite. + const monday = 'ai_budget_spent:p1:2026-08-18' + const alsoMonday = 'ai_budget_spent:p1:2026-08-18' + const tuesday = 'ai_budget_spent:p1:2026-08-19' + const otherProject = 'ai_budget_spent:p2:2026-08-18' + + expect(monday).toBe(alsoMonday) + expect(monday).not.toBe(tuesday) + expect(monday).not.toBe(otherProject) + }) +}) diff --git a/packages/notify/src/discord.ts b/packages/notify/src/discord.ts index 61e4a37..95590fa 100644 --- a/packages/notify/src/discord.ts +++ b/packages/notify/src/discord.ts @@ -11,6 +11,7 @@ const COLOR: Record = { rca_ready: 0x5319e7, suite_regressed: 0xdc2626, suite_slowed: 0xd97706, + ai_budget_spent: 0x6b7280, } export const formatDiscord = (event: NotificationEvent): DiscordPayload => ({ diff --git a/packages/notify/src/message.ts b/packages/notify/src/message.ts index 8802fee..e46ae59 100644 --- a/packages/notify/src/message.ts +++ b/packages/notify/src/message.ts @@ -1,5 +1,10 @@ export type NotificationType = - 'flaky_detected' | 'quarantine_changed' | 'rca_ready' | 'suite_regressed' | 'suite_slowed' + | 'flaky_detected' + | 'quarantine_changed' + | 'rca_ready' + | 'suite_regressed' + | 'suite_slowed' + | 'ai_budget_spent' export const NOTIFICATION_TYPES: readonly NotificationType[] = [ 'flaky_detected', @@ -7,6 +12,7 @@ export const NOTIFICATION_TYPES: readonly NotificationType[] = [ 'rca_ready', 'suite_regressed', 'suite_slowed', + 'ai_budget_spent', ] export interface NotificationField {