From a03ac7b5c0c80ce8dd464ba5a499eeda837c418a Mon Sep 17 00:00:00 2001 From: Andrii Kohut Date: Tue, 18 Aug 2026 15:25:35 +0200 Subject: [PATCH] feat: say when the AI budget runs out, on email or any other channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The budget stops root-cause analysis and recorded that in a metric counter. On a self-hosted instance nobody is scraping it, so explanations simply stopped appearing at some point in the afternoon and nothing anywhere said why. It now goes through the notification path that already exists, so email, Slack, Discord and signed webhooks all carry it and it appears as a checkbox beside every other event. Deduplicated per project per day. The budget is re-checked on every run, so a busy afternoon would otherwise send one of these per failing suite. The message says analysis is paused until tomorrow rather than only reporting numbers — someone reading it in a mail client has no dashboard open, and "budget exceeded" alone does not tell them anything stopped. My first test for this proved nothing: it fired the event on the bus and watched a channel receive it, so deleting the emit inside RCA left it passing. Found by deleting the emit. The real test drives processFailures until the budget is exhausted, and fails when the emit is removed. --- .changeset/budget-alert.md | 5 ++ apps/web/src/lib/notify-events.ts | 1 + .../worker/src/__tests__/budget-alert.test.ts | 42 +++++++++++++++ apps/worker/src/__tests__/rca.test.ts | 46 ++++++++++++++++ apps/worker/src/events.ts | 5 ++ apps/worker/src/notify.ts | 16 ++++++ apps/worker/src/rca.ts | 4 ++ docs/configuration.md | 4 ++ packages/notify/src/__tests__/budget.test.ts | 54 +++++++++++++++++++ packages/notify/src/discord.ts | 1 + packages/notify/src/message.ts | 8 ++- 11 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 .changeset/budget-alert.md create mode 100644 apps/worker/src/__tests__/budget-alert.test.ts create mode 100644 packages/notify/src/__tests__/budget.test.ts diff --git a/.changeset/budget-alert.md b/.changeset/budget-alert.md new file mode 100644 index 0000000..83ed887 --- /dev/null +++ b/.changeset/budget-alert.md @@ -0,0 +1,5 @@ +--- +'@flakemetry/notify': minor +--- + +New `ai_budget_spent` notification, sent when a project's daily LLM token budget runs out and root-cause analysis pauses. Subscribe on any channel — Slack, Discord, email or a signed webhook. Deduplicated per project per day, since the budget is re-checked on every run. diff --git a/apps/web/src/lib/notify-events.ts b/apps/web/src/lib/notify-events.ts index 7afe1af..47236be 100644 --- a/apps/web/src/lib/notify-events.ts +++ b/apps/web/src/lib/notify-events.ts @@ -4,4 +4,5 @@ export const NOTIFY_EVENTS = [ 'rca_ready', 'suite_regressed', 'suite_slowed', + 'ai_budget_spent', ] diff --git a/apps/worker/src/__tests__/budget-alert.test.ts b/apps/worker/src/__tests__/budget-alert.test.ts new file mode 100644 index 0000000..e048b2a --- /dev/null +++ b/apps/worker/src/__tests__/budget-alert.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from 'vitest' + +import { createEventBus } from '../events' +import { startNotifications } from '../notify' + +/** + * The budget alert is only useful if it travels: the worker emits it deep inside RCA, and + * three separate pieces have to agree — the event bus name, the notification type, and the + * channel's subscription list. Testing the formatter alone would pass with any of them + * broken. + */ +describe('the AI budget alert reaches a channel', () => { + it('dispatches to a subscribed channel when the budget is spent', async () => { + const delivered: string[] = [] + const events = createEventBus(() => undefined) + + const enabled = startNotifications( + events, + { + FLAKEMETRY_SLACK_WEBHOOK: 'https://hooks.slack.com/services/probe', + FLAKEMETRY_NOTIFY_EVENTS: 'ai_budget_spent', + }, + () => Promise.resolve([]), + ) + expect(enabled, 'notifications did not start — the rest proves nothing').toBe(true) + + const originalFetch = globalThis.fetch + globalThis.fetch = (async (_url: string, init: RequestInit) => { + delivered.push(String(init.body)) + return new Response('ok', { status: 200 }) + }) as unknown as typeof fetch + + try { + events.emit('ai.budget.spent', { projectId: 'p1', spent: 200_000, budget: 200_000 }) + await vi.waitFor(() => expect(delivered.length).toBeGreaterThan(0), { timeout: 5000 }) + } finally { + globalThis.fetch = originalFetch + } + + expect(delivered.join()).toContain('paused until tomorrow') + }) +}) diff --git a/apps/worker/src/__tests__/rca.test.ts b/apps/worker/src/__tests__/rca.test.ts index 022d514..08769ff 100644 --- a/apps/worker/src/__tests__/rca.test.ts +++ b/apps/worker/src/__tests__/rca.test.ts @@ -158,6 +158,52 @@ describe.skipIf(!hasDb)('processFailures', () => { expect(seen).toHaveLength(1) }) + it('announces that the budget is spent rather than only counting it', async () => { + const ctx = await seed() + const events = createEventBus() + const announced: DomainEventMap['ai.budget.spent'][] = [] + events.on('ai.budget.spent', (payload) => announced.push(payload)) + + // A budget of one token is already spent by the first report, so the second failure + // hits the ceiling. + await processFailures( + prisma, + { + orgId: ctx.orgId, + projectId: ctx.projectId, + now: NOW, + provider: fakeProvider(), + aiEnabled: true, + dailyTokenBudget: 1, + events, + }, + [failure(ctx.executionId)], + ) + await processFailures( + prisma, + { + orgId: ctx.orgId, + projectId: ctx.projectId, + now: NOW, + provider: fakeProvider(), + aiEnabled: true, + dailyTokenBudget: 1, + events, + }, + [ + failure( + await seedExecution(ctx, 'AssertionError: expected cart to contain 3 items'), + 'AssertionError: expected cart to contain 3 items', + ), + ], + ) + + // Without this the only trace is a metric counter, and a self-hosted instance whose + // explanations stopped at noon has nothing to tell anyone why. + expect(announced).toHaveLength(1) + expect(announced[0]).toMatchObject({ projectId: ctx.projectId, budget: 1 }) + }) + it('does not call the model when AI is disabled', async () => { const ctx = await seed() const provider = fakeProvider() diff --git a/apps/worker/src/events.ts b/apps/worker/src/events.ts index 38c40dc..2c9c110 100644 --- a/apps/worker/src/events.ts +++ b/apps/worker/src/events.ts @@ -53,6 +53,11 @@ export interface DomainEventMap { baselineFailRate: number total: number } + 'ai.budget.spent': { + projectId: string + spent: number + budget: number + } 'suite.slowed': { projectId: string suite: string diff --git a/apps/worker/src/notify.ts b/apps/worker/src/notify.ts index a81a7f9..81aff2c 100644 --- a/apps/worker/src/notify.ts +++ b/apps/worker/src/notify.ts @@ -83,6 +83,22 @@ export const startNotifications = ( }) }) + events.on('ai.budget.spent', (payload) => { + void dispatcher.dispatch({ + type: 'ai_budget_spent', + projectId: payload.projectId, + heading: 'AI budget spent for today', + summary: `Root-cause analysis is paused until tomorrow — ${payload.spent.toLocaleString()} of ${payload.budget.toLocaleString()} tokens used today.`, + fields: [ + { label: 'Spent', value: payload.spent.toLocaleString() }, + { label: 'Budget', value: payload.budget.toLocaleString() }, + ], + // Once per project per day. The budget is checked on every run, and a suite that + // keeps failing would otherwise send one of these for each of them. + dedupeKey: `ai_budget_spent:${payload.projectId}:${new Date().toISOString().slice(0, 10)}`, + }) + }) + events.on('quarantine.changed', (payload) => { void dispatcher.dispatch({ type: 'quarantine_changed', diff --git a/apps/worker/src/rca.ts b/apps/worker/src/rca.ts index a6067a1..33138a3 100644 --- a/apps/worker/src/rca.ts +++ b/apps/worker/src/rca.ts @@ -218,6 +218,10 @@ export const processFailures = async ( if (!group.isNew) continue if (spent >= budget) { workerMetrics.rcaBudgetExhausted.add(1) + // Said out loud, not only counted. A metric nobody self-hosting is scraping is the + // difference between "analysis is off today" and nobody ever finding out why the + // explanations stopped. + ctx.events?.emit('ai.budget.spent', { projectId: ctx.projectId, spent, budget }) break } diff --git a/docs/configuration.md b/docs/configuration.md index 12ba1e8..b59e352 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -139,6 +139,10 @@ without a provider gets you nothing — the worker has nothing to ask. Spend is bounded by `ai.dailyTokenBudget` per project, and only genuinely new error signatures reach the model at all; the rest are answered from the cluster's cached analysis. +When the budget runs out, the worker emits `ai_budget_spent` — subscribe to it on any +notification channel, including email, and the day analysis stops is a day someone hears +about. It is deduplicated per project per day, since the budget is re-checked on every run. + ### Notifications The worker pushes intelligence to Slack, Discord and email. Webhook delivery is best-effort and de-duplicated per channel so a flapping test can't spam a channel. Channels come from two places, applied together: **global env channels** (below) and **per-project channels** configured in **Settings → Notifications** (add a Slack/Discord webhook or an email address with an event filter). Events: `flaky_detected`, `quarantine_changed`, `rca_ready`, `suite_regressed` (a suite's fail-rate crossing its trailing baseline), and `suite_slowed` (a suite's average duration rising well above its trailing baseline). diff --git a/packages/notify/src/__tests__/budget.test.ts b/packages/notify/src/__tests__/budget.test.ts new file mode 100644 index 0000000..3d86761 --- /dev/null +++ b/packages/notify/src/__tests__/budget.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest' + +import { formatDiscord } from '../discord' +import { formatEmail } from '../email' +import { NOTIFICATION_TYPES, type NotificationEvent } from '../message' +import { formatSlack } from '../slack' + +const event: NotificationEvent = { + type: 'ai_budget_spent', + projectId: 'p1', + heading: 'AI budget spent for today', + summary: 'Root-cause analysis is paused until tomorrow — 200,000 of 200,000 tokens used today.', + fields: [ + { label: 'Spent', value: '200,000' }, + { label: 'Budget', value: '200,000' }, + ], + dedupeKey: 'ai_budget_spent:p1:2026-08-18', +} + +describe('the budget notification', () => { + it('says what happened and what it means, not just a number', () => { + const mail = formatEmail(event) + + // Someone reading this in a mail client has no dashboard open. "Budget exceeded" alone + // does not tell them analysis has stopped. + expect(mail.subject).toContain('AI budget spent') + expect(mail.text).toContain('paused until tomorrow') + expect(mail.text).toContain('Budget: 200,000') + }) + + it('renders on every channel, not only the one it was written for', () => { + expect(() => formatSlack(event)).not.toThrow() + expect(() => formatDiscord(event)).not.toThrow() + expect(formatDiscord(event).embeds).toHaveLength(1) + }) + + it('is a type the routing layer knows about', () => { + // A type the dashboard cannot offer as a checkbox is one nobody can subscribe to. + expect(NOTIFICATION_TYPES).toContain('ai_budget_spent') + }) + + it('dedupes per project per day', () => { + // The budget is re-checked on every run, so without a key that collapses them a busy + // afternoon sends one of these per failing suite. + const monday = 'ai_budget_spent:p1:2026-08-18' + const alsoMonday = 'ai_budget_spent:p1:2026-08-18' + const tuesday = 'ai_budget_spent:p1:2026-08-19' + const otherProject = 'ai_budget_spent:p2:2026-08-18' + + expect(monday).toBe(alsoMonday) + expect(monday).not.toBe(tuesday) + expect(monday).not.toBe(otherProject) + }) +}) diff --git a/packages/notify/src/discord.ts b/packages/notify/src/discord.ts index 61e4a37..95590fa 100644 --- a/packages/notify/src/discord.ts +++ b/packages/notify/src/discord.ts @@ -11,6 +11,7 @@ const COLOR: Record = { rca_ready: 0x5319e7, suite_regressed: 0xdc2626, suite_slowed: 0xd97706, + ai_budget_spent: 0x6b7280, } export const formatDiscord = (event: NotificationEvent): DiscordPayload => ({ diff --git a/packages/notify/src/message.ts b/packages/notify/src/message.ts index 8802fee..e46ae59 100644 --- a/packages/notify/src/message.ts +++ b/packages/notify/src/message.ts @@ -1,5 +1,10 @@ export type NotificationType = - 'flaky_detected' | 'quarantine_changed' | 'rca_ready' | 'suite_regressed' | 'suite_slowed' + | 'flaky_detected' + | 'quarantine_changed' + | 'rca_ready' + | 'suite_regressed' + | 'suite_slowed' + | 'ai_budget_spent' export const NOTIFICATION_TYPES: readonly NotificationType[] = [ 'flaky_detected', @@ -7,6 +12,7 @@ export const NOTIFICATION_TYPES: readonly NotificationType[] = [ 'rca_ready', 'suite_regressed', 'suite_slowed', + 'ai_budget_spent', ] export interface NotificationField {