diff --git a/app/api/health/route.ts b/app/api/health/route.ts index 29ee066c3..9f37ab811 100644 --- a/app/api/health/route.ts +++ b/app/api/health/route.ts @@ -39,6 +39,21 @@ async function checkRedis(): Promise { } } +// A fresh heartbeat does not mean the worker is still doing anything. The +// heartbeat runs on its own interval, so BullMQ's consumer can stop taking jobs +// — a dropped queue connection, a crashed consumer loop — while the process, +// and its heartbeat, stay perfectly alive. Health then keeps answering 200 +// while the backlog grows and nobody is served. Seen in production: 385 jobs +// waiting for hours behind an uptime monitor that never once alerted. +// +// A backlog with nothing in flight is the signal, and it is unambiguous: a +// healthy worker with a concurrency of 5 never leaves jobs waiting with zero +// active. The threshold only exists to ride out the moment between a job being +// enqueued and the worker picking it up. +const STUCK_QUEUE_MIN_WAITING = Number( + process.env.HEALTH_STUCK_QUEUE_WAITING ?? 25 +); + async function checkQueue(): Promise { try { const counts = await getDMQueue().getJobCounts( @@ -47,6 +62,15 @@ async function checkQueue(): Promise { "delayed", "failed" ); + const waiting = counts.waiting ?? 0; + const active = counts.active ?? 0; + if (waiting >= STUCK_QUEUE_MIN_WAITING && active === 0) { + return { + status: "error", + detail: `${waiting} jobs waiting with none active — the worker is not consuming`, + counts, + }; + } return { status: "ok", counts }; } catch (error) { return {