Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions app/api/health/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,21 @@ async function checkRedis(): Promise<HealthCheck> {
}
}

// A fresh heartbeat does not mean the worker is still doing anything. The
// heartbeat runs on its own interval, so BullMQ's consumer can stop taking jobs
// — a dropped queue connection, a crashed consumer loop — while the process,
// and its heartbeat, stay perfectly alive. Health then keeps answering 200
// while the backlog grows and nobody is served. Seen in production: 385 jobs
// waiting for hours behind an uptime monitor that never once alerted.
//
// A backlog with nothing in flight is the signal, and it is unambiguous: a
// healthy worker with a concurrency of 5 never leaves jobs waiting with zero
// active. The threshold only exists to ride out the moment between a job being
// enqueued and the worker picking it up.
const STUCK_QUEUE_MIN_WAITING = Number(
process.env.HEALTH_STUCK_QUEUE_WAITING ?? 25
);

async function checkQueue(): Promise<HealthCheck & { counts?: unknown }> {
try {
const counts = await getDMQueue().getJobCounts(
Expand All @@ -47,6 +62,15 @@ async function checkQueue(): Promise<HealthCheck & { counts?: unknown }> {
"delayed",
"failed"
);
const waiting = counts.waiting ?? 0;
const active = counts.active ?? 0;
if (waiting >= STUCK_QUEUE_MIN_WAITING && active === 0) {
return {
status: "error",
detail: `${waiting} jobs waiting with none active — the worker is not consuming`,
counts,
};
}
return { status: "ok", counts };
} catch (error) {
return {
Expand Down