From 57543f4034d7c0fd85f12cde0d18b45aa9b243a9 Mon Sep 17 00:00:00 2001
From: valentinpanizza <valentinpanizza21@gmail.com>
Date: Tue, 22 Sep 2026 19:17:03 -0300
Subject: [PATCH] fix(health): report degraded when the queue is backed up with
 nothing active
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A fresh heartbeat does not mean the worker is still consuming. The heartbeat
runs on its own interval in worker/dm-worker.ts, independent of the BullMQ
Worker, so the consumer can stop taking jobs — dropped queue connection,
crashed consumer loop — while the process and its heartbeat stay alive.

/api/health then keeps answering 200 while the backlog grows. On my instance
385 jobs sat waiting for hours behind an uptime monitor that never alerted;
every one was someone who had commented and was waiting for their link. All
three docs that mention the endpoint point people at it to confirm the worker
is healthy, and it reported healthy throughout.

checkQueue() already fetched the counts that reveal this and used them only as
reporting. Use them: a backlog with nothing in flight is unambiguous, since a
healthy worker with concurrency 5 never leaves jobs waiting while active is 0.
HEALTH_STUCK_QUEUE_WAITING (default 25) only exists to ride out the moment
between enqueue and pickup.
---
 app/api/health/route.ts | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/app/api/health/route.ts b/app/api/health/route.ts
index 29ee066c3..9f37ab811 100644
--- a/app/api/health/route.ts
+++ b/app/api/health/route.ts
@@ -39,6 +39,21 @@ async function checkRedis(): Promise<HealthCheck> {
   }
 }
 
+// A fresh heartbeat does not mean the worker is still doing anything. The
+// heartbeat runs on its own interval, so BullMQ's consumer can stop taking jobs
+// — a dropped queue connection, a crashed consumer loop — while the process,
+// and its heartbeat, stay perfectly alive. Health then keeps answering 200
+// while the backlog grows and nobody is served. Seen in production: 385 jobs
+// waiting for hours behind an uptime monitor that never once alerted.
+//
+// A backlog with nothing in flight is the signal, and it is unambiguous: a
+// healthy worker with a concurrency of 5 never leaves jobs waiting with zero
+// active. The threshold only exists to ride out the moment between a job being
+// enqueued and the worker picking it up.
+const STUCK_QUEUE_MIN_WAITING = Number(
+  process.env.HEALTH_STUCK_QUEUE_WAITING ?? 25
+);
+
 async function checkQueue(): Promise<HealthCheck & { counts?: unknown }> {
   try {
     const counts = await getDMQueue().getJobCounts(
@@ -47,6 +62,15 @@ async function checkQueue(): Promise<HealthCheck & { counts?: unknown }> {
       "delayed",
       "failed"
     );
+    const waiting = counts.waiting ?? 0;
+    const active = counts.active ?? 0;
+    if (waiting >= STUCK_QUEUE_MIN_WAITING && active === 0) {
+      return {
+        status: "error",
+        detail: `${waiting} jobs waiting with none active — the worker is not consuming`,
+        counts,
+      };
+    }
     return { status: "ok", counts };
   } catch (error) {
     return {