From eee3244e6a9211c485900b8ce3ea218416a56456 Mon Sep 17 00:00:00 2001
From: Mark McKee <1002775+mark-mckee@users.noreply.github.com>
Date: Sat, 9 May 2026 11:19:34 +0100
Subject: [PATCH] feat(monitors): aggregate service status across active
 monitors

Prior behavior: the most recent monitor check unconditionally overwrote the
linked service's status. With multiple monitors per service this caused
recovery in one monitor to mask real outages in another.

New behavior: when applying a status change, compute the worst (per
_STATUS_PRIORITY) across all active monitors on the service, treating the
just-completed monitor's candidate as its current contribution. Service
status is updated only when the aggregated state actually changes.

Preserves:
  - Single-monitor services: identical behavior.
  - Maintenance mode: still wins, untouched.
  - API-set status (token_auth, admin): untouched, bypasses aggregation.
---
 backend/app/tasks/monitors.py | 55 ++++++++++++++++++++++++++---------
 1 file changed, 41 insertions(+), 14 deletions(-)

diff --git a/backend/app/tasks/monitors.py b/backend/app/tasks/monitors.py
index 304bb90..694b271 100644
--- a/backend/app/tasks/monitors.py
+++ b/backend/app/tasks/monitors.py
@@ -38,13 +38,26 @@
 }
 
 
-def _worst(s1, s2):
-    """Return whichever of the two status strings has higher severity."""
-    if s1 is None:
-        return s2
-    if s2 is None:
-        return s1
-    return s1 if _STATUS_PRIORITY.get(s1, 0) >= _STATUS_PRIORITY.get(s2, 0) else s2
+def _compute_service_status(service, source_monitor=None, source_status=None):
+    """Compute the rolled-up service status across all active monitors.
+
+    For each active monitor on the service, take the worst status (per
+    _STATUS_PRIORITY). When `source_monitor` is provided, that monitor
+    contributes `source_status` rather than its stored last_status —
+    useful from inside run_single_monitor where the source monitor's
+    last_status has been updated in-memory but not yet persisted.
+
+    Monitors with no last_status yet (never run) are ignored.
+    """
+    aggregated = source_status
+    for m in Monitor.objects(service=service, active=True):
+        if source_monitor is not None and m.id == source_monitor.id:
+            contribution = source_status
+        else:
+            contribution = m.last_status
+        if contribution:
+            aggregated = _worst(aggregated, contribution)
+    return aggregated
 
 
 def _resolve_response_time(thresholds, response_ms, failure_status):
@@ -383,10 +396,17 @@ def run_single_monitor(monitor_id: str):
             monitor.pending_status = None
             monitor.pending_since = None
         elif monitor.confirm_seconds == 0:
-            # Immediate mode: apply right away.
-            action = _apply_service_status(
-                service, candidate, monitor.name, result
-            )
+            # Immediate mode: aggregate across all active monitors before applying.
+            aggregated = _compute_service_status(service, monitor, candidate)
+            if aggregated != current:
+                action = _apply_service_status(
+                    service, aggregated, monitor.name, result
+                )
+            else:
+                action = (
+                    f"{monitor.name}: {candidate} "
+                    f"(service unchanged, aggregated={aggregated})"
+                )
             monitor.pending_status = None
             monitor.pending_since = None
         else:
@@ -403,9 +423,16 @@ def run_single_monitor(monitor_id: str):
                 # Same candidate — check if the window has elapsed.
                 elapsed = (now - monitor.pending_since).total_seconds()
                 if elapsed >= monitor.confirm_seconds:
-                    action = _apply_service_status(
-                        service, candidate, monitor.name, result
-                    )
+                    aggregated = _compute_service_status(service, monitor, candidate)
+                    if aggregated != current:
+                        action = _apply_service_status(
+                            service, aggregated, monitor.name, result
+                        )
+                    else:
+                        action = (
+                            f"{monitor.name}: {candidate} confirmed "
+                            f"(service unchanged, aggregated={aggregated})"
+                        )
                     monitor.pending_status = None
                     monitor.pending_since = None
                 else: