feat(grafana): chart the tracker's event-loop delay

CHANGELOG.md

@@ -6,6 +6,8 @@ All notable changes to OpenStreamGrid are documented in this file.
 
 ### Added
 
+- A Grafana panel for the tracker's event-loop delay (p50, p99, max), the
+  signal that `docs/tracker-capacity.md` sizes trackers by.
 - `LOG_LEVEL` (`info`, `warn`, or `error`) for every service, to drop log
   entries below that level.
 - Optional Helm `PrometheusRule` (`tracker.metrics.prometheusRule`) that alerts

deploy/k8s/README.md

@@ -172,7 +172,8 @@ into Grafana and pick a Prometheus data source that scrapes the tracker.
 The dashboard covers active broadcasts and peers, P2P offload (P2P successes
 divided by P2P successes plus origin requests), P2P success rate, fallbacks,
 integrity failures, REST latency quantiles, peer churn, rate limiting,
-WebSocket signaling (subscribers by delivery mode, messages and bytes sent),
+WebSocket signaling (subscribers by delivery mode, messages and bytes sent,
+and the tracker's event-loop delay),
 and security (credential refusals, refused WebSocket upgrades, and REST
 responses by status class).
 

helm/openstreamgrid/dashboards/openstreamgrid.json

@@ -709,7 +709,7 @@
       "gridPos": {
         "x": 0,
         "y": 24,
-        "w": 12,
+        "w": 8,
         "h": 8
       },
       "targets": [
@@ -759,9 +759,9 @@
       "title": "WebSocket signaling traffic",
       "description": "Messages and payload bytes the tracker sends to peers.",
       "gridPos": {
-        "x": 12,
+        "x": 8,
         "y": 24,
-        "w": 12,
+        "w": 8,
         "h": 8
       },
       "targets": [
@@ -833,6 +833,78 @@
         "uid": "${datasource}"
       }
     },
+    {
+      "type": "timeseries",
+      "title": "Tracker event-loop delay",
+      "description": "How long callbacks waited for the tracker event loop since the previous scrape; every REST and WebSocket message pays it. See docs/tracker-capacity.md.",
+      "gridPos": {
+        "x": 16,
+        "y": 24,
+        "w": 8,
+        "h": 8
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "max(openstreamgrid_event_loop_delay_p50_seconds{job=~\"$job\"})",
+          "legendFormat": "p50",
+          "refId": "A",
+          "range": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "max(openstreamgrid_event_loop_delay_p99_seconds{job=~\"$job\"})",
+          "legendFormat": "p99",
+          "refId": "B",
+          "range": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "max(openstreamgrid_event_loop_delay_max_seconds{job=~\"$job\"})",
+          "legendFormat": "max",
+          "refId": "C",
+          "range": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 2,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "id": 19,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      }
+    },
     {
       "type": "row",
       "title": "Security",