From 36abfb2805c5771c894ab2b1a30632a9488c3a63 Mon Sep 17 00:00:00 2001 From: JEONG KITAE Date: Tue, 20 Oct 2026 09:55:04 +0900 Subject: [PATCH] feat(grafana): chart the tracker's event-loop delay The event-loop delay gauges were exported and used by the capacity guide and the EventLoopLag alert, but the dashboard did not show them. The Signaling row now has three panels: subscribers, signaling traffic, and event-loop delay (p50, p99, max, in seconds). --- CHANGELOG.md | 2 + deploy/k8s/README.md | 3 +- .../dashboards/openstreamgrid.json | 78 ++++++++++++++++++- 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c3c69c..a3971e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to OpenStreamGrid are documented in this file. ### Added +- A Grafana panel for the tracker's event-loop delay (p50, p99, max), the + signal that `docs/tracker-capacity.md` sizes trackers by. - Optional Helm `PrometheusRule` (`tracker.metrics.prometheusRule`) that alerts when no tracker is up, peers keep reporting integrity failures, most P2P attempts fall back to the origin, credential refusals spike, or the event diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index bf871ce..3c72766 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -172,7 +172,8 @@ into Grafana and pick a Prometheus data source that scrapes the tracker. The dashboard covers active broadcasts and peers, P2P offload (P2P successes divided by P2P successes plus origin requests), P2P success rate, fallbacks, integrity failures, REST latency quantiles, peer churn, rate limiting, -WebSocket signaling (subscribers by delivery mode, messages and bytes sent), +WebSocket signaling (subscribers by delivery mode, messages and bytes sent, +and the tracker's event-loop delay), and security (credential refusals, refused WebSocket upgrades, and REST responses by status class). diff --git a/helm/openstreamgrid/dashboards/openstreamgrid.json b/helm/openstreamgrid/dashboards/openstreamgrid.json index 2d20fce..5bb06ca 100644 --- a/helm/openstreamgrid/dashboards/openstreamgrid.json +++ b/helm/openstreamgrid/dashboards/openstreamgrid.json @@ -709,7 +709,7 @@ "gridPos": { "x": 0, "y": 24, - "w": 12, + "w": 8, "h": 8 }, "targets": [ @@ -759,9 +759,9 @@ "title": "WebSocket signaling traffic", "description": "Messages and payload bytes the tracker sends to peers.", "gridPos": { - "x": 12, + "x": 8, "y": 24, - "w": 12, + "w": 8, "h": 8 }, "targets": [ @@ -833,6 +833,78 @@ "uid": "${datasource}" } }, + { + "type": "timeseries", + "title": "Tracker event-loop delay", + "description": "How long callbacks waited for the tracker event loop since the previous scrape; every REST and WebSocket message pays it. See docs/tracker-capacity.md.", + "gridPos": { + "x": 16, + "y": 24, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(openstreamgrid_event_loop_delay_p50_seconds{job=~\"$job\"})", + "legendFormat": "p50", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(openstreamgrid_event_loop_delay_p99_seconds{job=~\"$job\"})", + "legendFormat": "p99", + "refId": "B", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(openstreamgrid_event_loop_delay_max_seconds{job=~\"$job\"})", + "legendFormat": "max", + "refId": "C", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "id": 19, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, { "type": "row", "title": "Security",