From 103e2fda831f7d0db48100b5b659225d92c3d0d2 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Thu, 9 Jul 2026 16:08:32 +0200 Subject: [PATCH] alertmanager: group by alertname, repeat 5m to 12h (RpcProviderErrorRate spammed slack every 5min) --- .../monitoring/alertmanager/alertmanager.yml.tmpl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/infrastructure/monitoring/alertmanager/alertmanager.yml.tmpl b/infrastructure/monitoring/alertmanager/alertmanager.yml.tmpl index fd0f9799..42083bef 100644 --- a/infrastructure/monitoring/alertmanager/alertmanager.yml.tmpl +++ b/infrastructure/monitoring/alertmanager/alertmanager.yml.tmpl @@ -2,10 +2,15 @@ global: resolve_timeout: 5m route: - group_by: ['alertname', 'chain', 'aggregator'] - group_wait: 10s - group_interval: 30s - repeat_interval: 5m + # Group by alertname ONLY: one Slack message listing every (provider, + # chain) pair instead of 7 parallel groups each re-notifying on its own + # clock. repeat_interval 5m -> 12h: a still-firing alert is a reminder, + # not a new event (2026-07-09: RpcProviderErrorRate spammed the channel + # every 5 minutes for hours). + group_by: ['alertname'] + group_wait: 30s + group_interval: 15m + repeat_interval: 12h # Default: dedicated OCB Slack channel (direct incoming webhook). receiver: 'slack-direct' routes: