From b754ddaaa4fa68f850e792e8f42b34ae86a564a8 Mon Sep 17 00:00:00 2001 From: Matthias Linhuber Date: Fri, 28 Aug 2026 18:53:38 +0200 Subject: [PATCH 1/3] feat: fix the Grafana dashboards and add sessions dashboards and alerting The dashboards were broken in three ways, none of which a render test could catch, and all of which look like a data problem rather than a bug. The namespace picker in dashboard-theiacloud.yaml was a hand-written list: theia, theia-staging, test1, test2, test3. The real namespaces have been eduide-test1 and friends since 2.0.0, so every option was a dead namespace and the dashboard was empty whatever you picked. It is now a query over the namespaces bootstrap already derives from the environments, so it cannot go stale again. An empty list renders ^$ rather than falling back to .*, so a cluster with no monitored environments shows an empty picker instead of graphing other tenants. Four of the six panels also selected on service=~"theia-.*", a label that stopped existing when the sessions stopped sitting behind theia-* Services. That returns zero series against the live cluster; without it, six. Those panels have been blank for months. The theia-cloud-sessions PodMonitor has never produced a sample. It scrapes port application on every session pod, but session pods are Theia IDEs serving HTML, so all 30 targets sit permanently down with 'unsupported Content-Type text/html' and ~1480 candidates are dropped per cycle. Removed, along with monitoring.sessionNamespaces. Nothing is lost: session data comes from cAdvisor, kubelet and kube-state-metrics, which need no PodMonitor. New: an EduIDE Sessions dashboard (usage, warm pool, startup latency, resources against limits, throttling, workspace bytes and inodes, oldest sessions) and an EduIDE Session Detail dashboard reached from it, scoped to a single pod. New: 16 alerts behind monitoring.alerting.enabled, and an AlertmanagerConfig with native Slack and Discord receivers. Only critical and warning reach the channels; minSeverity raises that cut-off without editing rules, and every threshold is a value. Session-level faults alert on rates across an environment rather than per pod, and exit codes 137 and 143 are excluded, because this platform runs student code and one notification per OOM kill is how a channel stops being read. Every alert expression was run against the live Prometheus on tum-student. The seven that return nothing are healthy zeros, verified by finding the same metrics and labels elsewhere on the cluster, not selectors that match nothing. Alerts carry namespace: eduide-system as a routing label, because the Prometheus Operator defaults alertmanagerConfigMatcherStrategy to OnNamespace and prepends that matcher to every route it generates. The environment is in eduide_namespace, which is also what grouping and any silence must use. Also adds a cert-manager ServiceMonitor: it exports certificate expiry but ships no ServiceMonitor, so nothing was watching the hand-renewed webview wildcard at all. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019qeiQRFu8xAMRYWPdZewjG --- charts/eduide-cluster/Chart.yaml | 2 +- charts/eduide-cluster/README.md | 27 +- charts/eduide-cluster/templates/_helpers.tpl | 42 + .../monitoring/alert-webhook-secret.yaml | 33 + .../monitoring/alertmanagerconfig.yaml | 103 + .../monitoring/dashboard-session-detail.yaml | 1215 +++++++++++ .../monitoring/dashboard-session-startup.yaml | 19 +- .../monitoring/dashboard-sessions.yaml | 1901 +++++++++++++++++ .../monitoring/dashboard-theiacloud.yaml | 60 +- .../monitoring/podmonitor-sessions.yaml | 34 - .../templates/monitoring/prometheusrule.yaml | 409 ++++ .../servicemonitor-certmanager.yaml | 34 + charts/eduide-cluster/values-example.yaml | 36 +- charts/eduide-cluster/values.yaml | 87 +- 14 files changed, 3912 insertions(+), 90 deletions(-) create mode 100644 charts/eduide-cluster/templates/_helpers.tpl create mode 100644 charts/eduide-cluster/templates/monitoring/alert-webhook-secret.yaml create mode 100644 charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml create mode 100644 charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml create mode 100644 charts/eduide-cluster/templates/monitoring/dashboard-sessions.yaml delete mode 100644 charts/eduide-cluster/templates/monitoring/podmonitor-sessions.yaml create mode 100644 charts/eduide-cluster/templates/monitoring/prometheusrule.yaml create mode 100644 charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml diff --git a/charts/eduide-cluster/Chart.yaml b/charts/eduide-cluster/Chart.yaml index e403beb..91fa447 100644 --- a/charts/eduide-cluster/Chart.yaml +++ b/charts/eduide-cluster/Chart.yaml @@ -4,5 +4,5 @@ description: | Cluster-scoped half of EduIDE: CRDs, the conversion webhook, ClusterRoles and cert-manager issuers. Install once per cluster, before any eduide release. type: application -version: 2.1.5 +version: 2.2.0 appVersion: "1.2.0" diff --git a/charts/eduide-cluster/README.md b/charts/eduide-cluster/README.md index e088a74..d6fea83 100644 --- a/charts/eduide-cluster/README.md +++ b/charts/eduide-cluster/README.md @@ -1,6 +1,6 @@ # eduide-cluster -![Version: 2.1.5](https://img.shields.io/badge/Version-2.1.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) +![Version: 2.2.0](https://img.shields.io/badge/Version-2.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) Cluster-scoped half of EduIDE: CRDs, the conversion webhook, ClusterRoles and cert-manager issuers. Install once per cluster, before any eduide release. @@ -50,10 +50,33 @@ cert-manager issuers. Install once per cluster, before any eduide release. | managedCertificates.enabled | bool | `false` | | | managedCertificates.issuerRef.kind | string | `"ClusterIssuer"` | | | managedCertificates.issuerRef.name | string | `"letsencrypt-prod"` | | -| monitoring | object | `{"dashboardNamespace":"cattle-dashboards","enabled":false,"namespace":"cattle-monitoring-system","sessionNamespaces":[],"targetNamespaces":[]}` | ------------------------------------------------------------------------ | +| monitoring | object | `{"alerting":{"channels":[],"enabled":false,"grafanaUrl":"","groupInterval":"5m","groupWait":"30s","minSeverity":"warning","namespace":"eduide-system","repeatInterval":"4h","runbookUrl":"https://eduide.github.io/Docs/admins/operations","thresholds":{"certExpiryDays":21,"componentRestarts":3,"sessionCrashPerHour":5,"sessionOOMPerHour":3,"startupSeconds":90,"volumePercent":85},"webhookSecret":{"create":false,"data":{},"name":"eduide-alert-webhooks"}},"certManager":{"enabled":false,"namespace":"cert-manager","portName":"http-metrics","serviceName":"cert-manager"},"dashboardNamespace":"cattle-dashboards","enabled":false,"namespace":"cattle-monitoring-system","targetNamespaces":[]}` | ------------------------------------------------------------------------ | +| monitoring.alerting.channels | list | `[]` | Where to send alerts. Each entry needs `name`, `type` (`slack` or `discord`) and `secretKey`; Slack entries may also set `channel`. An empty list with alerting enabled means alerts fire but notify nobody, so the chart fails the render instead. | +| monitoring.alerting.enabled | bool | `false` | Create the PrometheusRule and the AlertmanagerConfig. | +| monitoring.alerting.grafanaUrl | string | `""` | Base URL of the Grafana that serves the EduIDE dashboards, with no trailing slash. Used to deep-link a notification straight to the affected session. Left empty, notifications carry no dashboard link. | +| monitoring.alerting.groupInterval | string | `"5m"` | How long to wait before notifying about new alerts added to a group. | +| monitoring.alerting.groupWait | string | `"30s"` | How long to wait for more alerts in a group before the first notification. | +| monitoring.alerting.minSeverity | string | `"warning"` | Lowest severity that reaches the notification channels: `warning` or `critical`. Anything below stays in Alertmanager and on the dashboards. This is what keeps the channels worth reading. | +| monitoring.alerting.namespace | string | `"eduide-system"` | Namespace both resources go in. This is also the `namespace` label every EduIDE alert carries, and it is a routing artifact rather than the environment the alert is about. The Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to `OnNamespace`, which prepends `namespace = ` to every route generated from it, so an alert must claim this namespace to reach a receiver at all. The environment an alert is about is in `eduide_namespace`. Silence on that label, never on `namespace` - `namespace` is identical across every EduIDE alert and silencing it silences all of them. | +| monitoring.alerting.repeatInterval | string | `"4h"` | How long before an unresolved alert is sent again. | +| monitoring.alerting.runbookUrl | string | `"https://eduide.github.io/Docs/admins/operations"` | Base URL of the operations runbooks, with no trailing slash. | +| monitoring.alerting.thresholds.certExpiryDays | int | `21` | Days before certificate expiry to start warning. | +| monitoring.alerting.thresholds.componentRestarts | int | `3` | Restarts of a platform container within 15m before it counts as crash looping. | +| monitoring.alerting.thresholds.sessionCrashPerHour | int | `5` | Session crashes per hour in one environment before alerting. | +| monitoring.alerting.thresholds.sessionOOMPerHour | int | `3` | Session OOM kills per hour in one environment before alerting. | +| monitoring.alerting.thresholds.startupSeconds | int | `90` | p95 session startup, in seconds, that counts as too slow. | +| monitoring.alerting.thresholds.volumePercent | int | `85` | Workspace volume fill percentage that counts as nearly full. | +| monitoring.alerting.webhookSecret.create | bool | `false` | Create the Secret from `data` below. Off means the Secret already exists and is referenced by name only. | +| monitoring.alerting.webhookSecret.data | object | `{}` | Base64-encoded webhook URLs, keyed by the `secretKey` a channel names. Supplied by the workflow, not committed. | +| monitoring.alerting.webhookSecret.name | string | `"eduide-alert-webhooks"` | Name of the Secret channels read their URL from. | +| monitoring.certManager.enabled | bool | `false` | Create a ServiceMonitor for cert-manager's controller metrics. | +| monitoring.certManager.namespace | string | `"cert-manager"` | Namespace cert-manager runs in. | +| monitoring.certManager.portName | string | `"http-metrics"` | Name of the metrics port on that Service. | +| monitoring.certManager.serviceName | string | `"cert-manager"` | Name of cert-manager's metrics Service. | | monitoring.dashboardNamespace | string | `"cattle-dashboards"` | Namespace the Grafana dashboard ConfigMaps go in. Must already exist and be watched by Grafana's sidecar. `cattle-dashboards` is Rancher's. | | monitoring.enabled | bool | `false` | Create the PodMonitors and Grafana dashboards. Off by default because it is not portable: it needs the Prometheus Operator CRDs (`monitoring.coreos.com/v1`) to exist, and the two namespaces below are Rancher's. Helm does not create namespaces it was not told to, so on a cluster without them the install fails outright on the dashboards. Turn it on once you know where your Prometheus and Grafana look for these. | | monitoring.namespace | string | `"cattle-monitoring-system"` | Namespace the PodMonitors go in. Must be somewhere your Prometheus discovers. `cattle-monitoring-system` is Rancher's; change it for any other monitoring stack. | +| monitoring.targetNamespaces | list | `[]` | Namespaces to watch. Bootstrap derives this from the environments on the cluster; it is also the namespace list every dashboard's picker offers. | | operatorrole.name | string | `"operator-api-access"` | name for the operator's cluster role | | servicerole.name | string | `"service-api-access"` | name for the services' cluster role | | wildcardTLSSecret.certificate | string | `""` | | diff --git a/charts/eduide-cluster/templates/_helpers.tpl b/charts/eduide-cluster/templates/_helpers.tpl new file mode 100644 index 0000000..06b0eae --- /dev/null +++ b/charts/eduide-cluster/templates/_helpers.tpl @@ -0,0 +1,42 @@ +{{/* +A RE2 alternation of the namespaces this cluster monitors, anchored, for use in +a PromQL label matcher and in Grafana variable queries. + +The dashboards used to carry a hand-written list of namespaces. It went stale +the moment the environments were renamed - the picker still offered `theia`, +`theia-staging` and `test1`, none of which exist any more, so every panel was +empty whatever you selected. Deriving it from the same list the PodMonitors use +means it cannot drift again. + +An empty list yields `^$`, which matches nothing. That is deliberate: a cluster +where every environment opted out of monitoring should show an empty picker, +not silently fall back to `.*` and start graphing other tenants' namespaces. +*/}} +{{- define "eduide.monitoredNamespaceRegex" -}} +{{- $ns := .Values.monitoring.targetNamespaces | default list -}} +{{- if $ns -}} +^({{ join "|" $ns }})$ +{{- else -}} +^$ +{{- end -}} +{{- end -}} + +{{/* +The labels every EduIDE alert carries. + +`namespace` is a routing label and is NOT the environment the alert is about. +The Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to +`OnNamespace`, which prepends `namespace = ` to every route it generates. Our AlertmanagerConfig lives in one +namespace, so alerts have to claim that namespace or they reach no receiver at +all. + +The environment an alert is actually about is `eduide_namespace`, templated +from the series. Silences and inhibition rules must match on that one; +`namespace` is the same string on every EduIDE alert and matching it would +silence all of them at once. +*/}} +{{- define "eduide.alertRoutingLabels" -}} +namespace: {{ .Values.monitoring.alerting.namespace }} +eduide_namespace: '{{ "{{" }} $labels.namespace {{ "}}" }}' +{{- end -}} diff --git a/charts/eduide-cluster/templates/monitoring/alert-webhook-secret.yaml b/charts/eduide-cluster/templates/monitoring/alert-webhook-secret.yaml new file mode 100644 index 0000000..c02bd16 --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/alert-webhook-secret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.monitoring.enabled .Values.monitoring.alerting.enabled .Values.monitoring.alerting.webhookSecret.create }} +{{- /* +The webhook URLs the channels post to. + +A Slack or Discord webhook URL is a credential: anyone holding it can post into +the channel. It never goes in a values file in git and never on a `--set`, which +would put it in the process list and in Actions debug logs. Bootstrap reads it +from a GitHub Environment secret and passes it here already base64-encoded, the +same way the wildcard TLS certificate is handled. + +Chart-managed rather than applied alongside, so it is removed when the release +is, and so a channel referring to a key that was never supplied fails the deploy +instead of producing an AlertmanagerConfig that silently notifies nobody. +*/}} +{{- $a := .Values.monitoring.alerting }} +{{- range $i, $c := $a.channels }} +{{- if not (hasKey $a.webhookSecret.data $c.secretKey) }} +{{- fail (printf "channel %q reads key %q from secret %q, but that key was not supplied in monitoring.alerting.webhookSecret.data" $c.name $c.secretKey $a.webhookSecret.name) }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ $a.webhookSecret.name }} + namespace: {{ $a.namespace }} + labels: + app.kubernetes.io/part-of: eduide +data: + {{- range $k, $v := $a.webhookSecret.data }} + {{ $k }}: {{ $v | quote }} + {{- end }} +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml b/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml new file mode 100644 index 0000000..373511b --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml @@ -0,0 +1,103 @@ +{{- if and .Values.monitoring.enabled .Values.monitoring.alerting.enabled }} +{{- $a := .Values.monitoring.alerting }} +{{- /* +Where EduIDE's alerts go. + +Two things about this file are not obvious. + +**The route matches on `namespace`, and that label is a routing artifact.** The +Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to +`OnNamespace`, which silently prepends `namespace = {{ $a.namespace }}` to the +route generated from this resource. Every EduIDE alert therefore sets that +label, and carries the environment it is really about in `eduide_namespace`. +Grouping and any inhibition rule must use `eduide_namespace`; grouping on +`namespace` would merge an outage in production with one in staging into a +single notification, because the label is identical across all of them. + +**Only warning and above is notified.** Everything fires and is visible in +Alertmanager and on the dashboards, but the channels get a filtered subset. A +channel that receives every capacity blip stops being read, and then the +platform is unmonitored no matter how many rules exist. +*/}} +{{- if not $a.channels }} +{{- fail "monitoring.alerting.enabled is true but monitoring.alerting.channels is empty - alerts would fire and notify nobody" }} +{{- end }} +{{- $severities := list "critical" }} +{{- if eq ($a.minSeverity | toString) "warning" }} +{{- $severities = list "critical" "warning" }} +{{- else if ne ($a.minSeverity | toString) "critical" }} +{{- fail (printf "monitoring.alerting.minSeverity must be \"warning\" or \"critical\", got %q" ($a.minSeverity | toString)) }} +{{- end }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: eduide-alerts + namespace: {{ $a.namespace }} + labels: + app.kubernetes.io/part-of: eduide +spec: + route: + receiver: eduide-channels + groupBy: + - alertname + - eduide_namespace + groupWait: {{ $a.groupWait }} + groupInterval: {{ $a.groupInterval }} + repeatInterval: {{ $a.repeatInterval }} + matchers: + - name: severity + matchType: =~ + value: {{ join "|" $severities | quote }} + receivers: + - name: eduide-channels +{{- $slack := list }} +{{- $discord := list }} +{{- range $i, $c := $a.channels }} + {{- if not $c.name }}{{ fail (printf "monitoring.alerting.channels[%d] has no name" $i) }}{{ end }} + {{- if not $c.secretKey }}{{ fail (printf "channel %q names no secretKey - webhook URLs are only ever read from a Secret, never from values" $c.name) }}{{ end }} + {{- $type := $c.type | default "" | toString }} + {{- if eq $type "slack" }}{{ $slack = append $slack $c }} + {{- else if eq $type "discord" }}{{ $discord = append $discord $c }} + {{- else }}{{ fail (printf "channel %q has type %q; supported types are \"slack\" and \"discord\"" $c.name $type) }}{{ end }} +{{- end }} +{{- /* +The message body. Alertmanager templating, not Helm's - the doubled braces are +escapes. Written to stand on its own in a chat client: what broke, what it means +for students, and where to look, without needing the dashboard open. `.Alerts` +is grouped by alertname and environment, so one message covers one problem in +one environment however many pods it touched. +*/}} +{{- $text := `{{ range .Alerts }}*{{ .Annotations.summary }}* +{{ .Annotations.description }} +{{ if .Labels.eduide_namespace }}Environment: {{ .Labels.eduide_namespace }}{{ end }}{{ if .Labels.pod }} | Pod: {{ .Labels.pod }}{{ end }}{{ if .Labels.node }} | Node: {{ .Labels.node }}{{ end }} +Severity: {{ .Labels.severity }} | Since: {{ .StartsAt.Format "15:04 MST" }} +{{ if .Annotations.runbook_url }}Runbook: {{ .Annotations.runbook_url }} +{{ end }}{{ if .Annotations.dashboard_url }}Dashboard: {{ .Annotations.dashboard_url }} +{{ end }}{{ end }}` }} +{{- if $slack }} + slackConfigs: + {{- range $slack }} + - apiURL: + name: {{ $a.webhookSecret.name }} + key: {{ .secretKey }} + {{- if .channel }} + channel: {{ .channel | quote }} + {{- end }} + sendResolved: true + username: EduIDE + title: '{{ `{{ if eq .Status "firing" }}[FIRING {{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ .GroupLabels.alertname }}{{ if .GroupLabels.eduide_namespace }} - {{ .GroupLabels.eduide_namespace }}{{ end }}` }}' + text: {{ $text | quote }} + {{- end }} +{{- end }} +{{- if $discord }} + discordConfigs: + {{- range $discord }} + - apiURL: + name: {{ $a.webhookSecret.name }} + key: {{ .secretKey }} + sendResolved: true + title: '{{ `{{ if eq .Status "firing" }}[FIRING {{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ .GroupLabels.alertname }}{{ if .GroupLabels.eduide_namespace }} - {{ .GroupLabels.eduide_namespace }}{{ end }}` }}' + message: {{ $text | quote }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml new file mode 100644 index 0000000..a51a0d1 --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml @@ -0,0 +1,1215 @@ +{{- if .Values.monitoring.enabled }} +{{- /* +One session in detail, reached from the Oldest sessions table on the +sessions overview or by picking a pod directly. +*/}} +{{- $dashboardNs := .Values.monitoring.dashboardNamespace | toString | trim }} +{{- if not $dashboardNs }} +{{- fail "monitoring.enabled is true but monitoring.dashboardNamespace is empty" }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: eduide-dashboard-session-detail + namespace: {{ $dashboardNs }} +data: + eduide-session-detail.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "One session, in detail. Pick the environment and the pod; every panel is scoped to that pod.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "tags": [], + "targetBlank": false, + "title": "All sessions", + "tooltip": "Back to the sessions overview", + "type": "link", + "url": "/d/eduide-sessions/eduide-sessions" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 200, + "panels": [], + "title": "This session", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "How long this pod has existed. Not how long the student has been working - a warm-pool pod is created before anyone claims it.", + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 201, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - max(kube_pod_created{namespace=~\"$namespace\", pod=~\"$pod\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Age", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Both the IDE and its oauth2-proxy sidecar must be ready before the session serves traffic. 1 of 2 means the proxy is failing, which presents to the student as a login loop or a 403.", + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 202, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_status_ready{namespace=~\"$namespace\", pod=~\"$pod\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Ready containers", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Any restart of a session container loses the student's terminal state and unsaved editor buffers.", + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 203, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"$pod\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Limit minus current working set for the IDE container. When this approaches zero the kernel kills the container.", + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 204, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", resource=\"memory\", container!~\"oauth.*\"}) - sum(container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"$pod\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Memory headroom", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 209715200 + }, + { + "color": "green", + "value": 524288000 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percentage of the workspace volume in use, if this session has a persistent one.", + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 205, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(100 * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Workspace used", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 210, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The dashed limit line is what the container is killed at. The oauth2-proxy sidecar is included here on purpose: it is small, and a proxy eating memory is a real failure mode.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 211, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"$pod\", container!=\"\", container!=\"POD\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", resource=\"memory\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }} limit", + "range": true, + "refId": "B" + } + ], + "title": "Memory per container, against the limit", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "A container pinned at its limit is being throttled; see the panel below for how hard.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 212, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"$pod\", container!=\"\", container!=\"POD\"}[5m])", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", resource=\"cpu\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }} limit", + "range": true, + "refId": "B" + } + ], + "title": "CPU per container, against the limit", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Scheduling periods in which the container wanted CPU and was denied. Non-zero here with normal-looking CPU usage is the signature of a session that feels slow to the student but looks fine to the cluster.", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 213, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\", pod=~\"$pod\", container!=\"\", container!=\"POD\"}[5m])", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU throttled periods", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Traffic to and from the session. A live IDE holds an open WebSocket, so a flatline on both while the pod is Running means the student's connection dropped even though nothing restarted.", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 214, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[5m])", + "format": "time_series", + "instant": false, + "legendFormat": "received", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[5m])", + "format": "time_series", + "instant": false, + "legendFormat": "transmitted", + "range": true, + "refId": "B" + } + ], + "title": "Network", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Non-zero means the container is not running and why: ImagePullBackOff for a tag that was never built, CreateContainerConfigError for a missing secret or ConfigMap, CrashLoopBackOff for a container that keeps exiting.", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 215, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", pod=~\"$pod\"} > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}: {{ "{{" }}reason{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Container waiting reason", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 220, + "panels": [], + "title": "How it ended", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "What killed the container the last time it stopped. OOMKilled means it exceeded its memory limit. Error means the process exited non-zero. Completed with exit 143 or 137 is the garbage collector stopping an idle session, which is routine and not a fault.", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 221, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\", pod=~\"$pod\"} > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}: {{ "{{" }}reason{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Last termination reason", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "137 is SIGKILL, 143 is SIGTERM - both are normal shutdowns. Anything else came from inside the container.", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 222, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_status_last_terminated_exitcode{namespace=~\"$namespace\", pod=~\"$pod\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Last exit code", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "eduide" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"},pod)", + "hide": 0, + "includeAll": false, + "label": "Session pod", + "multi": false, + "name": "pod", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "EduIDE Session Detail", + "uid": "eduide-session-detail", + "version": 1, + "weekStart": "" + } +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-session-startup.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-session-startup.yaml index a33c8ef..96d9943 100644 --- a/charts/eduide-cluster/templates/monitoring/dashboard-session-startup.yaml +++ b/charts/eduide-cluster/templates/monitoring/dashboard-session-startup.yaml @@ -89,7 +89,7 @@ data: { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(application_application_theiacloud_session_startup_seconds_seconds_sum{namespace=\"$namespace\"}[5m]) \n/ \nrate(application_application_theiacloud_session_startup_seconds_seconds_count{namespace=\"$namespace\"}[5m])", + "expr": "rate(application_application_theiacloud_session_startup_seconds_seconds_sum{namespace=~\"$namespace\"}[5m]) \n/ \nrate(application_application_theiacloud_session_startup_seconds_seconds_count{namespace=~\"$namespace\"}[5m])", "instant": false, "interval": "", "legendFormat": "{{ "{{" }}app_definition{{ "}}" }}", @@ -152,7 +152,7 @@ data: { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=\"$namespace\", quantile=\"0.5\"}", + "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.5\"}", "instant": false, "legendFormat": "{{ "{{" }}app_definition{{ "}}" }}", "range": true, @@ -214,7 +214,7 @@ data: { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=\"$namespace\", quantile=\"0.95\"}", + "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.95\"}", "instant": false, "legendFormat": "{{ "{{" }}app_definition{{ "}}" }}", "range": true, @@ -276,7 +276,7 @@ data: { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=\"$namespace\", quantile=\"0.99\"}", + "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.99\"}", "instant": false, "legendFormat": "{{ "{{" }}app_definition{{ "}}" }}", "range": true, @@ -293,24 +293,25 @@ data: "templating": { "list": [ { - "current": { "selected": false, "text": "test1", "value": "test1" }, + "allValue": {{ include "eduide.monitoredNamespaceRegex" . | quote }}, + "current": {}, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(application_application_theiacloud_session_startup_seconds_seconds_count,namespace)", + "definition": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "Namespace", "multi": false, "name": "namespace", "options": [], "query": { "qryType": 1, - "query": "label_values(application_application_theiacloud_session_startup_seconds_seconds_count,namespace)", + "query": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, - "sort": 0, + "sort": 1, "type": "query" } ] diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-sessions.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-sessions.yaml new file mode 100644 index 0000000..ce7c80b --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/dashboard-sessions.yaml @@ -0,0 +1,1901 @@ +{{- if .Values.monitoring.enabled }} +{{- /* +Sessions overview. Everything here comes from cAdvisor, kubelet and +kube-state-metrics, plus the one startup timer the REST service exports. +Session pods themselves expose no metrics endpoint - they serve HTML - so +nothing on this dashboard needs a PodMonitor against them. +*/}} +{{- $dashboardNs := .Values.monitoring.dashboardNamespace | toString | trim }} +{{- if not $dashboardNs }} +{{- fail "monitoring.enabled is true but monitoring.dashboardNamespace is empty" }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: eduide-dashboard-sessions + namespace: {{ $dashboardNs }} +data: + eduide-sessions.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Who is using EduIDE right now, how well it is serving them, and what it is costing the cluster.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Right now", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Pods named session-*. One per student actually working. Warm-pool pods are renamed to session-* when a student claims one.", + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 101, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(kube_pod_info{namespace=~\"$namespace\", pod=~\"session-.*\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Sessions in use", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Pods named instance-*: started ahead of demand so a student does not wait for a cold start. If this reaches 0 the next student waits.", + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 102, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(kube_pod_info{namespace=~\"$namespace\", pod=~\"instance-.*\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Warm pool waiting", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Everything a student could be sitting in, claimed or not.", + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 103, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(kube_pod_info{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "IDE pods total", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "How many EduIDE namespaces Prometheus can currently see. A drop here means an environment stopped reporting, not that it has no sessions.", + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 104, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(count by (namespace) (kube_pod_info{namespace=~\"$namespace\"})) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Environments reporting", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 110, + "panels": [], + "title": "Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Claimed sessions over time. This is the closest thing to a count of students actively using EduIDE.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 111, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (namespace) (kube_pod_info{namespace=~\"$namespace\", pod=~\"session-.*\"})", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Sessions in use, by environment", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Unclaimed instance-* pods. Flat lines at 0 mean every start is a cold start; see the startup latency panel below for what that costs.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 112, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (namespace) (kube_pod_info{namespace=~\"$namespace\", pod=~\"instance-.*\"})", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Warm pool available, by environment", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Which language images are actually being used. The container is named after the app definition, so this is the app breakdown without needing a metric from the operator.", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 113, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"})", + "format": "table", + "instant": true, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": false, + "refId": "A" + } + ], + "title": "IDE pods by app image", + "type": "barchart", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "lineWidth": 1 + }, + "mappings": [], + "noValue": "No pods", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "barWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "auto", + "xTickLabelRotation": 0 + }, + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Derived from the REST service's startup timer, so it counts sessions the service actually served - not pods the operator created.", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 114, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespace) (rate(application_application_theiacloud_session_startup_seconds_seconds_count{namespace=~\"$namespace\"}[15m])) * 3600", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Session starts per hour", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "How long a student waits between asking for a session and getting one. These are decaying quantiles: with no traffic they hold their last value rather than dropping to zero, so read them alongside the starts per hour panel.", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 115, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (namespace) (application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.5\"})", + "format": "time_series", + "instant": false, + "legendFormat": "p50 {{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (namespace) (application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.95\"})", + "format": "time_series", + "instant": false, + "legendFormat": "p95 {{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (namespace) (application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.99\"})", + "format": "time_series", + "instant": false, + "legendFormat": "p99 {{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "C" + } + ], + "title": "Session startup latency", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 120, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sum across all IDE containers, excluding the oauth2-proxy sidecar. If used tracks close to limit the sessions are CPU starved and builds feel slow; if it sits far below, the limit is generous and more sessions fit per node.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 121, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"}[5m]))", + "format": "time_series", + "instant": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", resource=\"cpu\", container!~\"oauth.*\"})", + "format": "time_series", + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Session CPU: used against limit", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The gap between these two lines is what stands between a student and an OOM kill. A session killed for memory loses unsaved editor state; the workspace volume survives.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 122, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"})", + "format": "time_series", + "instant": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", resource=\"memory\", container!~\"oauth.*\"})", + "format": "time_series", + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Session memory: used against limit", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Share of scheduling periods in which a session container was held back at its CPU limit. Sustained throttling is what students report as \"the IDE is laggy\" while every pod looks healthy.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 123, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"}[5m])) / clamp_min(sum by (namespace) (rate(container_cpu_cfs_periods_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"}[5m])), 1)", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU throttling", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sessions are scheduled like any other pod. A node holding far more than its share is usually a sign the others are cordoned or full.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 124, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (node) (kube_pod_info{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"})", + "format": "table", + "instant": true, + "legendFormat": "{{ "{{" }}node{{ "}}" }}", + "range": false, + "refId": "A" + } + ], + "title": "IDE pods per node", + "type": "barchart", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "lineWidth": 1 + }, + "mappings": [], + "noValue": "No pods", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "barWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "auto", + "xTickLabelRotation": 0 + }, + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 130, + "panels": [], + "title": "Workspace storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "PersistentVolumeClaims in Bound phase. Persistent workspaces only; ephemeral installations have none and this reads 0 correctly.", + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 41 + }, + "id": 131, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolumeclaim_status_phase{namespace=~\"$namespace\", phase=\"Bound\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Workspace volumes bound", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "A pending claim means the session cannot start. Usually the storage class or a quota, not the cluster being full.", + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 41 + }, + "id": 132, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolumeclaim_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Volumes pending", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total capacity claimed by workspaces across the selected environments.", + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 41 + }, + "id": 133, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace=~\"$namespace\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Storage requested", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The single closest workspace to running out of space. A full workspace fails writes inside the IDE with errors that do not name the disk.", + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 41 + }, + "id": 134, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(100 * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Fullest workspace", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per claim, as a percentage of capacity. Watch for a claim that climbs and never comes down - that is a build cache nobody cleans up.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 135, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}/{{ "{{" }}persistentvolumeclaim{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Workspace volume usage", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Inodes run out long before bytes do when a session runs npm install or resolves a large Maven tree. The IDE reports \"no space left on device\" on a volume that looks half empty.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 136, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * kubelet_volume_stats_inodes_used{namespace=~\"$namespace\"} / kubelet_volume_stats_inodes{namespace=~\"$namespace\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}/{{ "{{" }}persistentvolumeclaim{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Workspace inode usage", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 140, + "panels": [], + "title": "Longest running", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sessions ranked by age. The garbage collector stops idle sessions, so a session far older than the rest is either genuinely in use or stuck. Click a row to open it in the session detail dashboard.", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 141, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "topk(25, time() - kube_pod_created{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"})", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Oldest sessions", + "type": "table", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Age" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 14400 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Open in session detail", + "url": "/d/eduide-session-detail/eduide-session-detail?var-namespace=${__data.fields.Environment}&var-pod=${__data.fields.Pod}" + } + ] + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Age" + } + ] + }, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "container": true, + "endpoint": true, + "instance": true, + "job": true, + "service": true, + "uid": true + }, + "renameByName": { + "namespace": "Environment", + "pod": "Pod", + "Value": "Age" + } + } + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "eduide" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "hide": 0, + "includeAll": true, + "label": "Environment", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "allValue": "{{ include "eduide.monitoredNamespaceRegex" . }}" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "EduIDE Sessions", + "uid": "eduide-sessions", + "version": 1, + "weekStart": "" + } +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-theiacloud.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-theiacloud.yaml index 7cf6e72..a0d3203 100644 --- a/charts/eduide-cluster/templates/monitoring/dashboard-theiacloud.yaml +++ b/charts/eduide-cluster/templates/monitoring/dashboard-theiacloud.yaml @@ -147,7 +147,7 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "count(count by (pod) (container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!=\"POD\"}))", + "expr": "count(count by (pod) (container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!=\"POD\"}))", "format": "time_series", "instant": false, "interval": "", @@ -246,7 +246,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\"}[1m])) by (pod)", + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=~\"$namespace\"}[1m])) by (pod)", "instant": false, "legendFormat": "__auto", "range": true, @@ -258,7 +258,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\"}[1m]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=~\"$namespace\"}[1m]))", "hide": false, "instant": false, "legendFormat": "TOTAL", @@ -357,7 +357,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", service=~\"theia-.*\"}", + "expr": "container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\"}", "instant": false, "legendFormat": "{{ "{{" }}pod{{ "}}" }}", "range": true, @@ -455,7 +455,7 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", service=~\"theia-.*\"}[1m])", + "expr": "rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\"}[1m])", "instant": false, "interval": "", "legendFormat": "{{ "{{" }}pod{{ "}}" }}", @@ -554,7 +554,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "avg by (container) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", service=~\"theia-.*\"})", + "expr": "avg by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\"})", "instant": false, "legendFormat": "{{ "{{" }}container{{ "}}" }}", "range": true, @@ -652,7 +652,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "avg by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", service=~\"theia-.*\"}[1m]))", + "expr": "avg by (container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\"}[1m]))", "instant": false, "legendFormat": "{{ "{{" }}container{{ "}}" }}", "range": true, @@ -1087,42 +1087,26 @@ data: "templating": { "list": [ { - "current": { "selected": false, "text": "theia", "value": "theia" }, + "allValue": {{ include "eduide.monitoredNamespaceRegex" . | quote }}, + "current": {}, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "Namespace", "multi": false, "name": "namespace", - "options": [ - { - "selected": false, - "text": "theia", - "value": "theia" - }, - { - "selected": false, - "text": "theia-staging", - "value": "theia-staging" - }, - { - "selected": true, - "text": "test1", - "value": "test1" - }, - { - "selected": false, - "text": "test2", - "value": "test2" - }, - { - "selected": false, - "text": "test3", - "value": "test3" - } - ], - "query": "theia,theia-staging,test1,test2,test3", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", "skipUrlSync": false, - "type": "custom" + "sort": 1, + "type": "query" } ] }, diff --git a/charts/eduide-cluster/templates/monitoring/podmonitor-sessions.yaml b/charts/eduide-cluster/templates/monitoring/podmonitor-sessions.yaml deleted file mode 100644 index d35e743..0000000 --- a/charts/eduide-cluster/templates/monitoring/podmonitor-sessions.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.monitoring.enabled }} -{{- /* -A PodMonitor naming no namespaces watches nothing, so it is not rendered at all -rather than created as a no-op. The list is derived by bootstrap-cluster.yml -from the environments on the cluster; an empty one means every environment -opted out, which is a supported state. A bare `helm install eduide-cluster` -lands here too, and must not fail. -*/}} -{{- if .Values.monitoring.sessionNamespaces }} -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: theia-cloud-sessions - namespace: {{ .Values.monitoring.namespace }} -spec: - selector: - matchExpressions: - - key: app - operator: NotIn - values: - - conversion-webhook - - landing-page - - operator - - service - namespaceSelector: - matchNames: - {{- range .Values.monitoring.sessionNamespaces }} - - {{ . }} - {{- end }} - podMetricsEndpoints: - - port: application - interval: 15s -{{- end }} -{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml b/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml new file mode 100644 index 0000000..3d68155 --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml @@ -0,0 +1,409 @@ +{{- if and .Values.monitoring.enabled .Values.monitoring.alerting.enabled }} +{{- /* +EduIDE's alerts. + +Two rules govern everything in here. + +**Every expression is checked against a live Prometheus before it ships.** The +dashboards in this directory spent months showing nothing because they selected +on `service=~"theia-.*"`, a label that stopped existing in 2.0.0 and that no +render test could catch. A selector that matches nothing produces an alert that +never fires, which looks exactly like a healthy platform. + +**Session-level faults alert on rates, never per pod.** This platform runs +student code. Sessions get OOM-killed and crash during normal exercise work, and +one notification per event turns the channel into something nobody reads - at +which point the platform is unmonitored no matter how many rules exist. + +Deployment names below are the real ones, verified against kube-state-metrics: +operator-deployment, service-deployment, landing-page-deployment, +garbage-collector, conversion-webhook. +*/}} +{{- $a := .Values.monitoring.alerting }} +{{- $t := $a.thresholds }} +{{- $ns := include "eduide.monitoredNamespaceRegex" . }} +{{- $sessions := `pod=~"(session-|instance-).*"` }} +{{- $ide := `container!="", container!~"oauth.*", container!="POD"` }} +{{- $runbook := printf "%s/incident-response" (trimSuffix "/" $a.runbookUrl) }} +{{- /* +A deep link into the session dashboards, when a Grafana URL is configured. The +notification is read by someone who is not at their desk, so a link that lands +on the right pod is worth more than the metric value. +*/}} +{{- $g := trimSuffix "/" $a.grafanaUrl }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: eduide-alerts + namespace: {{ $a.namespace }} + labels: + app.kubernetes.io/part-of: eduide +spec: + groups: + - name: eduide.platform + rules: + - alert: EduIDEComponentDown + expr: |- + kube_deployment_status_replicas_available{namespace=~"{{ $ns }}", deployment=~"(operator|service|landing-page)-deployment|garbage-collector"} == 0 + for: 5m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'EduIDE {{ "{{" }} $labels.deployment {{ "}}" }} is down in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + The {{ "{{" }} $labels.deployment {{ "}}" }} deployment in {{ "{{" }} $labels.namespace {{ "}}" }} has had no + available replicas for 5 minutes. + {{ "{{" }} if eq $labels.deployment "service-deployment" {{ "}}" }}Students cannot start new + sessions: the landing page calls this service to create one. Sessions already + running are unaffected.{{ "{{" }} end {{ "}}" }} + {{ "{{" }} if eq $labels.deployment "operator-deployment" {{ "}}" }}Nothing reconciles Session + or Workspace resources: new sessions will be accepted and never start, and + stopped ones will not be cleaned up. Running sessions keep working.{{ "{{" }} end {{ "}}" }} + {{ "{{" }} if eq $labels.deployment "landing-page-deployment" {{ "}}" }}Students get an error + page instead of the environment picker. Running sessions keep working and + direct session URLs still resolve.{{ "{{" }} end {{ "}}" }} + {{ "{{" }} if eq $labels.deployment "garbage-collector" {{ "}}" }}Idle sessions are no longer + being stopped, so the cluster will fill up with sessions nobody is using. + Nothing breaks immediately; capacity degrades over hours.{{ "{{" }} end {{ "}}" }} + Start with `kubectl -n {{ "{{" }} $labels.namespace {{ "}}" }} describe deploy {{ "{{" }} $labels.deployment {{ "}}" }}` + and the pod logs. + To silence this during planned maintenance, match on + eduide_namespace="{{ "{{" }} $labels.namespace {{ "}}" }}" - not on namespace. + runbook_url: {{ $runbook }} + + - alert: EduIDEConversionWebhookDown + expr: |- + kube_deployment_status_replicas_available{namespace="{{ .Release.Namespace }}", deployment="conversion-webhook"} == 0 + for: 5m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'The EduIDE conversion webhook is down, cluster-wide' + description: >- + The conversion webhook has had no available replicas for 5 minutes. + This is cluster-scoped, so it affects every environment at once, not one. + Every read and write of a Session, Workspace or AppDefinition goes through + it, so while it is down the operator cannot reconcile anything and + `kubectl get session` fails as well. Sessions already running keep serving + traffic; nothing new can start anywhere. + Check `kubectl -n {{ .Release.Namespace }} get pods -l app=conversion-webhook` + and whether its serving certificate is still valid. + runbook_url: {{ $runbook }} + + - alert: EduIDEComponentCrashLooping + expr: |- + increase(kube_pod_container_status_restarts_total{namespace=~"{{ $ns }}", pod!~"(session-|instance-).*"}[15m]) > {{ $t.componentRestarts }} + for: 5m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'EduIDE {{ "{{" }} $labels.container {{ "}}" }} is crash looping in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Container {{ "{{" }} $labels.container {{ "}}" }} in pod {{ "{{" }} $labels.pod {{ "}}" }} restarted + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} times in 15 minutes. This is a platform component, + not a student session, so it is a fault rather than something a student did. + A component that restarts this often is usually failing on startup: a + misconfigured secret, an unreachable Keycloak, or an image that does not + match the chart. `kubectl -n {{ "{{" }} $labels.namespace {{ "}}" }} logs {{ "{{" }} $labels.pod {{ "}}" }} + --previous` shows why the last attempt died. + runbook_url: {{ $runbook }} + + - alert: EduIDEServiceScrapeDown + expr: |- + up{job=~".*theia-cloud-service"} == 0 + for: 10m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Prometheus cannot scrape the EduIDE service in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + The REST service in {{ "{{" }} $labels.namespace {{ "}}" }} has not been scrapeable for 10 + minutes. This does not mean students are affected - the service may be + serving fine - but it does mean we are blind: session start counts and + startup latency stop being recorded, and EduIDESessionStartupSlow cannot + fire while this is true. Treat it as a monitoring outage for that + environment. Check the pod is running and that /q/metrics still answers. + runbook_url: {{ $runbook }} + + - alert: EduIDEWarmPoolEmpty + expr: |- + (sum by (namespace) (kube_deployment_status_replicas{namespace=~"{{ $ns }}", deployment=~"instance-.*"}) > 0) + and + (sum by (namespace) (kube_deployment_status_replicas_available{namespace=~"{{ $ns }}", deployment=~"instance-.*"}) == 0) + for: 10m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'The EduIDE warm pool is empty in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Every pre-warmed instance in {{ "{{" }} $labels.namespace {{ "}}" }} is unavailable, while the + environment is configured to keep some. Sessions still start, but each + student now waits for a cold start - pulling the image and booting Theia - + instead of being handed a running IDE. + This is expressed against desired replicas, so an environment configured + with no warm pool cannot trigger it. Common causes are a node that cannot + pull the image, insufficient cluster capacity, or every instance having + just been claimed at once. + `kubectl -n {{ "{{" }} $labels.namespace {{ "}}" }} get pods | grep instance-` is the first look. + runbook_url: {{ $runbook }} + + - name: eduide.sessions + rules: + - alert: EduIDESessionOOMKillSpike + expr: |- + sum by (namespace) ( + increase(kube_pod_container_status_last_terminated_reason{namespace=~"{{ $ns }}", {{ $sessions }}, reason="OOMKilled"}[1h]) + ) > {{ $t.sessionOOMPerHour }} + for: 10m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Sessions are being OOM-killed in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} session containers in {{ "{{" }} $labels.namespace {{ "}}" }} were killed + for exceeding their memory limit in the last hour. Each one ended a + student's session abruptly: unsaved editor buffers and terminal state are + gone, though the workspace volume survives. + A single OOM kill is normal - students write code that allocates. This + alerts on the rate, so it means either the memory limit is too low for the + work being set, or one exercise is pushing everyone over it at once. + The session detail dashboard shows memory against the limit per container. + Raise `appDefinitions.defaults.limitsMemory` for the environment if this + is the workload rather than a bug. + runbook_url: {{ $runbook }} + {{- if $g }} + dashboard_url: '{{ $g }}/d/eduide-sessions/eduide-sessions?var-namespace={{ "{{" }} $labels.namespace {{ "}}" }}' + {{- end }} + + {{- /* + Restarts, narrowed to containers whose last termination was an Error. + Counting exit codes directly does not work: the exit code is a gauge, so + it cannot be fed to increase(), and a gauge cannot distinguish one crash + from the same crash still being reported an hour later. + `and on(...)` filters the restart counter by the existence of an Error + termination for the same container, which is both valid PromQL and the + thing we actually mean. + */}} + - alert: EduIDESessionCrashSpike + expr: |- + sum by (namespace) ( + increase(kube_pod_container_status_restarts_total{namespace=~"{{ $ns }}", {{ $sessions }}}[1h]) + and on (namespace, pod, container) + (kube_pod_container_status_last_terminated_reason{namespace=~"{{ $ns }}", {{ $sessions }}, reason="Error"} == 1) + ) > {{ $t.sessionCrashPerHour }} + for: 10m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Sessions are crashing in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} session containers in {{ "{{" }} $labels.namespace {{ "}}" }} exited + with an unexpected status in the last hour. Exit codes 137 and 143 are + excluded: those are the garbage collector stopping an idle session, which + is routine and not a fault. + A cluster of real crashes usually means a broken image was just deployed, + so check what changed before looking at individual sessions. If a specific + language image is involved, the IDE container name in the session detail + dashboard names it. + runbook_url: {{ $runbook }} + {{- if $g }} + dashboard_url: '{{ $g }}/d/eduide-sessions/eduide-sessions?var-namespace={{ "{{" }} $labels.namespace {{ "}}" }}' + {{- end }} + + - alert: EduIDESessionStartupSlow + expr: |- + ( + max by (namespace) (application_application_theiacloud_session_startup_seconds_seconds{namespace=~"{{ $ns }}", quantile="0.95"}) > {{ $t.startupSeconds }} + ) + and + ( + sum by (namespace) (rate(application_application_theiacloud_session_startup_seconds_seconds_count{namespace=~"{{ $ns }}"}[15m])) > 0 + ) + for: 15m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Sessions are slow to start in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + 95th percentile session startup in {{ "{{" }} $labels.namespace {{ "}}" }} is + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} seconds. Students are waiting that long between + clicking through and getting an IDE, which is where they give up and mail + somebody. + The second half of this expression requires sessions to actually be + starting. Without it the alert would latch: these are decaying quantiles + and they hold their last value when idle, so one slow start on a Friday + evening would alert all weekend. + An empty warm pool is the usual cause - check EduIDEWarmPoolEmpty and the + warm pool panel first. Image pulls on a cold node are the other. + runbook_url: {{ $runbook }} + {{- if $g }} + dashboard_url: '{{ $g }}/d/eduide-sessions/eduide-sessions?var-namespace={{ "{{" }} $labels.namespace {{ "}}" }}' + {{- end }} + + - alert: EduIDESessionPodPending + expr: |- + kube_pod_status_phase{namespace=~"{{ $ns }}", {{ $sessions }}, phase="Pending"} == 1 + for: 5m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'A session pod is stuck Pending in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Pod {{ "{{" }} $labels.pod {{ "}}" }} in {{ "{{" }} $labels.namespace {{ "}}" }} has been Pending for + 5 minutes, which means it has not been scheduled onto a node at all. If a + student is waiting on this one, they are looking at a spinner. + Almost always cluster capacity, an unschedulable node, or a workspace + volume that cannot be provisioned. + `kubectl -n {{ "{{" }} $labels.namespace {{ "}}" }} describe pod {{ "{{" }} $labels.pod {{ "}}" }}` + names the reason in its events. + runbook_url: {{ $runbook }} + + - alert: EduIDESessionImagePullFailing + expr: |- + kube_pod_container_status_waiting_reason{namespace=~"{{ $ns }}", reason=~"ImagePullBackOff|ErrImagePull"} == 1 + for: 10m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'An EduIDE image cannot be pulled in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Container {{ "{{" }} $labels.container {{ "}}" }} in {{ "{{" }} $labels.pod {{ "}}" }} + ({{ "{{" }} $labels.namespace {{ "}}" }}) has been in {{ "{{" }} $labels.reason {{ "}}" }} for 10 minutes. + The usual cause is a tag that was never built. A pull request only builds + the images of the repository it came from, so a blanket image tag puts every + other image into this state; the chart has one version knob per source + repository for exactly this reason. Check that the tag the environment asks + for exists in the registry before looking at the cluster. + runbook_url: {{ $runbook }} + + - alert: EduIDESessionEvicted + expr: |- + sum by (namespace) (increase(kube_pod_status_reason{namespace=~"{{ $ns }}", reason="Evicted"}[1h])) > 0 + for: 5m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Pods are being evicted in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} pods were evicted from their nodes in + {{ "{{" }} $labels.namespace {{ "}}" }} in the last hour. Eviction means the node ran out + of a resource the pod was not limited on - almost always ephemeral disk + from build caches and dependency downloads inside sessions. + The students affected lost their session with no warning. Check node disk + pressure, and whether one image's build behaviour changed. + runbook_url: {{ $runbook }} + + - name: eduide.storage + rules: + - alert: EduIDEWorkspaceVolumeFilling + expr: |- + 100 * kubelet_volume_stats_used_bytes{namespace=~"{{ $ns }}"} + / kubelet_volume_stats_capacity_bytes{namespace=~"{{ $ns }}"} > {{ $t.volumePercent }} + for: 15m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'A workspace volume is nearly full in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Workspace volume {{ "{{" }} $labels.persistentvolumeclaim {{ "}}" }} in + {{ "{{" }} $labels.namespace {{ "}}" }} is {{ "{{" }} $value | printf "%.0f" {{ "}}" }}% full. + When it fills completely the IDE starts failing writes with errors that do + not mention the disk - saves fail, git operations fail, and the student + reports it as the editor being broken. + Usually a build cache or a node_modules tree that nobody clears. + runbook_url: {{ $runbook }} + + - alert: EduIDEWorkspaceInodesFilling + expr: |- + 100 * kubelet_volume_stats_inodes_used{namespace=~"{{ $ns }}"} + / kubelet_volume_stats_inodes{namespace=~"{{ $ns }}"} > {{ $t.volumePercent }} + for: 15m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'A workspace volume is running out of inodes in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Workspace volume {{ "{{" }} $labels.persistentvolumeclaim {{ "}}" }} in + {{ "{{" }} $labels.namespace {{ "}}" }} has used {{ "{{" }} $value | printf "%.0f" {{ "}}" }}% of its inodes. + This is worth alerting on separately from bytes because it is the one that + surprises people: a single npm install creates tens of thousands of small + files, so a volume that looks half empty starts returning "no space left on + device". Checking free space will not show the problem. + runbook_url: {{ $runbook }} + + - alert: EduIDEPVCPending + expr: |- + kube_persistentvolumeclaim_status_phase{namespace=~"{{ $ns }}", phase="Pending"} == 1 + for: 10m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'A workspace volume will not provision in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + PersistentVolumeClaim {{ "{{" }} $labels.persistentvolumeclaim {{ "}}" }} in + {{ "{{" }} $labels.namespace {{ "}}" }} has been Pending for 10 minutes, so the session + that needs it cannot start. + The storage class is the first thing to check - it is a cluster property + set once in clusters/.yaml, and a claim naming a class the cluster + does not offer stays Pending forever without any other symptom. + runbook_url: {{ $runbook }} + +{{- if .Values.monitoring.certManager.enabled }} + - name: eduide.certificates + rules: + - alert: EduIDECertificateExpiringSoon + expr: |- + (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < {{ $t.certExpiryDays }} + for: 1h + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Certificate {{ "{{" }} $labels.name {{ "}}" }} expires in {{ "{{" }} $value | printf "%.0f" {{ "}}" }} days' + description: >- + Certificate {{ "{{" }} $labels.name {{ "}}" }} in {{ "{{" }} $labels.namespace {{ "}}" }} expires in + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} days and has not renewed. + cert-manager renews the ACME certificates on its own, so this firing for one + of those means renewal is failing and needs looking at now rather than on + the day. The webview wildcard is different: it is bought from Harica through + RBG and installed by hand, nothing renews it, and when it lapses every + preview in every session on the cluster breaks at once - reported as "the + IDE is broken", not as a certificate problem. + runbook_url: {{ $runbook }} + + - alert: EduIDECertificateNotReady + expr: |- + certmanager_certificate_ready_status{condition="True"} == 0 + for: 1h + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Certificate {{ "{{" }} $labels.name {{ "}}" }} has not been ready for an hour' + description: >- + Certificate {{ "{{" }} $labels.name {{ "}}" }} in {{ "{{" }} $labels.namespace {{ "}}" }} has been + Ready=False for an hour. Until it issues, the Gateway listener for that + hostname serves whatever the Secret already held - which may be an expired + or entirely unrelated certificate, since Gateway API never checks that a + certificate matches the hostname it is serving. The listener will keep + reporting Programmed=True throughout. + Read down the chain before changing anything: CertificateRequest, then + Order, then Challenge. A challenge stuck with a 404 means the name has no + listener; a DNS or connection error means it does not resolve here at all. + runbook_url: {{ $runbook }} +{{- end }} +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml b/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml new file mode 100644 index 0000000..bc58c80 --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.monitoring.enabled .Values.monitoring.certManager.enabled }} +{{- /* +Scrape cert-manager, so certificates can be alerted on before they lapse. + +cert-manager exports the expiry timestamp and readiness of every Certificate it +manages, but ships no ServiceMonitor, so on a stock install nothing collects +them. The result is a cluster where a certificate can be failing to renew for +months with nothing to notice - which is how test3 served a certificate for the +wrong hostnames for 184 days. + +cert-manager is platform-owned, so this only creates the ServiceMonitor. It does +not install, configure or upgrade cert-manager itself. +*/}} +{{- $cm := .Values.monitoring.certManager }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: eduide-cert-manager + namespace: {{ .Values.monitoring.namespace }} + labels: + app.kubernetes.io/part-of: eduide +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ $cm.serviceName }} + app.kubernetes.io/component: controller + namespaceSelector: + matchNames: + - {{ $cm.namespace }} + endpoints: + - port: {{ $cm.portName }} + interval: 60s + honorLabels: true +{{- end }} diff --git a/charts/eduide-cluster/values-example.yaml b/charts/eduide-cluster/values-example.yaml index 10d7e45..d8988e3 100644 --- a/charts/eduide-cluster/values-example.yaml +++ b/charts/eduide-cluster/values-example.yaml @@ -121,9 +121,35 @@ wildcardTLSSecret: # --------------------------------------------------------------------------- # Needs the Prometheus Operator CRDs, and both namespaces must already exist and # be watched by your Prometheus and Grafana. The defaults are Rancher's names. +# Monitoring and alerting are switched on here, unlike most of this file's +# defaults, because CI renders the chart from this example and validates the +# result against the CRD schemas. Left off, the PrometheusRule, the +# AlertmanagerConfig and every dashboard would render to nothing and would never +# be schema-checked - which is exactly how the dashboards came to select on a +# label that had not existed since 2.0.0. monitoring: - enabled: false - # namespace: monitoring - # dashboardNamespace: monitoring - # targetNamespaces: [eduide-prod] - # sessionNamespaces: [eduide-prod] + enabled: true + namespace: monitoring + dashboardNamespace: monitoring + targetNamespaces: [eduide-prod] + certManager: + enabled: true + alerting: + enabled: true + namespace: eduide-system + grafanaUrl: https://grafana.example.org + webhookSecret: + create: true + # Base64 of a placeholder URL. A real deployment gets these from a GitHub + # Environment secret; a webhook URL is a credential and never lives here. + data: + slack-platform: aHR0cHM6Ly9leGFtcGxlLm9yZy9zbGFjay13ZWJob29r + discord-platform: aHR0cHM6Ly9leGFtcGxlLm9yZy9kaXNjb3JkLXdlYmhvb2s= + channels: + - name: platform-slack + type: slack + secretKey: slack-platform + channel: "#eduide-alerts" + - name: platform-discord + type: discord + secretKey: discord-platform diff --git a/charts/eduide-cluster/values.yaml b/charts/eduide-cluster/values.yaml index 30d5289..617602d 100644 --- a/charts/eduide-cluster/values.yaml +++ b/charts/eduide-cluster/values.yaml @@ -205,5 +205,90 @@ monitoring: # -- Namespace the Grafana dashboard ConfigMaps go in. Must already exist and # be watched by Grafana's sidecar. `cattle-dashboards` is Rancher's. dashboardNamespace: cattle-dashboards + # -- Namespaces to watch. Bootstrap derives this from the environments on the + # cluster; it is also the namespace list every dashboard's picker offers. targetNamespaces: [] - sessionNamespaces: [] + + # Scrape cert-manager, so certificate expiry can be alerted on. + # + # cert-manager exports `certmanager_certificate_expiration_timestamp_seconds` + # but ships no ServiceMonitor, so by default nothing watches it. The webview + # wildcard is renewed by hand once a year and nothing warns anyone; when it + # lapses, every preview in every session on the cluster breaks at once and it + # gets reported as "the IDE is broken". + certManager: + # -- Create a ServiceMonitor for cert-manager's controller metrics. + enabled: false + # -- Namespace cert-manager runs in. + namespace: cert-manager + # -- Name of cert-manager's metrics Service. + serviceName: cert-manager + # -- Name of the metrics port on that Service. + portName: http-metrics + + # Alerting: a PrometheusRule plus an AlertmanagerConfig with the channels to + # notify. Independent of `monitoring.enabled` only in the sense that it needs + # it - alerting off with monitoring on is fine, the reverse is not. + alerting: + # -- Create the PrometheusRule and the AlertmanagerConfig. + enabled: false + # -- Namespace both resources go in. + # + # This is also the `namespace` label every EduIDE alert carries, and it is + # a routing artifact rather than the environment the alert is about. The + # Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to + # `OnNamespace`, which prepends `namespace = ` to every route generated from it, so an alert must claim this + # namespace to reach a receiver at all. + # + # The environment an alert is about is in `eduide_namespace`. Silence on + # that label, never on `namespace` - `namespace` is identical across every + # EduIDE alert and silencing it silences all of them. + namespace: eduide-system + # -- Lowest severity that reaches the notification channels: `warning` or + # `critical`. Anything below stays in Alertmanager and on the dashboards. + # This is what keeps the channels worth reading. + minSeverity: warning + # -- How long to wait for more alerts in a group before the first notification. + groupWait: 30s + # -- How long to wait before notifying about new alerts added to a group. + groupInterval: 5m + # -- How long before an unresolved alert is sent again. + repeatInterval: 4h + # -- Base URL of the Grafana that serves the EduIDE dashboards, with no + # trailing slash. Used to deep-link a notification straight to the affected + # session. Left empty, notifications carry no dashboard link. + grafanaUrl: "" + # -- Base URL of the operations runbooks, with no trailing slash. + runbookUrl: https://eduide.github.io/Docs/admins/operations + # The Secret holding the webhook URLs. Never put a webhook URL in a values + # file or on a `--set` - it goes in the process list and in Actions debug + # logs. Bootstrap writes this Secret from GitHub Environment secrets. + webhookSecret: + # -- Create the Secret from `data` below. Off means the Secret already + # exists and is referenced by name only. + create: false + # -- Name of the Secret channels read their URL from. + name: eduide-alert-webhooks + # -- Base64-encoded webhook URLs, keyed by the `secretKey` a channel + # names. Supplied by the workflow, not committed. + data: {} + # -- Where to send alerts. Each entry needs `name`, `type` (`slack` or + # `discord`) and `secretKey`; Slack entries may also set `channel`. An + # empty list with alerting enabled means alerts fire but notify nobody, so + # the chart fails the render instead. + channels: [] + # Thresholds, so tuning noise down does not need a chart release. + thresholds: + # -- Restarts of a platform container within 15m before it counts as crash looping. + componentRestarts: 3 + # -- Session OOM kills per hour in one environment before alerting. + sessionOOMPerHour: 3 + # -- Session crashes per hour in one environment before alerting. + sessionCrashPerHour: 5 + # -- p95 session startup, in seconds, that counts as too slow. + startupSeconds: 90 + # -- Workspace volume fill percentage that counts as nearly full. + volumePercent: 85 + # -- Days before certificate expiry to start warning. + certExpiryDays: 21 From 799e66c2bd1fffe24391db292d1ee95e2c660427 Mon Sep 17 00:00:00 2001 From: Matthias Linhuber Date: Fri, 28 Aug 2026 19:22:39 +0200 Subject: [PATCH 2/3] fix: count OOM kills off a counter, and scope three over-broad queries Review feedback. EduIDESessionOOMKillSpike ran increase() over kube_pod_container_status_last_terminated_reason, which Prometheus reports as a gauge holding only the most recent reason: a container that OOMs ten times reports the same 1 throughout, so the alert counted nothing reliable. It now counts kube_pod_container_status_restarts_total, a real counter, joined with and on() to containers whose last exit was an OOM kill - the same shape EduIDESessionCrashSpike already used. The session detail dashboard's workspace panel took a namespace-wide max. kubelet_volume_stats_* is keyed by claim rather than by pod, so on a dashboard scoped to one session it could report another student's full volume as this session's. Joined through kube_pod_spec_volumes_persistentvolumeclaims_info. EduIDEServiceScrapeDown had no namespace selector, so it could alert on a job outside the namespaces this chart monitors. Alerting enabled with monitoring disabled rendered neither the PrometheusRule nor the AlertmanagerConfig and reported success, which is the failure mode this whole change exists to remove. It now fails with a sentence, checked before the guard rather than inside it, where a configuration that renders nothing could not report itself. certManager.serviceName was inserted into ServiceMonitor.spec.selector.matchLabels, where it matches a label value and not metadata.name, so the name suggested something the field could not do. Replaced with certManager.selectorLabels. Both changed alert expressions were re-run against the live Prometheus: healthy zeros for the EduIDE namespaces, and the OOM join returns three series unrestricted, which is what proves the join itself works. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019qeiQRFu8xAMRYWPdZewjG --- charts/eduide-cluster/README.md | 4 ++-- charts/eduide-cluster/templates/_helpers.tpl | 15 ++++++++++++++ .../monitoring/dashboard-session-detail.yaml | 4 ++-- .../templates/monitoring/prometheusrule.yaml | 20 +++++++++++++++++-- .../servicemonitor-certmanager.yaml | 10 ++++++++-- charts/eduide-cluster/values.yaml | 15 ++++++++++++-- 6 files changed, 58 insertions(+), 10 deletions(-) diff --git a/charts/eduide-cluster/README.md b/charts/eduide-cluster/README.md index d6fea83..7c612e5 100644 --- a/charts/eduide-cluster/README.md +++ b/charts/eduide-cluster/README.md @@ -50,7 +50,7 @@ cert-manager issuers. Install once per cluster, before any eduide release. | managedCertificates.enabled | bool | `false` | | | managedCertificates.issuerRef.kind | string | `"ClusterIssuer"` | | | managedCertificates.issuerRef.name | string | `"letsencrypt-prod"` | | -| monitoring | object | `{"alerting":{"channels":[],"enabled":false,"grafanaUrl":"","groupInterval":"5m","groupWait":"30s","minSeverity":"warning","namespace":"eduide-system","repeatInterval":"4h","runbookUrl":"https://eduide.github.io/Docs/admins/operations","thresholds":{"certExpiryDays":21,"componentRestarts":3,"sessionCrashPerHour":5,"sessionOOMPerHour":3,"startupSeconds":90,"volumePercent":85},"webhookSecret":{"create":false,"data":{},"name":"eduide-alert-webhooks"}},"certManager":{"enabled":false,"namespace":"cert-manager","portName":"http-metrics","serviceName":"cert-manager"},"dashboardNamespace":"cattle-dashboards","enabled":false,"namespace":"cattle-monitoring-system","targetNamespaces":[]}` | ------------------------------------------------------------------------ | +| monitoring | object | `{"alerting":{"channels":[],"enabled":false,"grafanaUrl":"","groupInterval":"5m","groupWait":"30s","minSeverity":"warning","namespace":"eduide-system","repeatInterval":"4h","runbookUrl":"https://eduide.github.io/Docs/admins/operations","thresholds":{"certExpiryDays":21,"componentRestarts":3,"sessionCrashPerHour":5,"sessionOOMPerHour":3,"startupSeconds":90,"volumePercent":85},"webhookSecret":{"create":false,"data":{},"name":"eduide-alert-webhooks"}},"certManager":{"enabled":false,"namespace":"cert-manager","portName":"http-metrics","selectorLabels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/name":"cert-manager"}},"dashboardNamespace":"cattle-dashboards","enabled":false,"namespace":"cattle-monitoring-system","targetNamespaces":[]}` | ------------------------------------------------------------------------ | | monitoring.alerting.channels | list | `[]` | Where to send alerts. Each entry needs `name`, `type` (`slack` or `discord`) and `secretKey`; Slack entries may also set `channel`. An empty list with alerting enabled means alerts fire but notify nobody, so the chart fails the render instead. | | monitoring.alerting.enabled | bool | `false` | Create the PrometheusRule and the AlertmanagerConfig. | | monitoring.alerting.grafanaUrl | string | `""` | Base URL of the Grafana that serves the EduIDE dashboards, with no trailing slash. Used to deep-link a notification straight to the affected session. Left empty, notifications carry no dashboard link. | @@ -72,7 +72,7 @@ cert-manager issuers. Install once per cluster, before any eduide release. | monitoring.certManager.enabled | bool | `false` | Create a ServiceMonitor for cert-manager's controller metrics. | | monitoring.certManager.namespace | string | `"cert-manager"` | Namespace cert-manager runs in. | | monitoring.certManager.portName | string | `"http-metrics"` | Name of the metrics port on that Service. | -| monitoring.certManager.serviceName | string | `"cert-manager"` | Name of cert-manager's metrics Service. | +| monitoring.certManager.selectorLabels | object | `{"app.kubernetes.io/component":"controller","app.kubernetes.io/name":"cert-manager"}` | Labels identifying cert-manager's controller Service. A ServiceMonitor selects on Service **labels**; there is no matcher for `metadata.name`. These are the labels cert-manager's own chart sets, and they are values because a release installed under a different name keeps the labels and changes their values. Get them wrong and the ServiceMonitor is created with no targets and reports no error. Check with: kubectl -n cert-manager get svc cert-manager --show-labels | | monitoring.dashboardNamespace | string | `"cattle-dashboards"` | Namespace the Grafana dashboard ConfigMaps go in. Must already exist and be watched by Grafana's sidecar. `cattle-dashboards` is Rancher's. | | monitoring.enabled | bool | `false` | Create the PodMonitors and Grafana dashboards. Off by default because it is not portable: it needs the Prometheus Operator CRDs (`monitoring.coreos.com/v1`) to exist, and the two namespaces below are Rancher's. Helm does not create namespaces it was not told to, so on a cluster without them the install fails outright on the dashboards. Turn it on once you know where your Prometheus and Grafana look for these. | | monitoring.namespace | string | `"cattle-monitoring-system"` | Namespace the PodMonitors go in. Must be somewhere your Prometheus discovers. `cattle-monitoring-system` is Rancher's; change it for any other monitoring stack. | diff --git a/charts/eduide-cluster/templates/_helpers.tpl b/charts/eduide-cluster/templates/_helpers.tpl index 06b0eae..44ac463 100644 --- a/charts/eduide-cluster/templates/_helpers.tpl +++ b/charts/eduide-cluster/templates/_helpers.tpl @@ -21,6 +21,21 @@ not silently fall back to `.*` and start graphing other tenants' namespaces. {{- end -}} {{- end -}} +{{/* +Alerting depends on monitoring, and saying so beats rendering nothing. + +Both the PrometheusRule and the AlertmanagerConfig are gated on +`monitoring.enabled` as well as `monitoring.alerting.enabled`, because alerts +built on metrics nobody scrapes are decoration. Asking for alerting with +monitoring off used to render neither resource and succeed, which is the worst +outcome: the install reports fine and the feature is absent. +*/}} +{{- define "eduide.checkAlertingPrerequisites" -}} +{{- if and .Values.monitoring.alerting.enabled (not .Values.monitoring.enabled) -}} +{{- fail "monitoring.alerting.enabled is true but monitoring.enabled is false - alerting needs the PodMonitors and the metrics they collect, so this would install nothing at all" -}} +{{- end -}} +{{- end -}} + {{/* The labels every EduIDE alert carries. diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml index a51a0d1..4d6504f 100644 --- a/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml +++ b/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml @@ -344,7 +344,7 @@ data: "type": "prometheus", "uid": "prometheus" }, - "description": "Percentage of the workspace volume in use, if this session has a persistent one.", + "description": "Percentage of this session's own workspace volume in use, if it has a persistent one. Joined to the pod through kube_pod_spec_volumes_persistentvolumeclaims_info: kubelet_volume_stats_* is keyed by claim, not by pod, so an unjoined max would report the largest volume anywhere in the namespace and attribute another student's full disk to this session.", "gridPos": { "h": 5, "w": 6, @@ -359,7 +359,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max(100 * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"})", + "expr": "max(100 * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"} and on (namespace, persistentvolumeclaim) kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=~\"$namespace\", pod=~\"$pod\"})", "format": "time_series", "instant": false, "legendFormat": "", diff --git a/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml b/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml index 3d68155..0e2ed11 100644 --- a/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml +++ b/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml @@ -1,3 +1,8 @@ +{{- /* +Called before the guard below, deliberately: inside it, a misconfiguration that +renders nothing could not report itself. +*/}} +{{- include "eduide.checkAlertingPrerequisites" . }} {{- if and .Values.monitoring.enabled .Values.monitoring.alerting.enabled }} {{- /* EduIDE's alerts. @@ -113,7 +118,7 @@ spec: - alert: EduIDEServiceScrapeDown expr: |- - up{job=~".*theia-cloud-service"} == 0 + up{job=~".*theia-cloud-service", namespace=~"{{ $ns }}"} == 0 for: 10m labels: severity: critical @@ -154,10 +159,21 @@ spec: - name: eduide.sessions rules: + {{- /* + Counted off the restart counter, not off the termination reason. + `kube_pod_container_status_last_terminated_reason` is a gauge holding + only the most recent reason, so increase() over it counts nothing + reliable: a container that OOMs ten times reports the same 1 throughout. + `kube_pod_container_status_restarts_total` is a real counter, and + `and on(...)` narrows it to containers whose last exit was an OOM kill. + Same shape as EduIDESessionCrashSpike below, deliberately. + */}} - alert: EduIDESessionOOMKillSpike expr: |- sum by (namespace) ( - increase(kube_pod_container_status_last_terminated_reason{namespace=~"{{ $ns }}", {{ $sessions }}, reason="OOMKilled"}[1h]) + increase(kube_pod_container_status_restarts_total{namespace=~"{{ $ns }}", {{ $sessions }}}[1h]) + and on (namespace, pod, container) + (kube_pod_container_status_last_terminated_reason{namespace=~"{{ $ns }}", {{ $sessions }}, reason="OOMKilled"} == 1) ) > {{ $t.sessionOOMPerHour }} for: 10m labels: diff --git a/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml b/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml index bc58c80..c3a5e3e 100644 --- a/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml +++ b/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml @@ -20,10 +20,16 @@ metadata: labels: app.kubernetes.io/part-of: eduide spec: + {{- /* + Selected by label, not by name. A ServiceMonitor's selector matches Service + *labels*; there is no name matcher. The values are configurable because a + cert-manager installed under a different release name keeps these labels but + changes their values, and a selector that matches nothing produces a + ServiceMonitor with no targets and no error. + */}} selector: matchLabels: - app.kubernetes.io/name: {{ $cm.serviceName }} - app.kubernetes.io/component: controller + {{- toYaml $cm.selectorLabels | nindent 6 }} namespaceSelector: matchNames: - {{ $cm.namespace }} diff --git a/charts/eduide-cluster/values.yaml b/charts/eduide-cluster/values.yaml index 617602d..1404dd8 100644 --- a/charts/eduide-cluster/values.yaml +++ b/charts/eduide-cluster/values.yaml @@ -221,8 +221,19 @@ monitoring: enabled: false # -- Namespace cert-manager runs in. namespace: cert-manager - # -- Name of cert-manager's metrics Service. - serviceName: cert-manager + # -- Labels identifying cert-manager's controller Service. + # + # A ServiceMonitor selects on Service **labels**; there is no matcher for + # `metadata.name`. These are the labels cert-manager's own chart sets, and + # they are values because a release installed under a different name keeps + # the labels and changes their values. Get them wrong and the ServiceMonitor + # is created with no targets and reports no error. + # + # Check with: + # kubectl -n cert-manager get svc cert-manager --show-labels + selectorLabels: + app.kubernetes.io/name: cert-manager + app.kubernetes.io/component: controller # -- Name of the metrics port on that Service. portName: http-metrics From 6fb46f8aa47541ce314c8f990cc658945525c2b3 Mon Sep 17 00:00:00 2001 From: Matthias Linhuber Date: Fri, 28 Aug 2026 19:33:14 +0200 Subject: [PATCH 3/3] feat: let a channel be scoped to one installation One cluster can host installations that belong to different people. Bonn and Mannheim both live on the eduide cluster and each has its own Discord, so a single receiver would have sent each of them the other's incidents. A channel may now name `environments`, a list of namespaces. Each scoped channel gets a sub-route matched on eduide_namespace, and first match wins. Anything no scoped channel claims goes to every channel, deliberately. A certificate expiring or the conversion webhook failing is cluster-scoped and belongs to no tenant namespace; dropping those for failing to match a tenant route would silently lose the alerts that matter most. The receiver bodies moved into _helpers.tpl. A Go template `define` may not sit inside an `if`, and the whole AlertmanagerConfig is guarded by one - sharing the definition also means the message format cannot drift between the catch-all receiver and the scoped ones. Channel names are now checked for uniqueness, since they become receiver names. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019qeiQRFu8xAMRYWPdZewjG --- charts/eduide-cluster/README.md | 2 +- charts/eduide-cluster/templates/_helpers.tpl | 44 +++++++ .../monitoring/alertmanagerconfig.yaml | 113 ++++++++++-------- charts/eduide-cluster/values.yaml | 21 ++++ 4 files changed, 130 insertions(+), 50 deletions(-) diff --git a/charts/eduide-cluster/README.md b/charts/eduide-cluster/README.md index 7c612e5..72da256 100644 --- a/charts/eduide-cluster/README.md +++ b/charts/eduide-cluster/README.md @@ -51,7 +51,7 @@ cert-manager issuers. Install once per cluster, before any eduide release. | managedCertificates.issuerRef.kind | string | `"ClusterIssuer"` | | | managedCertificates.issuerRef.name | string | `"letsencrypt-prod"` | | | monitoring | object | `{"alerting":{"channels":[],"enabled":false,"grafanaUrl":"","groupInterval":"5m","groupWait":"30s","minSeverity":"warning","namespace":"eduide-system","repeatInterval":"4h","runbookUrl":"https://eduide.github.io/Docs/admins/operations","thresholds":{"certExpiryDays":21,"componentRestarts":3,"sessionCrashPerHour":5,"sessionOOMPerHour":3,"startupSeconds":90,"volumePercent":85},"webhookSecret":{"create":false,"data":{},"name":"eduide-alert-webhooks"}},"certManager":{"enabled":false,"namespace":"cert-manager","portName":"http-metrics","selectorLabels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/name":"cert-manager"}},"dashboardNamespace":"cattle-dashboards","enabled":false,"namespace":"cattle-monitoring-system","targetNamespaces":[]}` | ------------------------------------------------------------------------ | -| monitoring.alerting.channels | list | `[]` | Where to send alerts. Each entry needs `name`, `type` (`slack` or `discord`) and `secretKey`; Slack entries may also set `channel`. An empty list with alerting enabled means alerts fire but notify nobody, so the chart fails the render instead. | +| monitoring.alerting.channels | list | `[]` | Where to send alerts. Each entry needs `name`, `type` (`slack` or `discord`) and `secretKey`; Slack entries may also set `channel`. An empty list with alerting enabled means alerts fire but notify nobody, so the chart fails the render instead. A channel may set `environments: [ns, ...]` to receive only that installation's alerts. One cluster can host installations belonging to different people - Bonn and Mannheim share a cluster and each has its own Discord - and without scoping, both would see the other's incidents. Anything no scoped channel claims goes to **every** channel. That is how cluster-scoped alerts still get out: a certificate expiring or the conversion webhook failing is not about any one tenant's namespace, and dropping it for failing to match a tenant route would lose exactly the alerts that matter most. channels: - name: mannheim type: discord secretKey: discord-mannheim environments: [eduide-mannheim] - name: platform type: slack secretKey: slack-platform channel: "#eduide-alerts" | | monitoring.alerting.enabled | bool | `false` | Create the PrometheusRule and the AlertmanagerConfig. | | monitoring.alerting.grafanaUrl | string | `""` | Base URL of the Grafana that serves the EduIDE dashboards, with no trailing slash. Used to deep-link a notification straight to the affected session. Left empty, notifications carry no dashboard link. | | monitoring.alerting.groupInterval | string | `"5m"` | How long to wait before notifying about new alerts added to a group. | diff --git a/charts/eduide-cluster/templates/_helpers.tpl b/charts/eduide-cluster/templates/_helpers.tpl index 44ac463..6d7eddd 100644 --- a/charts/eduide-cluster/templates/_helpers.tpl +++ b/charts/eduide-cluster/templates/_helpers.tpl @@ -55,3 +55,47 @@ silence all of them at once. namespace: {{ .Values.monitoring.alerting.namespace }} eduide_namespace: '{{ "{{" }} $labels.namespace {{ "}}" }}' {{- end -}} + +{{/* +The receiver bodies for a set of channels. + +Shared between the catch-all receiver and each environment-scoped one, so a +change to the message format cannot apply to one and not the other. Lives here +rather than in the template because a `define` may not sit inside an `if`, and +the whole AlertmanagerConfig is guarded by one. +*/}} +{{- define "eduide.receiverConfigs" }} +{{- $ctx := .ctx }} +{{- $a := $ctx.Values.monitoring.alerting }} +{{- $slack := list }} +{{- $discord := list }} +{{- range .channels }} + {{- if eq (.type | toString) "slack" }}{{ $slack = append $slack . }}{{ else }}{{ $discord = append $discord . }}{{ end }} +{{- end }} +{{- if $slack }} + slackConfigs: + {{- range $slack }} + - apiURL: + name: {{ $a.webhookSecret.name }} + key: {{ .secretKey }} + {{- if .channel }} + channel: {{ .channel | quote }} + {{- end }} + sendResolved: true + username: EduIDE + title: {{ $.title | quote }} + text: {{ $.text | quote }} + {{- end }} +{{- end }} +{{- if $discord }} + discordConfigs: + {{- range $discord }} + - apiURL: + name: {{ $a.webhookSecret.name }} + key: {{ .secretKey }} + sendResolved: true + title: {{ $.title | quote }} + message: {{ $.text | quote }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml b/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml index 373511b..68949d4 100644 --- a/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml +++ b/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml @@ -1,9 +1,10 @@ +{{- include "eduide.checkAlertingPrerequisites" . }} {{- if and .Values.monitoring.enabled .Values.monitoring.alerting.enabled }} {{- $a := .Values.monitoring.alerting }} {{- /* Where EduIDE's alerts go. -Two things about this file are not obvious. +Three things about this file are not obvious. **The route matches on `namespace`, and that label is a routing artifact.** The Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to @@ -14,6 +15,17 @@ Grouping and any inhibition rule must use `eduide_namespace`; grouping on `namespace` would merge an outage in production with one in staging into a single notification, because the label is identical across all of them. +**A channel may be scoped to environments.** Several installations can share one +cluster and belong to different people: Bonn and Mannheim both live on the +`eduide` cluster and each has its own Discord. A channel naming `environments` +gets a sub-route matching those namespaces, and first match wins. + +**Anything not claimed by a scoped channel goes to every channel.** The +top-level receiver holds all of them deliberately. Cluster-scoped alerts - the +conversion webhook being down, a certificate about to expire - are not about any +one tenant's namespace, and dropping them because they failed to match a tenant +route would silently lose exactly the alerts that matter most. + **Only warning and above is notified.** Everything fires and is visible in Alertmanager and on the dashboards, but the channels get a filtered subset. A channel that receives every capacity blip stops being read, and then the @@ -28,37 +40,20 @@ platform is unmonitored no matter how many rules exist. {{- else if ne ($a.minSeverity | toString) "critical" }} {{- fail (printf "monitoring.alerting.minSeverity must be \"warning\" or \"critical\", got %q" ($a.minSeverity | toString)) }} {{- end }} -apiVersion: monitoring.coreos.com/v1alpha1 -kind: AlertmanagerConfig -metadata: - name: eduide-alerts - namespace: {{ $a.namespace }} - labels: - app.kubernetes.io/part-of: eduide -spec: - route: - receiver: eduide-channels - groupBy: - - alertname - - eduide_namespace - groupWait: {{ $a.groupWait }} - groupInterval: {{ $a.groupInterval }} - repeatInterval: {{ $a.repeatInterval }} - matchers: - - name: severity - matchType: =~ - value: {{ join "|" $severities | quote }} - receivers: - - name: eduide-channels {{- $slack := list }} {{- $discord := list }} +{{- $scoped := list }} +{{- $names := dict }} {{- range $i, $c := $a.channels }} {{- if not $c.name }}{{ fail (printf "monitoring.alerting.channels[%d] has no name" $i) }}{{ end }} + {{- if hasKey $names $c.name }}{{ fail (printf "two channels are both called %q; receiver names must be unique" $c.name) }}{{ end }} + {{- $_ := set $names $c.name true }} {{- if not $c.secretKey }}{{ fail (printf "channel %q names no secretKey - webhook URLs are only ever read from a Secret, never from values" $c.name) }}{{ end }} {{- $type := $c.type | default "" | toString }} {{- if eq $type "slack" }}{{ $slack = append $slack $c }} {{- else if eq $type "discord" }}{{ $discord = append $discord $c }} {{- else }}{{ fail (printf "channel %q has type %q; supported types are \"slack\" and \"discord\"" $c.name $type) }}{{ end }} + {{- if $c.environments }}{{ $scoped = append $scoped $c }}{{ end }} {{- end }} {{- /* The message body. Alertmanager templating, not Helm's - the doubled braces are @@ -74,30 +69,50 @@ Severity: {{ .Labels.severity }} | Since: {{ .StartsAt.Format "15:04 MST" }} {{ if .Annotations.runbook_url }}Runbook: {{ .Annotations.runbook_url }} {{ end }}{{ if .Annotations.dashboard_url }}Dashboard: {{ .Annotations.dashboard_url }} {{ end }}{{ end }}` }} -{{- if $slack }} - slackConfigs: - {{- range $slack }} - - apiURL: - name: {{ $a.webhookSecret.name }} - key: {{ .secretKey }} - {{- if .channel }} - channel: {{ .channel | quote }} - {{- end }} - sendResolved: true - username: EduIDE - title: '{{ `{{ if eq .Status "firing" }}[FIRING {{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ .GroupLabels.alertname }}{{ if .GroupLabels.eduide_namespace }} - {{ .GroupLabels.eduide_namespace }}{{ end }}` }}' - text: {{ $text | quote }} - {{- end }} -{{- end }} -{{- if $discord }} - discordConfigs: - {{- range $discord }} - - apiURL: - name: {{ $a.webhookSecret.name }} - key: {{ .secretKey }} - sendResolved: true - title: '{{ `{{ if eq .Status "firing" }}[FIRING {{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ .GroupLabels.alertname }}{{ if .GroupLabels.eduide_namespace }} - {{ .GroupLabels.eduide_namespace }}{{ end }}` }}' - message: {{ $text | quote }} - {{- end }} -{{- end }} +{{- $title := `{{ if eq .Status "firing" }}[FIRING {{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ .GroupLabels.alertname }}{{ if .GroupLabels.eduide_namespace }} - {{ .GroupLabels.eduide_namespace }}{{ end }}` }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: eduide-alerts + namespace: {{ $a.namespace }} + labels: + app.kubernetes.io/part-of: eduide +spec: + route: + receiver: eduide-all + groupBy: + - alertname + - eduide_namespace + groupWait: {{ $a.groupWait }} + groupInterval: {{ $a.groupInterval }} + repeatInterval: {{ $a.repeatInterval }} + matchers: + - name: severity + matchType: =~ + value: {{ join "|" $severities | quote }} + {{- if $scoped }} + routes: + {{- range $scoped }} + - receiver: {{ printf "eduide-%s" .name }} + groupBy: + - alertname + - eduide_namespace + groupWait: {{ $a.groupWait }} + groupInterval: {{ $a.groupInterval }} + repeatInterval: {{ $a.repeatInterval }} + matchers: + - name: eduide_namespace + matchType: =~ + value: {{ printf "^(%s)$" (join "|" .environments) | quote }} + {{- end }} + {{- end }} + receivers: + {{- /* Every channel. Reached by anything no scoped route claimed, which is + how cluster-scoped alerts still get out. */}} + - name: eduide-all +{{- include "eduide.receiverConfigs" (dict "ctx" $ "channels" $a.channels "title" $title "text" $text) }} + {{- range $scoped }} + - name: {{ printf "eduide-%s" .name }} +{{- include "eduide.receiverConfigs" (dict "ctx" $ "channels" (list .) "title" $title "text" $text) }} + {{- end }} {{- end }} diff --git a/charts/eduide-cluster/values.yaml b/charts/eduide-cluster/values.yaml index 1404dd8..5676420 100644 --- a/charts/eduide-cluster/values.yaml +++ b/charts/eduide-cluster/values.yaml @@ -288,6 +288,27 @@ monitoring: # `discord`) and `secretKey`; Slack entries may also set `channel`. An # empty list with alerting enabled means alerts fire but notify nobody, so # the chart fails the render instead. + # + # A channel may set `environments: [ns, ...]` to receive only that + # installation's alerts. One cluster can host installations belonging to + # different people - Bonn and Mannheim share a cluster and each has its own + # Discord - and without scoping, both would see the other's incidents. + # + # Anything no scoped channel claims goes to **every** channel. That is how + # cluster-scoped alerts still get out: a certificate expiring or the + # conversion webhook failing is not about any one tenant's namespace, and + # dropping it for failing to match a tenant route would lose exactly the + # alerts that matter most. + # + # channels: + # - name: mannheim + # type: discord + # secretKey: discord-mannheim + # environments: [eduide-mannheim] + # - name: platform + # type: slack + # secretKey: slack-platform + # channel: "#eduide-alerts" channels: [] # Thresholds, so tuning noise down does not need a chart release. thresholds: