diff --git a/charts/eduide-cluster/Chart.yaml b/charts/eduide-cluster/Chart.yaml index e403beb..91fa447 100644 --- a/charts/eduide-cluster/Chart.yaml +++ b/charts/eduide-cluster/Chart.yaml @@ -4,5 +4,5 @@ description: | Cluster-scoped half of EduIDE: CRDs, the conversion webhook, ClusterRoles and cert-manager issuers. Install once per cluster, before any eduide release. type: application -version: 2.1.5 +version: 2.2.0 appVersion: "1.2.0" diff --git a/charts/eduide-cluster/README.md b/charts/eduide-cluster/README.md index e088a74..72da256 100644 --- a/charts/eduide-cluster/README.md +++ b/charts/eduide-cluster/README.md @@ -1,6 +1,6 @@ # eduide-cluster -![Version: 2.1.5](https://img.shields.io/badge/Version-2.1.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) +![Version: 2.2.0](https://img.shields.io/badge/Version-2.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) Cluster-scoped half of EduIDE: CRDs, the conversion webhook, ClusterRoles and cert-manager issuers. Install once per cluster, before any eduide release. @@ -50,10 +50,33 @@ cert-manager issuers. Install once per cluster, before any eduide release. | managedCertificates.enabled | bool | `false` | | | managedCertificates.issuerRef.kind | string | `"ClusterIssuer"` | | | managedCertificates.issuerRef.name | string | `"letsencrypt-prod"` | | -| monitoring | object | `{"dashboardNamespace":"cattle-dashboards","enabled":false,"namespace":"cattle-monitoring-system","sessionNamespaces":[],"targetNamespaces":[]}` | ------------------------------------------------------------------------ | +| monitoring | object | `{"alerting":{"channels":[],"enabled":false,"grafanaUrl":"","groupInterval":"5m","groupWait":"30s","minSeverity":"warning","namespace":"eduide-system","repeatInterval":"4h","runbookUrl":"https://eduide.github.io/Docs/admins/operations","thresholds":{"certExpiryDays":21,"componentRestarts":3,"sessionCrashPerHour":5,"sessionOOMPerHour":3,"startupSeconds":90,"volumePercent":85},"webhookSecret":{"create":false,"data":{},"name":"eduide-alert-webhooks"}},"certManager":{"enabled":false,"namespace":"cert-manager","portName":"http-metrics","selectorLabels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/name":"cert-manager"}},"dashboardNamespace":"cattle-dashboards","enabled":false,"namespace":"cattle-monitoring-system","targetNamespaces":[]}` | ------------------------------------------------------------------------ | +| monitoring.alerting.channels | list | `[]` | Where to send alerts. Each entry needs `name`, `type` (`slack` or `discord`) and `secretKey`; Slack entries may also set `channel`. An empty list with alerting enabled means alerts fire but notify nobody, so the chart fails the render instead. A channel may set `environments: [ns, ...]` to receive only that installation's alerts. One cluster can host installations belonging to different people - Bonn and Mannheim share a cluster and each has its own Discord - and without scoping, both would see the other's incidents. Anything no scoped channel claims goes to **every** channel. That is how cluster-scoped alerts still get out: a certificate expiring or the conversion webhook failing is not about any one tenant's namespace, and dropping it for failing to match a tenant route would lose exactly the alerts that matter most. channels: - name: mannheim type: discord secretKey: discord-mannheim environments: [eduide-mannheim] - name: platform type: slack secretKey: slack-platform channel: "#eduide-alerts" | +| monitoring.alerting.enabled | bool | `false` | Create the PrometheusRule and the AlertmanagerConfig. | +| monitoring.alerting.grafanaUrl | string | `""` | Base URL of the Grafana that serves the EduIDE dashboards, with no trailing slash. Used to deep-link a notification straight to the affected session. Left empty, notifications carry no dashboard link. | +| monitoring.alerting.groupInterval | string | `"5m"` | How long to wait before notifying about new alerts added to a group. | +| monitoring.alerting.groupWait | string | `"30s"` | How long to wait for more alerts in a group before the first notification. | +| monitoring.alerting.minSeverity | string | `"warning"` | Lowest severity that reaches the notification channels: `warning` or `critical`. Anything below stays in Alertmanager and on the dashboards. This is what keeps the channels worth reading. | +| monitoring.alerting.namespace | string | `"eduide-system"` | Namespace both resources go in. This is also the `namespace` label every EduIDE alert carries, and it is a routing artifact rather than the environment the alert is about. The Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to `OnNamespace`, which prepends `namespace = ` to every route generated from it, so an alert must claim this namespace to reach a receiver at all. The environment an alert is about is in `eduide_namespace`. Silence on that label, never on `namespace` - `namespace` is identical across every EduIDE alert and silencing it silences all of them. | +| monitoring.alerting.repeatInterval | string | `"4h"` | How long before an unresolved alert is sent again. | +| monitoring.alerting.runbookUrl | string | `"https://eduide.github.io/Docs/admins/operations"` | Base URL of the operations runbooks, with no trailing slash. | +| monitoring.alerting.thresholds.certExpiryDays | int | `21` | Days before certificate expiry to start warning. | +| monitoring.alerting.thresholds.componentRestarts | int | `3` | Restarts of a platform container within 15m before it counts as crash looping. | +| monitoring.alerting.thresholds.sessionCrashPerHour | int | `5` | Session crashes per hour in one environment before alerting. | +| monitoring.alerting.thresholds.sessionOOMPerHour | int | `3` | Session OOM kills per hour in one environment before alerting. | +| monitoring.alerting.thresholds.startupSeconds | int | `90` | p95 session startup, in seconds, that counts as too slow. | +| monitoring.alerting.thresholds.volumePercent | int | `85` | Workspace volume fill percentage that counts as nearly full. | +| monitoring.alerting.webhookSecret.create | bool | `false` | Create the Secret from `data` below. Off means the Secret already exists and is referenced by name only. | +| monitoring.alerting.webhookSecret.data | object | `{}` | Base64-encoded webhook URLs, keyed by the `secretKey` a channel names. Supplied by the workflow, not committed. | +| monitoring.alerting.webhookSecret.name | string | `"eduide-alert-webhooks"` | Name of the Secret channels read their URL from. | +| monitoring.certManager.enabled | bool | `false` | Create a ServiceMonitor for cert-manager's controller metrics. | +| monitoring.certManager.namespace | string | `"cert-manager"` | Namespace cert-manager runs in. | +| monitoring.certManager.portName | string | `"http-metrics"` | Name of the metrics port on that Service. | +| monitoring.certManager.selectorLabels | object | `{"app.kubernetes.io/component":"controller","app.kubernetes.io/name":"cert-manager"}` | Labels identifying cert-manager's controller Service. A ServiceMonitor selects on Service **labels**; there is no matcher for `metadata.name`. These are the labels cert-manager's own chart sets, and they are values because a release installed under a different name keeps the labels and changes their values. Get them wrong and the ServiceMonitor is created with no targets and reports no error. Check with: kubectl -n cert-manager get svc cert-manager --show-labels | | monitoring.dashboardNamespace | string | `"cattle-dashboards"` | Namespace the Grafana dashboard ConfigMaps go in. Must already exist and be watched by Grafana's sidecar. `cattle-dashboards` is Rancher's. | | monitoring.enabled | bool | `false` | Create the PodMonitors and Grafana dashboards. Off by default because it is not portable: it needs the Prometheus Operator CRDs (`monitoring.coreos.com/v1`) to exist, and the two namespaces below are Rancher's. Helm does not create namespaces it was not told to, so on a cluster without them the install fails outright on the dashboards. Turn it on once you know where your Prometheus and Grafana look for these. | | monitoring.namespace | string | `"cattle-monitoring-system"` | Namespace the PodMonitors go in. Must be somewhere your Prometheus discovers. `cattle-monitoring-system` is Rancher's; change it for any other monitoring stack. | +| monitoring.targetNamespaces | list | `[]` | Namespaces to watch. Bootstrap derives this from the environments on the cluster; it is also the namespace list every dashboard's picker offers. | | operatorrole.name | string | `"operator-api-access"` | name for the operator's cluster role | | servicerole.name | string | `"service-api-access"` | name for the services' cluster role | | wildcardTLSSecret.certificate | string | `""` | | diff --git a/charts/eduide-cluster/templates/_helpers.tpl b/charts/eduide-cluster/templates/_helpers.tpl new file mode 100644 index 0000000..6d7eddd --- /dev/null +++ b/charts/eduide-cluster/templates/_helpers.tpl @@ -0,0 +1,101 @@ +{{/* +A RE2 alternation of the namespaces this cluster monitors, anchored, for use in +a PromQL label matcher and in Grafana variable queries. + +The dashboards used to carry a hand-written list of namespaces. It went stale +the moment the environments were renamed - the picker still offered `theia`, +`theia-staging` and `test1`, none of which exist any more, so every panel was +empty whatever you selected. Deriving it from the same list the PodMonitors use +means it cannot drift again. + +An empty list yields `^$`, which matches nothing. That is deliberate: a cluster +where every environment opted out of monitoring should show an empty picker, +not silently fall back to `.*` and start graphing other tenants' namespaces. +*/}} +{{- define "eduide.monitoredNamespaceRegex" -}} +{{- $ns := .Values.monitoring.targetNamespaces | default list -}} +{{- if $ns -}} +^({{ join "|" $ns }})$ +{{- else -}} +^$ +{{- end -}} +{{- end -}} + +{{/* +Alerting depends on monitoring, and saying so beats rendering nothing. + +Both the PrometheusRule and the AlertmanagerConfig are gated on +`monitoring.enabled` as well as `monitoring.alerting.enabled`, because alerts +built on metrics nobody scrapes are decoration. Asking for alerting with +monitoring off used to render neither resource and succeed, which is the worst +outcome: the install reports fine and the feature is absent. +*/}} +{{- define "eduide.checkAlertingPrerequisites" -}} +{{- if and .Values.monitoring.alerting.enabled (not .Values.monitoring.enabled) -}} +{{- fail "monitoring.alerting.enabled is true but monitoring.enabled is false - alerting needs the PodMonitors and the metrics they collect, so this would install nothing at all" -}} +{{- end -}} +{{- end -}} + +{{/* +The labels every EduIDE alert carries. + +`namespace` is a routing label and is NOT the environment the alert is about. +The Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to +`OnNamespace`, which prepends `namespace = ` to every route it generates. Our AlertmanagerConfig lives in one +namespace, so alerts have to claim that namespace or they reach no receiver at +all. + +The environment an alert is actually about is `eduide_namespace`, templated +from the series. Silences and inhibition rules must match on that one; +`namespace` is the same string on every EduIDE alert and matching it would +silence all of them at once. +*/}} +{{- define "eduide.alertRoutingLabels" -}} +namespace: {{ .Values.monitoring.alerting.namespace }} +eduide_namespace: '{{ "{{" }} $labels.namespace {{ "}}" }}' +{{- end -}} + +{{/* +The receiver bodies for a set of channels. + +Shared between the catch-all receiver and each environment-scoped one, so a +change to the message format cannot apply to one and not the other. Lives here +rather than in the template because a `define` may not sit inside an `if`, and +the whole AlertmanagerConfig is guarded by one. +*/}} +{{- define "eduide.receiverConfigs" }} +{{- $ctx := .ctx }} +{{- $a := $ctx.Values.monitoring.alerting }} +{{- $slack := list }} +{{- $discord := list }} +{{- range .channels }} + {{- if eq (.type | toString) "slack" }}{{ $slack = append $slack . }}{{ else }}{{ $discord = append $discord . }}{{ end }} +{{- end }} +{{- if $slack }} + slackConfigs: + {{- range $slack }} + - apiURL: + name: {{ $a.webhookSecret.name }} + key: {{ .secretKey }} + {{- if .channel }} + channel: {{ .channel | quote }} + {{- end }} + sendResolved: true + username: EduIDE + title: {{ $.title | quote }} + text: {{ $.text | quote }} + {{- end }} +{{- end }} +{{- if $discord }} + discordConfigs: + {{- range $discord }} + - apiURL: + name: {{ $a.webhookSecret.name }} + key: {{ .secretKey }} + sendResolved: true + title: {{ $.title | quote }} + message: {{ $.text | quote }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/alert-webhook-secret.yaml b/charts/eduide-cluster/templates/monitoring/alert-webhook-secret.yaml new file mode 100644 index 0000000..c02bd16 --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/alert-webhook-secret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.monitoring.enabled .Values.monitoring.alerting.enabled .Values.monitoring.alerting.webhookSecret.create }} +{{- /* +The webhook URLs the channels post to. + +A Slack or Discord webhook URL is a credential: anyone holding it can post into +the channel. It never goes in a values file in git and never on a `--set`, which +would put it in the process list and in Actions debug logs. Bootstrap reads it +from a GitHub Environment secret and passes it here already base64-encoded, the +same way the wildcard TLS certificate is handled. + +Chart-managed rather than applied alongside, so it is removed when the release +is, and so a channel referring to a key that was never supplied fails the deploy +instead of producing an AlertmanagerConfig that silently notifies nobody. +*/}} +{{- $a := .Values.monitoring.alerting }} +{{- range $i, $c := $a.channels }} +{{- if not (hasKey $a.webhookSecret.data $c.secretKey) }} +{{- fail (printf "channel %q reads key %q from secret %q, but that key was not supplied in monitoring.alerting.webhookSecret.data" $c.name $c.secretKey $a.webhookSecret.name) }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ $a.webhookSecret.name }} + namespace: {{ $a.namespace }} + labels: + app.kubernetes.io/part-of: eduide +data: + {{- range $k, $v := $a.webhookSecret.data }} + {{ $k }}: {{ $v | quote }} + {{- end }} +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml b/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml new file mode 100644 index 0000000..68949d4 --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/alertmanagerconfig.yaml @@ -0,0 +1,118 @@ +{{- include "eduide.checkAlertingPrerequisites" . }} +{{- if and .Values.monitoring.enabled .Values.monitoring.alerting.enabled }} +{{- $a := .Values.monitoring.alerting }} +{{- /* +Where EduIDE's alerts go. + +Three things about this file are not obvious. + +**The route matches on `namespace`, and that label is a routing artifact.** The +Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to +`OnNamespace`, which silently prepends `namespace = {{ $a.namespace }}` to the +route generated from this resource. Every EduIDE alert therefore sets that +label, and carries the environment it is really about in `eduide_namespace`. +Grouping and any inhibition rule must use `eduide_namespace`; grouping on +`namespace` would merge an outage in production with one in staging into a +single notification, because the label is identical across all of them. + +**A channel may be scoped to environments.** Several installations can share one +cluster and belong to different people: Bonn and Mannheim both live on the +`eduide` cluster and each has its own Discord. A channel naming `environments` +gets a sub-route matching those namespaces, and first match wins. + +**Anything not claimed by a scoped channel goes to every channel.** The +top-level receiver holds all of them deliberately. Cluster-scoped alerts - the +conversion webhook being down, a certificate about to expire - are not about any +one tenant's namespace, and dropping them because they failed to match a tenant +route would silently lose exactly the alerts that matter most. + +**Only warning and above is notified.** Everything fires and is visible in +Alertmanager and on the dashboards, but the channels get a filtered subset. A +channel that receives every capacity blip stops being read, and then the +platform is unmonitored no matter how many rules exist. +*/}} +{{- if not $a.channels }} +{{- fail "monitoring.alerting.enabled is true but monitoring.alerting.channels is empty - alerts would fire and notify nobody" }} +{{- end }} +{{- $severities := list "critical" }} +{{- if eq ($a.minSeverity | toString) "warning" }} +{{- $severities = list "critical" "warning" }} +{{- else if ne ($a.minSeverity | toString) "critical" }} +{{- fail (printf "monitoring.alerting.minSeverity must be \"warning\" or \"critical\", got %q" ($a.minSeverity | toString)) }} +{{- end }} +{{- $slack := list }} +{{- $discord := list }} +{{- $scoped := list }} +{{- $names := dict }} +{{- range $i, $c := $a.channels }} + {{- if not $c.name }}{{ fail (printf "monitoring.alerting.channels[%d] has no name" $i) }}{{ end }} + {{- if hasKey $names $c.name }}{{ fail (printf "two channels are both called %q; receiver names must be unique" $c.name) }}{{ end }} + {{- $_ := set $names $c.name true }} + {{- if not $c.secretKey }}{{ fail (printf "channel %q names no secretKey - webhook URLs are only ever read from a Secret, never from values" $c.name) }}{{ end }} + {{- $type := $c.type | default "" | toString }} + {{- if eq $type "slack" }}{{ $slack = append $slack $c }} + {{- else if eq $type "discord" }}{{ $discord = append $discord $c }} + {{- else }}{{ fail (printf "channel %q has type %q; supported types are \"slack\" and \"discord\"" $c.name $type) }}{{ end }} + {{- if $c.environments }}{{ $scoped = append $scoped $c }}{{ end }} +{{- end }} +{{- /* +The message body. Alertmanager templating, not Helm's - the doubled braces are +escapes. Written to stand on its own in a chat client: what broke, what it means +for students, and where to look, without needing the dashboard open. `.Alerts` +is grouped by alertname and environment, so one message covers one problem in +one environment however many pods it touched. +*/}} +{{- $text := `{{ range .Alerts }}*{{ .Annotations.summary }}* +{{ .Annotations.description }} +{{ if .Labels.eduide_namespace }}Environment: {{ .Labels.eduide_namespace }}{{ end }}{{ if .Labels.pod }} | Pod: {{ .Labels.pod }}{{ end }}{{ if .Labels.node }} | Node: {{ .Labels.node }}{{ end }} +Severity: {{ .Labels.severity }} | Since: {{ .StartsAt.Format "15:04 MST" }} +{{ if .Annotations.runbook_url }}Runbook: {{ .Annotations.runbook_url }} +{{ end }}{{ if .Annotations.dashboard_url }}Dashboard: {{ .Annotations.dashboard_url }} +{{ end }}{{ end }}` }} +{{- $title := `{{ if eq .Status "firing" }}[FIRING {{ .Alerts.Firing | len }}]{{ else }}[RESOLVED]{{ end }} {{ .GroupLabels.alertname }}{{ if .GroupLabels.eduide_namespace }} - {{ .GroupLabels.eduide_namespace }}{{ end }}` }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: eduide-alerts + namespace: {{ $a.namespace }} + labels: + app.kubernetes.io/part-of: eduide +spec: + route: + receiver: eduide-all + groupBy: + - alertname + - eduide_namespace + groupWait: {{ $a.groupWait }} + groupInterval: {{ $a.groupInterval }} + repeatInterval: {{ $a.repeatInterval }} + matchers: + - name: severity + matchType: =~ + value: {{ join "|" $severities | quote }} + {{- if $scoped }} + routes: + {{- range $scoped }} + - receiver: {{ printf "eduide-%s" .name }} + groupBy: + - alertname + - eduide_namespace + groupWait: {{ $a.groupWait }} + groupInterval: {{ $a.groupInterval }} + repeatInterval: {{ $a.repeatInterval }} + matchers: + - name: eduide_namespace + matchType: =~ + value: {{ printf "^(%s)$" (join "|" .environments) | quote }} + {{- end }} + {{- end }} + receivers: + {{- /* Every channel. Reached by anything no scoped route claimed, which is + how cluster-scoped alerts still get out. */}} + - name: eduide-all +{{- include "eduide.receiverConfigs" (dict "ctx" $ "channels" $a.channels "title" $title "text" $text) }} + {{- range $scoped }} + - name: {{ printf "eduide-%s" .name }} +{{- include "eduide.receiverConfigs" (dict "ctx" $ "channels" (list .) "title" $title "text" $text) }} + {{- end }} +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml new file mode 100644 index 0000000..4d6504f --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/dashboard-session-detail.yaml @@ -0,0 +1,1215 @@ +{{- if .Values.monitoring.enabled }} +{{- /* +One session in detail, reached from the Oldest sessions table on the +sessions overview or by picking a pod directly. +*/}} +{{- $dashboardNs := .Values.monitoring.dashboardNamespace | toString | trim }} +{{- if not $dashboardNs }} +{{- fail "monitoring.enabled is true but monitoring.dashboardNamespace is empty" }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: eduide-dashboard-session-detail + namespace: {{ $dashboardNs }} +data: + eduide-session-detail.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "One session, in detail. Pick the environment and the pod; every panel is scoped to that pod.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "tags": [], + "targetBlank": false, + "title": "All sessions", + "tooltip": "Back to the sessions overview", + "type": "link", + "url": "/d/eduide-sessions/eduide-sessions" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 200, + "panels": [], + "title": "This session", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "How long this pod has existed. Not how long the student has been working - a warm-pool pod is created before anyone claims it.", + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 201, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - max(kube_pod_created{namespace=~\"$namespace\", pod=~\"$pod\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Age", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Both the IDE and its oauth2-proxy sidecar must be ready before the session serves traffic. 1 of 2 means the proxy is failing, which presents to the student as a login loop or a 403.", + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 202, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_status_ready{namespace=~\"$namespace\", pod=~\"$pod\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Ready containers", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Any restart of a session container loses the student's terminal state and unsaved editor buffers.", + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 203, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"$pod\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Limit minus current working set for the IDE container. When this approaches zero the kernel kills the container.", + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 204, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", resource=\"memory\", container!~\"oauth.*\"}) - sum(container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"$pod\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Memory headroom", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 209715200 + }, + { + "color": "green", + "value": 524288000 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percentage of this session's own workspace volume in use, if it has a persistent one. Joined to the pod through kube_pod_spec_volumes_persistentvolumeclaims_info: kubelet_volume_stats_* is keyed by claim, not by pod, so an unjoined max would report the largest volume anywhere in the namespace and attribute another student's full disk to this session.", + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 205, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(100 * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"} and on (namespace, persistentvolumeclaim) kube_pod_spec_volumes_persistentvolumeclaims_info{namespace=~\"$namespace\", pod=~\"$pod\"})", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Workspace used", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 210, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The dashed limit line is what the container is killed at. The oauth2-proxy sidecar is included here on purpose: it is small, and a proxy eating memory is a real failure mode.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 211, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"$pod\", container!=\"\", container!=\"POD\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", resource=\"memory\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }} limit", + "range": true, + "refId": "B" + } + ], + "title": "Memory per container, against the limit", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "A container pinned at its limit is being throttled; see the panel below for how hard.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 212, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"$pod\", container!=\"\", container!=\"POD\"}[5m])", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", resource=\"cpu\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }} limit", + "range": true, + "refId": "B" + } + ], + "title": "CPU per container, against the limit", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Scheduling periods in which the container wanted CPU and was denied. Non-zero here with normal-looking CPU usage is the signature of a session that feels slow to the student but looks fine to the cluster.", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 213, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\", pod=~\"$pod\", container!=\"\", container!=\"POD\"}[5m])", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU throttled periods", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Traffic to and from the session. A live IDE holds an open WebSocket, so a flatline on both while the pod is Running means the student's connection dropped even though nothing restarted.", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 214, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[5m])", + "format": "time_series", + "instant": false, + "legendFormat": "received", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[5m])", + "format": "time_series", + "instant": false, + "legendFormat": "transmitted", + "range": true, + "refId": "B" + } + ], + "title": "Network", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Non-zero means the container is not running and why: ImagePullBackOff for a tag that was never built, CreateContainerConfigError for a missing secret or ConfigMap, CrashLoopBackOff for a container that keeps exiting.", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 215, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", pod=~\"$pod\"} > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}: {{ "{{" }}reason{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Container waiting reason", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 220, + "panels": [], + "title": "How it ended", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "What killed the container the last time it stopped. OOMKilled means it exceeded its memory limit. Error means the process exited non-zero. Completed with exit 143 or 137 is the garbage collector stopping an idle session, which is routine and not a fault.", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 221, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\", pod=~\"$pod\"} > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}: {{ "{{" }}reason{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Last termination reason", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "137 is SIGKILL, 143 is SIGTERM - both are normal shutdowns. Anything else came from inside the container.", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 222, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_status_last_terminated_exitcode{namespace=~\"$namespace\", pod=~\"$pod\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Last exit code", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "eduide" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"},pod)", + "hide": 0, + "includeAll": false, + "label": "Session pod", + "multi": false, + "name": "pod", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "EduIDE Session Detail", + "uid": "eduide-session-detail", + "version": 1, + "weekStart": "" + } +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-session-startup.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-session-startup.yaml index a33c8ef..96d9943 100644 --- a/charts/eduide-cluster/templates/monitoring/dashboard-session-startup.yaml +++ b/charts/eduide-cluster/templates/monitoring/dashboard-session-startup.yaml @@ -89,7 +89,7 @@ data: { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(application_application_theiacloud_session_startup_seconds_seconds_sum{namespace=\"$namespace\"}[5m]) \n/ \nrate(application_application_theiacloud_session_startup_seconds_seconds_count{namespace=\"$namespace\"}[5m])", + "expr": "rate(application_application_theiacloud_session_startup_seconds_seconds_sum{namespace=~\"$namespace\"}[5m]) \n/ \nrate(application_application_theiacloud_session_startup_seconds_seconds_count{namespace=~\"$namespace\"}[5m])", "instant": false, "interval": "", "legendFormat": "{{ "{{" }}app_definition{{ "}}" }}", @@ -152,7 +152,7 @@ data: { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=\"$namespace\", quantile=\"0.5\"}", + "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.5\"}", "instant": false, "legendFormat": "{{ "{{" }}app_definition{{ "}}" }}", "range": true, @@ -214,7 +214,7 @@ data: { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=\"$namespace\", quantile=\"0.95\"}", + "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.95\"}", "instant": false, "legendFormat": "{{ "{{" }}app_definition{{ "}}" }}", "range": true, @@ -276,7 +276,7 @@ data: { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=\"$namespace\", quantile=\"0.99\"}", + "expr": "application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.99\"}", "instant": false, "legendFormat": "{{ "{{" }}app_definition{{ "}}" }}", "range": true, @@ -293,24 +293,25 @@ data: "templating": { "list": [ { - "current": { "selected": false, "text": "test1", "value": "test1" }, + "allValue": {{ include "eduide.monitoredNamespaceRegex" . | quote }}, + "current": {}, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(application_application_theiacloud_session_startup_seconds_seconds_count,namespace)", + "definition": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "Namespace", "multi": false, "name": "namespace", "options": [], "query": { "qryType": 1, - "query": "label_values(application_application_theiacloud_session_startup_seconds_seconds_count,namespace)", + "query": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, - "sort": 0, + "sort": 1, "type": "query" } ] diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-sessions.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-sessions.yaml new file mode 100644 index 0000000..ce7c80b --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/dashboard-sessions.yaml @@ -0,0 +1,1901 @@ +{{- if .Values.monitoring.enabled }} +{{- /* +Sessions overview. Everything here comes from cAdvisor, kubelet and +kube-state-metrics, plus the one startup timer the REST service exports. +Session pods themselves expose no metrics endpoint - they serve HTML - so +nothing on this dashboard needs a PodMonitor against them. +*/}} +{{- $dashboardNs := .Values.monitoring.dashboardNamespace | toString | trim }} +{{- if not $dashboardNs }} +{{- fail "monitoring.enabled is true but monitoring.dashboardNamespace is empty" }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: eduide-dashboard-sessions + namespace: {{ $dashboardNs }} +data: + eduide-sessions.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Who is using EduIDE right now, how well it is serving them, and what it is costing the cluster.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Right now", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Pods named session-*. One per student actually working. Warm-pool pods are renamed to session-* when a student claims one.", + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 101, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(kube_pod_info{namespace=~\"$namespace\", pod=~\"session-.*\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Sessions in use", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Pods named instance-*: started ahead of demand so a student does not wait for a cold start. If this reaches 0 the next student waits.", + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 102, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(kube_pod_info{namespace=~\"$namespace\", pod=~\"instance-.*\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Warm pool waiting", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Everything a student could be sitting in, claimed or not.", + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 103, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(kube_pod_info{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "IDE pods total", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "How many EduIDE namespaces Prometheus can currently see. A drop here means an environment stopped reporting, not that it has no sessions.", + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 104, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(count by (namespace) (kube_pod_info{namespace=~\"$namespace\"})) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Environments reporting", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 110, + "panels": [], + "title": "Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Claimed sessions over time. This is the closest thing to a count of students actively using EduIDE.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 111, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (namespace) (kube_pod_info{namespace=~\"$namespace\", pod=~\"session-.*\"})", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Sessions in use, by environment", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Unclaimed instance-* pods. Flat lines at 0 mean every start is a cold start; see the startup latency panel below for what that costs.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 112, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (namespace) (kube_pod_info{namespace=~\"$namespace\", pod=~\"instance-.*\"})", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Warm pool available, by environment", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Which language images are actually being used. The container is named after the app definition, so this is the app breakdown without needing a metric from the operator.", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 113, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"})", + "format": "table", + "instant": true, + "legendFormat": "{{ "{{" }}container{{ "}}" }}", + "range": false, + "refId": "A" + } + ], + "title": "IDE pods by app image", + "type": "barchart", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "lineWidth": 1 + }, + "mappings": [], + "noValue": "No pods", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "barWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "auto", + "xTickLabelRotation": 0 + }, + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Derived from the REST service's startup timer, so it counts sessions the service actually served - not pods the operator created.", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 114, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespace) (rate(application_application_theiacloud_session_startup_seconds_seconds_count{namespace=~\"$namespace\"}[15m])) * 3600", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Session starts per hour", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "How long a student waits between asking for a session and getting one. These are decaying quantiles: with no traffic they hold their last value rather than dropping to zero, so read them alongside the starts per hour panel.", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 115, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (namespace) (application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.5\"})", + "format": "time_series", + "instant": false, + "legendFormat": "p50 {{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (namespace) (application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.95\"})", + "format": "time_series", + "instant": false, + "legendFormat": "p95 {{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (namespace) (application_application_theiacloud_session_startup_seconds_seconds{namespace=~\"$namespace\", quantile=\"0.99\"})", + "format": "time_series", + "instant": false, + "legendFormat": "p99 {{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "C" + } + ], + "title": "Session startup latency", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 120, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sum across all IDE containers, excluding the oauth2-proxy sidecar. If used tracks close to limit the sessions are CPU starved and builds feel slow; if it sits far below, the limit is generous and more sessions fit per node.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 121, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"}[5m]))", + "format": "time_series", + "instant": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", resource=\"cpu\", container!~\"oauth.*\"})", + "format": "time_series", + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Session CPU: used against limit", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The gap between these two lines is what stands between a student and an OOM kill. A session killed for memory loses unsaved editor state; the workspace volume survives.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 122, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"})", + "format": "time_series", + "instant": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", resource=\"memory\", container!~\"oauth.*\"})", + "format": "time_series", + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Session memory: used against limit", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Share of scheduling periods in which a session container was held back at its CPU limit. Sustained throttling is what students report as \"the IDE is laggy\" while every pod looks healthy.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 123, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespace) (rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"}[5m])) / clamp_min(sum by (namespace) (rate(container_cpu_cfs_periods_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", container!=\"POD\"}[5m])), 1)", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU throttling", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sessions are scheduled like any other pod. A node holding far more than its share is usually a sign the others are cordoned or full.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 124, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (node) (kube_pod_info{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"})", + "format": "table", + "instant": true, + "legendFormat": "{{ "{{" }}node{{ "}}" }}", + "range": false, + "refId": "A" + } + ], + "title": "IDE pods per node", + "type": "barchart", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "lineWidth": 1 + }, + "mappings": [], + "noValue": "No pods", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "barWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "auto", + "xTickLabelRotation": 0 + }, + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 130, + "panels": [], + "title": "Workspace storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "PersistentVolumeClaims in Bound phase. Persistent workspaces only; ephemeral installations have none and this reads 0 correctly.", + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 41 + }, + "id": 131, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolumeclaim_status_phase{namespace=~\"$namespace\", phase=\"Bound\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Workspace volumes bound", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "A pending claim means the session cannot start. Usually the storage class or a quota, not the cluster being full.", + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 41 + }, + "id": 132, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolumeclaim_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Volumes pending", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total capacity claimed by workspaces across the selected environments.", + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 41 + }, + "id": 133, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace=~\"$namespace\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Storage requested", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The single closest workspace to running out of space. A full workspace fails writes inside the IDE with errors that do not name the disk.", + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 41 + }, + "id": 134, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(100 * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}) or vector(0)", + "format": "time_series", + "instant": false, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Fullest workspace", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per claim, as a percentage of capacity. Watch for a claim that climbs and never comes down - that is a build cache nobody cleans up.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 135, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}/{{ "{{" }}persistentvolumeclaim{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Workspace volume usage", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Inodes run out long before bytes do when a session runs npm install or resolves a large Maven tree. The IDE reports \"no space left on device\" on a volume that looks half empty.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 136, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * kubelet_volume_stats_inodes_used{namespace=~\"$namespace\"} / kubelet_volume_stats_inodes{namespace=~\"$namespace\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{ "{{" }}namespace{{ "}}" }}/{{ "{{" }}persistentvolumeclaim{{ "}}" }}", + "range": true, + "refId": "A" + } + ], + "title": "Workspace inode usage", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 140, + "panels": [], + "title": "Longest running", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sessions ranked by age. The garbage collector stops idle sessions, so a session far older than the rest is either genuinely in use or stuck. Click a row to open it in the session detail dashboard.", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 141, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "topk(25, time() - kube_pod_created{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\"})", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Oldest sessions", + "type": "table", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Age" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 14400 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Open in session detail", + "url": "/d/eduide-session-detail/eduide-session-detail?var-namespace=${__data.fields.Environment}&var-pod=${__data.fields.Pod}" + } + ] + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Age" + } + ] + }, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "container": true, + "endpoint": true, + "instance": true, + "job": true, + "service": true, + "uid": true + }, + "renameByName": { + "namespace": "Environment", + "pod": "Pod", + "Value": "Age" + } + } + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "eduide" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "hide": 0, + "includeAll": true, + "label": "Environment", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "allValue": "{{ include "eduide.monitoredNamespaceRegex" . }}" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "EduIDE Sessions", + "uid": "eduide-sessions", + "version": 1, + "weekStart": "" + } +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/dashboard-theiacloud.yaml b/charts/eduide-cluster/templates/monitoring/dashboard-theiacloud.yaml index 7cf6e72..a0d3203 100644 --- a/charts/eduide-cluster/templates/monitoring/dashboard-theiacloud.yaml +++ b/charts/eduide-cluster/templates/monitoring/dashboard-theiacloud.yaml @@ -147,7 +147,7 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "count(count by (pod) (container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!=\"POD\"}))", + "expr": "count(count by (pod) (container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!=\"POD\"}))", "format": "time_series", "instant": false, "interval": "", @@ -246,7 +246,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\"}[1m])) by (pod)", + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=~\"$namespace\"}[1m])) by (pod)", "instant": false, "legendFormat": "__auto", "range": true, @@ -258,7 +258,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\"}[1m]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",container!=\"\", job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=~\"$namespace\"}[1m]))", "hide": false, "instant": false, "legendFormat": "TOTAL", @@ -357,7 +357,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", service=~\"theia-.*\"}", + "expr": "container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\"}", "instant": false, "legendFormat": "{{ "{{" }}pod{{ "}}" }}", "range": true, @@ -455,7 +455,7 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", service=~\"theia-.*\"}[1m])", + "expr": "rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\"}[1m])", "instant": false, "interval": "", "legendFormat": "{{ "{{" }}pod{{ "}}" }}", @@ -554,7 +554,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "avg by (container) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", service=~\"theia-.*\"})", + "expr": "avg by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\"})", "instant": false, "legendFormat": "{{ "{{" }}container{{ "}}" }}", "range": true, @@ -652,7 +652,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "avg by (container) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\", service=~\"theia-.*\"}[1m]))", + "expr": "avg by (container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", pod=~\"(session-|instance-).*\", container!=\"\", container!~\"oauth.*\"}[1m]))", "instant": false, "legendFormat": "{{ "{{" }}container{{ "}}" }}", "range": true, @@ -1087,42 +1087,26 @@ data: "templating": { "list": [ { - "current": { "selected": false, "text": "theia", "value": "theia" }, + "allValue": {{ include "eduide.monitoredNamespaceRegex" . | quote }}, + "current": {}, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "Namespace", "multi": false, "name": "namespace", - "options": [ - { - "selected": false, - "text": "theia", - "value": "theia" - }, - { - "selected": false, - "text": "theia-staging", - "value": "theia-staging" - }, - { - "selected": true, - "text": "test1", - "value": "test1" - }, - { - "selected": false, - "text": "test2", - "value": "test2" - }, - { - "selected": false, - "text": "test3", - "value": "test3" - } - ], - "query": "theia,theia-staging,test1,test2,test3", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info{namespace=~\"{{ include "eduide.monitoredNamespaceRegex" . }}\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", "skipUrlSync": false, - "type": "custom" + "sort": 1, + "type": "query" } ] }, diff --git a/charts/eduide-cluster/templates/monitoring/podmonitor-sessions.yaml b/charts/eduide-cluster/templates/monitoring/podmonitor-sessions.yaml deleted file mode 100644 index d35e743..0000000 --- a/charts/eduide-cluster/templates/monitoring/podmonitor-sessions.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.monitoring.enabled }} -{{- /* -A PodMonitor naming no namespaces watches nothing, so it is not rendered at all -rather than created as a no-op. The list is derived by bootstrap-cluster.yml -from the environments on the cluster; an empty one means every environment -opted out, which is a supported state. A bare `helm install eduide-cluster` -lands here too, and must not fail. -*/}} -{{- if .Values.monitoring.sessionNamespaces }} -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: theia-cloud-sessions - namespace: {{ .Values.monitoring.namespace }} -spec: - selector: - matchExpressions: - - key: app - operator: NotIn - values: - - conversion-webhook - - landing-page - - operator - - service - namespaceSelector: - matchNames: - {{- range .Values.monitoring.sessionNamespaces }} - - {{ . }} - {{- end }} - podMetricsEndpoints: - - port: application - interval: 15s -{{- end }} -{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml b/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml new file mode 100644 index 0000000..0e2ed11 --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/prometheusrule.yaml @@ -0,0 +1,425 @@ +{{- /* +Called before the guard below, deliberately: inside it, a misconfiguration that +renders nothing could not report itself. +*/}} +{{- include "eduide.checkAlertingPrerequisites" . }} +{{- if and .Values.monitoring.enabled .Values.monitoring.alerting.enabled }} +{{- /* +EduIDE's alerts. + +Two rules govern everything in here. + +**Every expression is checked against a live Prometheus before it ships.** The +dashboards in this directory spent months showing nothing because they selected +on `service=~"theia-.*"`, a label that stopped existing in 2.0.0 and that no +render test could catch. A selector that matches nothing produces an alert that +never fires, which looks exactly like a healthy platform. + +**Session-level faults alert on rates, never per pod.** This platform runs +student code. Sessions get OOM-killed and crash during normal exercise work, and +one notification per event turns the channel into something nobody reads - at +which point the platform is unmonitored no matter how many rules exist. + +Deployment names below are the real ones, verified against kube-state-metrics: +operator-deployment, service-deployment, landing-page-deployment, +garbage-collector, conversion-webhook. +*/}} +{{- $a := .Values.monitoring.alerting }} +{{- $t := $a.thresholds }} +{{- $ns := include "eduide.monitoredNamespaceRegex" . }} +{{- $sessions := `pod=~"(session-|instance-).*"` }} +{{- $ide := `container!="", container!~"oauth.*", container!="POD"` }} +{{- $runbook := printf "%s/incident-response" (trimSuffix "/" $a.runbookUrl) }} +{{- /* +A deep link into the session dashboards, when a Grafana URL is configured. The +notification is read by someone who is not at their desk, so a link that lands +on the right pod is worth more than the metric value. +*/}} +{{- $g := trimSuffix "/" $a.grafanaUrl }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: eduide-alerts + namespace: {{ $a.namespace }} + labels: + app.kubernetes.io/part-of: eduide +spec: + groups: + - name: eduide.platform + rules: + - alert: EduIDEComponentDown + expr: |- + kube_deployment_status_replicas_available{namespace=~"{{ $ns }}", deployment=~"(operator|service|landing-page)-deployment|garbage-collector"} == 0 + for: 5m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'EduIDE {{ "{{" }} $labels.deployment {{ "}}" }} is down in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + The {{ "{{" }} $labels.deployment {{ "}}" }} deployment in {{ "{{" }} $labels.namespace {{ "}}" }} has had no + available replicas for 5 minutes. + {{ "{{" }} if eq $labels.deployment "service-deployment" {{ "}}" }}Students cannot start new + sessions: the landing page calls this service to create one. Sessions already + running are unaffected.{{ "{{" }} end {{ "}}" }} + {{ "{{" }} if eq $labels.deployment "operator-deployment" {{ "}}" }}Nothing reconciles Session + or Workspace resources: new sessions will be accepted and never start, and + stopped ones will not be cleaned up. Running sessions keep working.{{ "{{" }} end {{ "}}" }} + {{ "{{" }} if eq $labels.deployment "landing-page-deployment" {{ "}}" }}Students get an error + page instead of the environment picker. Running sessions keep working and + direct session URLs still resolve.{{ "{{" }} end {{ "}}" }} + {{ "{{" }} if eq $labels.deployment "garbage-collector" {{ "}}" }}Idle sessions are no longer + being stopped, so the cluster will fill up with sessions nobody is using. + Nothing breaks immediately; capacity degrades over hours.{{ "{{" }} end {{ "}}" }} + Start with `kubectl -n {{ "{{" }} $labels.namespace {{ "}}" }} describe deploy {{ "{{" }} $labels.deployment {{ "}}" }}` + and the pod logs. + To silence this during planned maintenance, match on + eduide_namespace="{{ "{{" }} $labels.namespace {{ "}}" }}" - not on namespace. + runbook_url: {{ $runbook }} + + - alert: EduIDEConversionWebhookDown + expr: |- + kube_deployment_status_replicas_available{namespace="{{ .Release.Namespace }}", deployment="conversion-webhook"} == 0 + for: 5m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'The EduIDE conversion webhook is down, cluster-wide' + description: >- + The conversion webhook has had no available replicas for 5 minutes. + This is cluster-scoped, so it affects every environment at once, not one. + Every read and write of a Session, Workspace or AppDefinition goes through + it, so while it is down the operator cannot reconcile anything and + `kubectl get session` fails as well. Sessions already running keep serving + traffic; nothing new can start anywhere. + Check `kubectl -n {{ .Release.Namespace }} get pods -l app=conversion-webhook` + and whether its serving certificate is still valid. + runbook_url: {{ $runbook }} + + - alert: EduIDEComponentCrashLooping + expr: |- + increase(kube_pod_container_status_restarts_total{namespace=~"{{ $ns }}", pod!~"(session-|instance-).*"}[15m]) > {{ $t.componentRestarts }} + for: 5m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'EduIDE {{ "{{" }} $labels.container {{ "}}" }} is crash looping in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Container {{ "{{" }} $labels.container {{ "}}" }} in pod {{ "{{" }} $labels.pod {{ "}}" }} restarted + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} times in 15 minutes. This is a platform component, + not a student session, so it is a fault rather than something a student did. + A component that restarts this often is usually failing on startup: a + misconfigured secret, an unreachable Keycloak, or an image that does not + match the chart. `kubectl -n {{ "{{" }} $labels.namespace {{ "}}" }} logs {{ "{{" }} $labels.pod {{ "}}" }} + --previous` shows why the last attempt died. + runbook_url: {{ $runbook }} + + - alert: EduIDEServiceScrapeDown + expr: |- + up{job=~".*theia-cloud-service", namespace=~"{{ $ns }}"} == 0 + for: 10m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Prometheus cannot scrape the EduIDE service in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + The REST service in {{ "{{" }} $labels.namespace {{ "}}" }} has not been scrapeable for 10 + minutes. This does not mean students are affected - the service may be + serving fine - but it does mean we are blind: session start counts and + startup latency stop being recorded, and EduIDESessionStartupSlow cannot + fire while this is true. Treat it as a monitoring outage for that + environment. Check the pod is running and that /q/metrics still answers. + runbook_url: {{ $runbook }} + + - alert: EduIDEWarmPoolEmpty + expr: |- + (sum by (namespace) (kube_deployment_status_replicas{namespace=~"{{ $ns }}", deployment=~"instance-.*"}) > 0) + and + (sum by (namespace) (kube_deployment_status_replicas_available{namespace=~"{{ $ns }}", deployment=~"instance-.*"}) == 0) + for: 10m + labels: + severity: critical + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'The EduIDE warm pool is empty in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Every pre-warmed instance in {{ "{{" }} $labels.namespace {{ "}}" }} is unavailable, while the + environment is configured to keep some. Sessions still start, but each + student now waits for a cold start - pulling the image and booting Theia - + instead of being handed a running IDE. + This is expressed against desired replicas, so an environment configured + with no warm pool cannot trigger it. Common causes are a node that cannot + pull the image, insufficient cluster capacity, or every instance having + just been claimed at once. + `kubectl -n {{ "{{" }} $labels.namespace {{ "}}" }} get pods | grep instance-` is the first look. + runbook_url: {{ $runbook }} + + - name: eduide.sessions + rules: + {{- /* + Counted off the restart counter, not off the termination reason. + `kube_pod_container_status_last_terminated_reason` is a gauge holding + only the most recent reason, so increase() over it counts nothing + reliable: a container that OOMs ten times reports the same 1 throughout. + `kube_pod_container_status_restarts_total` is a real counter, and + `and on(...)` narrows it to containers whose last exit was an OOM kill. + Same shape as EduIDESessionCrashSpike below, deliberately. + */}} + - alert: EduIDESessionOOMKillSpike + expr: |- + sum by (namespace) ( + increase(kube_pod_container_status_restarts_total{namespace=~"{{ $ns }}", {{ $sessions }}}[1h]) + and on (namespace, pod, container) + (kube_pod_container_status_last_terminated_reason{namespace=~"{{ $ns }}", {{ $sessions }}, reason="OOMKilled"} == 1) + ) > {{ $t.sessionOOMPerHour }} + for: 10m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Sessions are being OOM-killed in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} session containers in {{ "{{" }} $labels.namespace {{ "}}" }} were killed + for exceeding their memory limit in the last hour. Each one ended a + student's session abruptly: unsaved editor buffers and terminal state are + gone, though the workspace volume survives. + A single OOM kill is normal - students write code that allocates. This + alerts on the rate, so it means either the memory limit is too low for the + work being set, or one exercise is pushing everyone over it at once. + The session detail dashboard shows memory against the limit per container. + Raise `appDefinitions.defaults.limitsMemory` for the environment if this + is the workload rather than a bug. + runbook_url: {{ $runbook }} + {{- if $g }} + dashboard_url: '{{ $g }}/d/eduide-sessions/eduide-sessions?var-namespace={{ "{{" }} $labels.namespace {{ "}}" }}' + {{- end }} + + {{- /* + Restarts, narrowed to containers whose last termination was an Error. + Counting exit codes directly does not work: the exit code is a gauge, so + it cannot be fed to increase(), and a gauge cannot distinguish one crash + from the same crash still being reported an hour later. + `and on(...)` filters the restart counter by the existence of an Error + termination for the same container, which is both valid PromQL and the + thing we actually mean. + */}} + - alert: EduIDESessionCrashSpike + expr: |- + sum by (namespace) ( + increase(kube_pod_container_status_restarts_total{namespace=~"{{ $ns }}", {{ $sessions }}}[1h]) + and on (namespace, pod, container) + (kube_pod_container_status_last_terminated_reason{namespace=~"{{ $ns }}", {{ $sessions }}, reason="Error"} == 1) + ) > {{ $t.sessionCrashPerHour }} + for: 10m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Sessions are crashing in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} session containers in {{ "{{" }} $labels.namespace {{ "}}" }} exited + with an unexpected status in the last hour. Exit codes 137 and 143 are + excluded: those are the garbage collector stopping an idle session, which + is routine and not a fault. + A cluster of real crashes usually means a broken image was just deployed, + so check what changed before looking at individual sessions. If a specific + language image is involved, the IDE container name in the session detail + dashboard names it. + runbook_url: {{ $runbook }} + {{- if $g }} + dashboard_url: '{{ $g }}/d/eduide-sessions/eduide-sessions?var-namespace={{ "{{" }} $labels.namespace {{ "}}" }}' + {{- end }} + + - alert: EduIDESessionStartupSlow + expr: |- + ( + max by (namespace) (application_application_theiacloud_session_startup_seconds_seconds{namespace=~"{{ $ns }}", quantile="0.95"}) > {{ $t.startupSeconds }} + ) + and + ( + sum by (namespace) (rate(application_application_theiacloud_session_startup_seconds_seconds_count{namespace=~"{{ $ns }}"}[15m])) > 0 + ) + for: 15m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Sessions are slow to start in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + 95th percentile session startup in {{ "{{" }} $labels.namespace {{ "}}" }} is + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} seconds. Students are waiting that long between + clicking through and getting an IDE, which is where they give up and mail + somebody. + The second half of this expression requires sessions to actually be + starting. Without it the alert would latch: these are decaying quantiles + and they hold their last value when idle, so one slow start on a Friday + evening would alert all weekend. + An empty warm pool is the usual cause - check EduIDEWarmPoolEmpty and the + warm pool panel first. Image pulls on a cold node are the other. + runbook_url: {{ $runbook }} + {{- if $g }} + dashboard_url: '{{ $g }}/d/eduide-sessions/eduide-sessions?var-namespace={{ "{{" }} $labels.namespace {{ "}}" }}' + {{- end }} + + - alert: EduIDESessionPodPending + expr: |- + kube_pod_status_phase{namespace=~"{{ $ns }}", {{ $sessions }}, phase="Pending"} == 1 + for: 5m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'A session pod is stuck Pending in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Pod {{ "{{" }} $labels.pod {{ "}}" }} in {{ "{{" }} $labels.namespace {{ "}}" }} has been Pending for + 5 minutes, which means it has not been scheduled onto a node at all. If a + student is waiting on this one, they are looking at a spinner. + Almost always cluster capacity, an unschedulable node, or a workspace + volume that cannot be provisioned. + `kubectl -n {{ "{{" }} $labels.namespace {{ "}}" }} describe pod {{ "{{" }} $labels.pod {{ "}}" }}` + names the reason in its events. + runbook_url: {{ $runbook }} + + - alert: EduIDESessionImagePullFailing + expr: |- + kube_pod_container_status_waiting_reason{namespace=~"{{ $ns }}", reason=~"ImagePullBackOff|ErrImagePull"} == 1 + for: 10m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'An EduIDE image cannot be pulled in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Container {{ "{{" }} $labels.container {{ "}}" }} in {{ "{{" }} $labels.pod {{ "}}" }} + ({{ "{{" }} $labels.namespace {{ "}}" }}) has been in {{ "{{" }} $labels.reason {{ "}}" }} for 10 minutes. + The usual cause is a tag that was never built. A pull request only builds + the images of the repository it came from, so a blanket image tag puts every + other image into this state; the chart has one version knob per source + repository for exactly this reason. Check that the tag the environment asks + for exists in the registry before looking at the cluster. + runbook_url: {{ $runbook }} + + - alert: EduIDESessionEvicted + expr: |- + sum by (namespace) (increase(kube_pod_status_reason{namespace=~"{{ $ns }}", reason="Evicted"}[1h])) > 0 + for: 5m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Pods are being evicted in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} pods were evicted from their nodes in + {{ "{{" }} $labels.namespace {{ "}}" }} in the last hour. Eviction means the node ran out + of a resource the pod was not limited on - almost always ephemeral disk + from build caches and dependency downloads inside sessions. + The students affected lost their session with no warning. Check node disk + pressure, and whether one image's build behaviour changed. + runbook_url: {{ $runbook }} + + - name: eduide.storage + rules: + - alert: EduIDEWorkspaceVolumeFilling + expr: |- + 100 * kubelet_volume_stats_used_bytes{namespace=~"{{ $ns }}"} + / kubelet_volume_stats_capacity_bytes{namespace=~"{{ $ns }}"} > {{ $t.volumePercent }} + for: 15m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'A workspace volume is nearly full in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Workspace volume {{ "{{" }} $labels.persistentvolumeclaim {{ "}}" }} in + {{ "{{" }} $labels.namespace {{ "}}" }} is {{ "{{" }} $value | printf "%.0f" {{ "}}" }}% full. + When it fills completely the IDE starts failing writes with errors that do + not mention the disk - saves fail, git operations fail, and the student + reports it as the editor being broken. + Usually a build cache or a node_modules tree that nobody clears. + runbook_url: {{ $runbook }} + + - alert: EduIDEWorkspaceInodesFilling + expr: |- + 100 * kubelet_volume_stats_inodes_used{namespace=~"{{ $ns }}"} + / kubelet_volume_stats_inodes{namespace=~"{{ $ns }}"} > {{ $t.volumePercent }} + for: 15m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'A workspace volume is running out of inodes in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + Workspace volume {{ "{{" }} $labels.persistentvolumeclaim {{ "}}" }} in + {{ "{{" }} $labels.namespace {{ "}}" }} has used {{ "{{" }} $value | printf "%.0f" {{ "}}" }}% of its inodes. + This is worth alerting on separately from bytes because it is the one that + surprises people: a single npm install creates tens of thousands of small + files, so a volume that looks half empty starts returning "no space left on + device". Checking free space will not show the problem. + runbook_url: {{ $runbook }} + + - alert: EduIDEPVCPending + expr: |- + kube_persistentvolumeclaim_status_phase{namespace=~"{{ $ns }}", phase="Pending"} == 1 + for: 10m + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'A workspace volume will not provision in {{ "{{" }} $labels.namespace {{ "}}" }}' + description: >- + PersistentVolumeClaim {{ "{{" }} $labels.persistentvolumeclaim {{ "}}" }} in + {{ "{{" }} $labels.namespace {{ "}}" }} has been Pending for 10 minutes, so the session + that needs it cannot start. + The storage class is the first thing to check - it is a cluster property + set once in clusters/.yaml, and a claim naming a class the cluster + does not offer stays Pending forever without any other symptom. + runbook_url: {{ $runbook }} + +{{- if .Values.monitoring.certManager.enabled }} + - name: eduide.certificates + rules: + - alert: EduIDECertificateExpiringSoon + expr: |- + (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < {{ $t.certExpiryDays }} + for: 1h + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Certificate {{ "{{" }} $labels.name {{ "}}" }} expires in {{ "{{" }} $value | printf "%.0f" {{ "}}" }} days' + description: >- + Certificate {{ "{{" }} $labels.name {{ "}}" }} in {{ "{{" }} $labels.namespace {{ "}}" }} expires in + {{ "{{" }} $value | printf "%.0f" {{ "}}" }} days and has not renewed. + cert-manager renews the ACME certificates on its own, so this firing for one + of those means renewal is failing and needs looking at now rather than on + the day. The webview wildcard is different: it is bought from Harica through + RBG and installed by hand, nothing renews it, and when it lapses every + preview in every session on the cluster breaks at once - reported as "the + IDE is broken", not as a certificate problem. + runbook_url: {{ $runbook }} + + - alert: EduIDECertificateNotReady + expr: |- + certmanager_certificate_ready_status{condition="True"} == 0 + for: 1h + labels: + severity: warning + {{- include "eduide.alertRoutingLabels" . | nindent 12 }} + annotations: + summary: 'Certificate {{ "{{" }} $labels.name {{ "}}" }} has not been ready for an hour' + description: >- + Certificate {{ "{{" }} $labels.name {{ "}}" }} in {{ "{{" }} $labels.namespace {{ "}}" }} has been + Ready=False for an hour. Until it issues, the Gateway listener for that + hostname serves whatever the Secret already held - which may be an expired + or entirely unrelated certificate, since Gateway API never checks that a + certificate matches the hostname it is serving. The listener will keep + reporting Programmed=True throughout. + Read down the chain before changing anything: CertificateRequest, then + Order, then Challenge. A challenge stuck with a 404 means the name has no + listener; a DNS or connection error means it does not resolve here at all. + runbook_url: {{ $runbook }} +{{- end }} +{{- end }} diff --git a/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml b/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml new file mode 100644 index 0000000..c3a5e3e --- /dev/null +++ b/charts/eduide-cluster/templates/monitoring/servicemonitor-certmanager.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.monitoring.enabled .Values.monitoring.certManager.enabled }} +{{- /* +Scrape cert-manager, so certificates can be alerted on before they lapse. + +cert-manager exports the expiry timestamp and readiness of every Certificate it +manages, but ships no ServiceMonitor, so on a stock install nothing collects +them. The result is a cluster where a certificate can be failing to renew for +months with nothing to notice - which is how test3 served a certificate for the +wrong hostnames for 184 days. + +cert-manager is platform-owned, so this only creates the ServiceMonitor. It does +not install, configure or upgrade cert-manager itself. +*/}} +{{- $cm := .Values.monitoring.certManager }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: eduide-cert-manager + namespace: {{ .Values.monitoring.namespace }} + labels: + app.kubernetes.io/part-of: eduide +spec: + {{- /* + Selected by label, not by name. A ServiceMonitor's selector matches Service + *labels*; there is no name matcher. The values are configurable because a + cert-manager installed under a different release name keeps these labels but + changes their values, and a selector that matches nothing produces a + ServiceMonitor with no targets and no error. + */}} + selector: + matchLabels: + {{- toYaml $cm.selectorLabels | nindent 6 }} + namespaceSelector: + matchNames: + - {{ $cm.namespace }} + endpoints: + - port: {{ $cm.portName }} + interval: 60s + honorLabels: true +{{- end }} diff --git a/charts/eduide-cluster/values-example.yaml b/charts/eduide-cluster/values-example.yaml index 10d7e45..d8988e3 100644 --- a/charts/eduide-cluster/values-example.yaml +++ b/charts/eduide-cluster/values-example.yaml @@ -121,9 +121,35 @@ wildcardTLSSecret: # --------------------------------------------------------------------------- # Needs the Prometheus Operator CRDs, and both namespaces must already exist and # be watched by your Prometheus and Grafana. The defaults are Rancher's names. +# Monitoring and alerting are switched on here, unlike most of this file's +# defaults, because CI renders the chart from this example and validates the +# result against the CRD schemas. Left off, the PrometheusRule, the +# AlertmanagerConfig and every dashboard would render to nothing and would never +# be schema-checked - which is exactly how the dashboards came to select on a +# label that had not existed since 2.0.0. monitoring: - enabled: false - # namespace: monitoring - # dashboardNamespace: monitoring - # targetNamespaces: [eduide-prod] - # sessionNamespaces: [eduide-prod] + enabled: true + namespace: monitoring + dashboardNamespace: monitoring + targetNamespaces: [eduide-prod] + certManager: + enabled: true + alerting: + enabled: true + namespace: eduide-system + grafanaUrl: https://grafana.example.org + webhookSecret: + create: true + # Base64 of a placeholder URL. A real deployment gets these from a GitHub + # Environment secret; a webhook URL is a credential and never lives here. + data: + slack-platform: aHR0cHM6Ly9leGFtcGxlLm9yZy9zbGFjay13ZWJob29r + discord-platform: aHR0cHM6Ly9leGFtcGxlLm9yZy9kaXNjb3JkLXdlYmhvb2s= + channels: + - name: platform-slack + type: slack + secretKey: slack-platform + channel: "#eduide-alerts" + - name: platform-discord + type: discord + secretKey: discord-platform diff --git a/charts/eduide-cluster/values.yaml b/charts/eduide-cluster/values.yaml index 30d5289..5676420 100644 --- a/charts/eduide-cluster/values.yaml +++ b/charts/eduide-cluster/values.yaml @@ -205,5 +205,122 @@ monitoring: # -- Namespace the Grafana dashboard ConfigMaps go in. Must already exist and # be watched by Grafana's sidecar. `cattle-dashboards` is Rancher's. dashboardNamespace: cattle-dashboards + # -- Namespaces to watch. Bootstrap derives this from the environments on the + # cluster; it is also the namespace list every dashboard's picker offers. targetNamespaces: [] - sessionNamespaces: [] + + # Scrape cert-manager, so certificate expiry can be alerted on. + # + # cert-manager exports `certmanager_certificate_expiration_timestamp_seconds` + # but ships no ServiceMonitor, so by default nothing watches it. The webview + # wildcard is renewed by hand once a year and nothing warns anyone; when it + # lapses, every preview in every session on the cluster breaks at once and it + # gets reported as "the IDE is broken". + certManager: + # -- Create a ServiceMonitor for cert-manager's controller metrics. + enabled: false + # -- Namespace cert-manager runs in. + namespace: cert-manager + # -- Labels identifying cert-manager's controller Service. + # + # A ServiceMonitor selects on Service **labels**; there is no matcher for + # `metadata.name`. These are the labels cert-manager's own chart sets, and + # they are values because a release installed under a different name keeps + # the labels and changes their values. Get them wrong and the ServiceMonitor + # is created with no targets and reports no error. + # + # Check with: + # kubectl -n cert-manager get svc cert-manager --show-labels + selectorLabels: + app.kubernetes.io/name: cert-manager + app.kubernetes.io/component: controller + # -- Name of the metrics port on that Service. + portName: http-metrics + + # Alerting: a PrometheusRule plus an AlertmanagerConfig with the channels to + # notify. Independent of `monitoring.enabled` only in the sense that it needs + # it - alerting off with monitoring on is fine, the reverse is not. + alerting: + # -- Create the PrometheusRule and the AlertmanagerConfig. + enabled: false + # -- Namespace both resources go in. + # + # This is also the `namespace` label every EduIDE alert carries, and it is + # a routing artifact rather than the environment the alert is about. The + # Prometheus Operator defaults `alertmanagerConfigMatcherStrategy` to + # `OnNamespace`, which prepends `namespace = ` to every route generated from it, so an alert must claim this + # namespace to reach a receiver at all. + # + # The environment an alert is about is in `eduide_namespace`. Silence on + # that label, never on `namespace` - `namespace` is identical across every + # EduIDE alert and silencing it silences all of them. + namespace: eduide-system + # -- Lowest severity that reaches the notification channels: `warning` or + # `critical`. Anything below stays in Alertmanager and on the dashboards. + # This is what keeps the channels worth reading. + minSeverity: warning + # -- How long to wait for more alerts in a group before the first notification. + groupWait: 30s + # -- How long to wait before notifying about new alerts added to a group. + groupInterval: 5m + # -- How long before an unresolved alert is sent again. + repeatInterval: 4h + # -- Base URL of the Grafana that serves the EduIDE dashboards, with no + # trailing slash. Used to deep-link a notification straight to the affected + # session. Left empty, notifications carry no dashboard link. + grafanaUrl: "" + # -- Base URL of the operations runbooks, with no trailing slash. + runbookUrl: https://eduide.github.io/Docs/admins/operations + # The Secret holding the webhook URLs. Never put a webhook URL in a values + # file or on a `--set` - it goes in the process list and in Actions debug + # logs. Bootstrap writes this Secret from GitHub Environment secrets. + webhookSecret: + # -- Create the Secret from `data` below. Off means the Secret already + # exists and is referenced by name only. + create: false + # -- Name of the Secret channels read their URL from. + name: eduide-alert-webhooks + # -- Base64-encoded webhook URLs, keyed by the `secretKey` a channel + # names. Supplied by the workflow, not committed. + data: {} + # -- Where to send alerts. Each entry needs `name`, `type` (`slack` or + # `discord`) and `secretKey`; Slack entries may also set `channel`. An + # empty list with alerting enabled means alerts fire but notify nobody, so + # the chart fails the render instead. + # + # A channel may set `environments: [ns, ...]` to receive only that + # installation's alerts. One cluster can host installations belonging to + # different people - Bonn and Mannheim share a cluster and each has its own + # Discord - and without scoping, both would see the other's incidents. + # + # Anything no scoped channel claims goes to **every** channel. That is how + # cluster-scoped alerts still get out: a certificate expiring or the + # conversion webhook failing is not about any one tenant's namespace, and + # dropping it for failing to match a tenant route would lose exactly the + # alerts that matter most. + # + # channels: + # - name: mannheim + # type: discord + # secretKey: discord-mannheim + # environments: [eduide-mannheim] + # - name: platform + # type: slack + # secretKey: slack-platform + # channel: "#eduide-alerts" + channels: [] + # Thresholds, so tuning noise down does not need a chart release. + thresholds: + # -- Restarts of a platform container within 15m before it counts as crash looping. + componentRestarts: 3 + # -- Session OOM kills per hour in one environment before alerting. + sessionOOMPerHour: 3 + # -- Session crashes per hour in one environment before alerting. + sessionCrashPerHour: 5 + # -- p95 session startup, in seconds, that counts as too slow. + startupSeconds: 90 + # -- Workspace volume fill percentage that counts as nearly full. + volumePercent: 85 + # -- Days before certificate expiry to start warning. + certExpiryDays: 21