From 73b2df6f55981dc7734d12a62064b2edf6a73216 Mon Sep 17 00:00:00 2001 From: Vladislav Gusev Date: Mon, 7 Sep 2026 16:09:22 +0300 Subject: [PATCH] [mariadb] slow-log housekeeping, dedicated slow-log volume, PVC-free alert - prune slow-query logs older than slow_query_log.cleanup.max_age_days (default 14) on pod init - optionally write the slow query log to a dedicated PVC (slow_query_log.persistence) - opt-in data-PVC free-space alert at alerts.pvc_free_enabled (threshold alerts.pvc_free_threshold, default 0.20; severity alerts.pvc_free_severity, default warning) - extract dataDir/dataPvcName/slowLogDir/slowLogPvcName path helpers so PVC names and the volume alert stay in sync - fix context label typo (datbase) on the too-many-connections alert - chart version bumped to 0.43.0 --- common/mariadb/CHANGELOG.md | 7 +++ common/mariadb/Chart.yaml | 2 +- common/mariadb/templates/_helpers.tpl | 29 ++++++++++++ common/mariadb/templates/alerts.yaml | 3 ++ .../templates/alerts/_mysql.alerts.tpl | 2 +- .../templates/alerts/_volume.alerts.tpl | 15 ++++++ common/mariadb/templates/deployment.yaml | 47 ++++++++++++++++++- common/mariadb/templates/pvc.yaml | 22 +++++++++ common/mariadb/values.yaml | 22 ++++++++- 9 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 common/mariadb/templates/alerts/_volume.alerts.tpl diff --git a/common/mariadb/CHANGELOG.md b/common/mariadb/CHANGELOG.md index 5327e005346..817feab41aa 100644 --- a/common/mariadb/CHANGELOG.md +++ b/common/mariadb/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## v0.43.0 - 2026/09/11 +* prune slow-query logs older than `slow_query_log.cleanup.max_age_days` (default 14) on pod init +* optionally write the slow query log to a dedicated PVC (`slow_query_log.persistence`); it is written per-pod as `/-slow.log` so `cleanup` prunes it across restarts, and the PVC auto-provisions as `-mariadb-logs` by default +* opt-in data-PVC free-space alert at `alerts.pvc_free_enabled` (threshold `alerts.pvc_free_threshold`, default 0.20; severity `alerts.pvc_free_severity`, default warning) +* fix `context` label typo (`datbase`) on the too-many-connections alert +* chart version bumped + ## v0.42.2 - 2026/08/25 * MariaDB version updated to [10.11.19](https://mariadb.com/docs/release-notes/community-server/10.11/10.11.19) * `maria-back-me-up` updated to `10.11-20260825091346` diff --git a/common/mariadb/Chart.yaml b/common/mariadb/Chart.yaml index 0c8c10919be..3a5a6e3b0d9 100644 --- a/common/mariadb/Chart.yaml +++ b/common/mariadb/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 description: A Helm chart for Kubernetes name: mariadb -version: 0.42.2 +version: 0.43.0 # scripts/docker-entyrpoint.sh should be updated when appVersion is updated appVersion: 10.11.19 dependencies: diff --git a/common/mariadb/templates/_helpers.tpl b/common/mariadb/templates/_helpers.tpl index 57f466aa632..f50080c244a 100644 --- a/common/mariadb/templates/_helpers.tpl +++ b/common/mariadb/templates/_helpers.tpl @@ -12,6 +12,35 @@ {{- end -}} {{- end -}} +{{/* MariaDB data directory (datadir mount point). */}} +{{- define "mariadb.dataDir" -}} +/var/lib/mysql +{{- end -}} + +{{/* Name of the data PVC; keeps the deployment claimName and the volume alert in sync. */}} +{{- define "mariadb.dataPvcName" -}} +{{- .Values.persistence_claim.name | default (include "fullName" .) -}} +{{- end -}} + +{{/* Name of the dedicated slow-log PVC referenced by the deployment. */}} +{{- define "mariadb.slowLogPvcName" -}} +{{- .Values.slow_query_log.persistence.name | default (printf "%s-logs" (include "fullName" .)) -}} +{{- end -}} + +{{/* Directory the slow query log lives in (dedicated volume mount point). */}} +{{- define "mariadb.slowLogDir" -}} +{{- .Values.slow_query_log.dir -}} +{{- end -}} + +{{/* Directory scanned by the slow-log cleanup: the dedicated volume when enabled, else the datadir. */}} +{{- define "mariadb.slowLogCleanupDir" -}} +{{- if .Values.slow_query_log.persistence.enabled -}} +{{- include "mariadb.slowLogDir" . -}} +{{- else -}} +{{- include "mariadb.dataDir" . -}} +{{- end -}} +{{- end -}} + {{- define "mariadb.resolve_secret_squote" -}} {{- $str := . -}} {{- if (hasPrefix "vault+kvv2" $str ) -}} diff --git a/common/mariadb/templates/alerts.yaml b/common/mariadb/templates/alerts.yaml index 90a32924a47..6ad0b8c4ea8 100644 --- a/common/mariadb/templates/alerts.yaml +++ b/common/mariadb/templates/alerts.yaml @@ -30,4 +30,7 @@ spec: {{- if .Values.alerts.alert_db_not_ready }} {{ include (print .Template.BasePath "/alerts/_health.alerts.tpl") . | indent 2 }} {{- end }} +{{- if .Values.alerts.pvc_free_enabled }} +{{ include (print .Template.BasePath "/alerts/_volume.alerts.tpl") . | indent 2 }} +{{- end }} {{- end}} diff --git a/common/mariadb/templates/alerts/_mysql.alerts.tpl b/common/mariadb/templates/alerts/_mysql.alerts.tpl index 7fdb39ad1df..f7bbb0da92e 100644 --- a/common/mariadb/templates/alerts/_mysql.alerts.tpl +++ b/common/mariadb/templates/alerts/_mysql.alerts.tpl @@ -4,7 +4,7 @@ expr: (mysql_global_variables_max_connections{app_kubernetes_io_instance=~"{{ include "fullName" . }}"} - mysql_global_status_threads_connected{app_kubernetes_io_instance=~"{{ include "fullName" . }}"} < 200) for: 10m labels: - context: datbase + context: database service: {{ include "alerts.service" . }} severity: info tier: {{ required ".Values.alerts.tier missing" .Values.alerts.tier }} diff --git a/common/mariadb/templates/alerts/_volume.alerts.tpl b/common/mariadb/templates/alerts/_volume.alerts.tpl new file mode 100644 index 00000000000..c7db16e3e97 --- /dev/null +++ b/common/mariadb/templates/alerts/_volume.alerts.tpl @@ -0,0 +1,15 @@ +- name: volume.alerts + rules: + - alert: {{ include "alerts.service" . | title }}MariaDBVolumeNearlyFull + {{- $pvc := include "mariadb.dataPvcName" . }} + expr: (kubelet_volume_stats_available_bytes{persistentvolumeclaim="{{ $pvc }}"} / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim="{{ $pvc }}"}) < {{ .Values.alerts.pvc_free_threshold }} + for: 10m + labels: + context: database + service: {{ include "alerts.service" . }} + severity: {{ .Values.alerts.pvc_free_severity | default "warning" }} + tier: {{ required ".Values.alerts.tier missing" .Values.alerts.tier }} + support_group: {{ required ".Values.alerts.support_group missing" .Values.alerts.support_group }} + annotations: + description: The {{ include "fullName" . }} data volume has less than {{ mulf .Values.alerts.pvc_free_threshold 100 }}% free space left. If it fills up, MariaDB stops accepting writes. + summary: {{ include "fullName" . }} data volume is running low on free space. diff --git a/common/mariadb/templates/deployment.yaml b/common/mariadb/templates/deployment.yaml index eaf7c6c87e6..b33a29058ce 100644 --- a/common/mariadb/templates/deployment.yaml +++ b/common/mariadb/templates/deployment.yaml @@ -81,6 +81,32 @@ spec: readOnly: true - name: init-file-root-sql mountPath: /etc/mysql/init-file +{{- if or .Values.slow_query_log.persistence.enabled .Values.slow_query_log.cleanup.enabled }} + - name: slow-log-prepare + image: {{ required ".Values.global.dockerHubMirrorAlternateRegion is missing" .Values.global.dockerHubMirrorAlternateRegion }}/{{ .Values.image }} + imagePullPolicy: {{ default "IfNotPresent" .Values.imagePullPolicy | quote }} + securityContext: + runAsUser: 0 + command: + - /bin/bash + - -c + - | +{{- if .Values.slow_query_log.persistence.enabled }} + [ "$(stat -c %U {{ include "mariadb.slowLogDir" . }})" = mysql ] || chown -R mysql:mysql {{ include "mariadb.slowLogDir" . }} +{{- end }} +{{- if .Values.slow_query_log.cleanup.enabled }} + find {{ include "mariadb.slowLogCleanupDir" . }} -maxdepth 1 -type f -name '*slow*.log*' -mtime +{{ .Values.slow_query_log.cleanup.max_age_days }} -delete 2>/dev/null || true +{{- end }} + exit 0 + volumeMounts: +{{- if .Values.slow_query_log.persistence.enabled }} + - name: mariadb-logs-storage + mountPath: {{ include "mariadb.slowLogDir" . }} +{{- else if .Values.persistence_claim.enabled }} + - name: mariadb-persistent-storage + mountPath: {{ include "mariadb.dataDir" . }} +{{- end }} +{{- end }} {{- if .Values.global.mariadb.native_sidecar.enabled }} {{- else }} containers: @@ -200,6 +226,14 @@ spec: secretKeyRef: name: mariadb-{{.Values.name}} key: root-password +{{- if .Values.slow_query_log.persistence.enabled }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + args: + - --slow-query-log-file={{ required "slow_query_log.dir required when slow_query_log.persistence.enabled" .Values.slow_query_log.dir }}/$(POD_NAME)-slow.log +{{- end }} ports: - name: mariadb containerPort: 3306 @@ -230,8 +264,12 @@ spec: mountPath: /run/mysqld {{- if .Values.persistence_claim.enabled }} - name: mariadb-persistent-storage - mountPath: /var/lib/mysql + mountPath: {{ include "mariadb.dataDir" . }} readOnly: false +{{- end }} +{{- if .Values.slow_query_log.persistence.enabled }} + - name: mariadb-logs-storage + mountPath: {{ include "mariadb.slowLogDir" . }} {{- end }} - mountPath: /usr/local/bin/docker-entrypoint.sh subPath: docker-entrypoint.sh @@ -271,7 +309,12 @@ spec: {{- if .Values.persistence_claim.enabled }} - name: mariadb-persistent-storage persistentVolumeClaim: - claimName: {{ .Values.persistence_claim.name | default (include "fullName" . ) }} + claimName: {{ include "mariadb.dataPvcName" . }} +{{- end }} +{{- if .Values.slow_query_log.persistence.enabled }} + - name: mariadb-logs-storage + persistentVolumeClaim: + claimName: {{ include "mariadb.slowLogPvcName" . }} {{- end }} - name: initdb secret: diff --git a/common/mariadb/templates/pvc.yaml b/common/mariadb/templates/pvc.yaml index c32c3e8ba25..fda2febf840 100644 --- a/common/mariadb/templates/pvc.yaml +++ b/common/mariadb/templates/pvc.yaml @@ -19,3 +19,25 @@ spec: storageClassName: {{ .Values.persistence_claim.storage_class }} {{- end }} {{ end }} +{{- if and .Values.slow_query_log.persistence.enabled .Values.slow_query_log.persistence.autoprovision }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ include "mariadb.slowLogPvcName" . }} + labels: + {{- include "mariadb.labels" (list $ "noversion" "mariadb" "persistentvolumeclaim" "database") | indent 4 }} +spec: +{{- if .Values.slow_query_log.persistence.access_modes }} + accessModes: + {{- range $accessMode := .Values.slow_query_log.persistence.access_modes }} + - {{ $accessMode }} + {{- end }} +{{- end }} + resources: + requests: + storage: {{ .Values.slow_query_log.persistence.size }} + {{- if .Values.slow_query_log.persistence.storage_class }} + storageClassName: {{ .Values.slow_query_log.persistence.storage_class }} + {{- end }} +{{ end }} diff --git a/common/mariadb/values.yaml b/common/mariadb/values.yaml index 24eb30b9106..09db78edda3 100644 --- a/common/mariadb/values.yaml +++ b/common/mariadb/values.yaml @@ -19,12 +19,27 @@ log_warnings: "2" query_cache_size: "0" query_cache_type: "0" join_buffer_size: "4M" -long_query_time: 5 binlog_format: "MIXED" expire_logs_days: 10 character_set_server: "utf8" # Should be utf8mb4, but we started with this collation_server: "utf8_general_ci" +long_query_time: 5 +slow_query_log: + dir: /var/log/mysql-slow # ignored when persistence.enabled=false + cleanup: + enabled: true # removes old slowlogs on pod init + max_age_days: 14 + persistence: + enabled: false + autoprovision: false + # name: defaults to -mariadb-logs + size: "5Gi" + access_modes: + - ReadWriteOnce + # storage_class: + + db_performance_schema: # Enabling performance schema only (without enabling instruments for data collections). # Instrument can be enabled separately. Only Performance Schema Enablement does not cause any # overhead. @@ -348,6 +363,11 @@ alerts: # Ratio of free pages below which MariaDBBufferPoolExhausted (critical) fires. buffer_pool_critical_threshold: 0.025 + # Warn when the data PVC free ratio drops below pvc_free_threshold. + pvc_free_enabled: false + pvc_free_threshold: 0.20 + pvc_free_severity: warning + vpa: # https://github.com/sapcc/vpa_butler # The maximum available capacity is split evenly across containers specified in the Deployment, StatefulSet or DaemonSet to derive the upper recommendation bound. This does not work out for pods with a single resource-hungry container with several sidecar containers