From 61deeb1f12e79f844a1910156e03d3b7735d2845 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:47:29 -0400 Subject: [PATCH 1/3] storage: bounded PVC autoscaling, because the fill-up alert could never fire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit workspace-minio-pvc hit 100% and took out every push to registry.socioprophet.ai. PR #1091 fixes why it filled (zot kept every tag forever). This fixes why nobody knew. The second half is not "no alert was configured". kube-prometheus-stack ships KubePersistentVolumeFillingUp and it is loaded and healthy in the observability Prometheus right now — state=inactive, health=ok, lastError=none. It queries kubelet_volume_stats_available_bytes{job="kubelet"}. That Prometheus has no kubelet scrape job at all; its job labels are apiserver, kube-state-metrics, node-exporter, otel-collector and the stack's own components. The metric has zero series, so the rule evaluates an empty set forever and sits at inactive no matter what the disks do. On a dashboard that is indistinguishable from "volumes fine". The cause is structural, not a misconfiguration: GKE Autopilot's Warden denies nodes/proxy cluster-wide, so nothing in-cluster can scrape kubelet here. That also rules out every off-the-shelf volume-autoscaler controller — they all read exactly that metric, so one would deploy cleanly, report healthy, and never act, reproducing the failure it was installed to prevent. GKE's managed collection does scrape kubelet and publishes kubelet_volume_stats_* into Google Managed Prometheus over a PromQL endpoint. Verified: 9 PVC-keyed series matching df inside the pods. A CronJob reads that and patches PVCs — no CRDs, no operator, stdlib only. Deliberately the same metric names, so if in-cluster scraping is ever fixed this repoints via PROM_BASE alone. Bounds, because a GCE PD can never be shrunk and every expansion is permanent spend: fire at 75%; step +50% floored at 5Gi and capped at 50Gi per cycle, so a runaway push loop provisions at most 200Gi/hour; a mandatory per-PVC maxGi with no default and no wildcard, so an unconsidered volume is never grown; and a 700Gi estate budget against 402Gi provisioned today as the brake on correlated growth. Every outcome alerts — growing a disk is a symptom, not a success. An empty PromQL result raises rather than reporting all-clear, and an enrolled PVC with no data alerts instead of being skipped: three estate volumes currently publish no stats (mesh-xl-cache is bound but mounted by nothing, arcticdb-gateway-data's pod is in CreateContainerConfigError, workspace-mail-backup has been Pending 12 days), and silently skipping them is the same defect class. postgres-data-pvc is enrolled but can never be grown — static PV, no StorageClass. It is listed so it raises PvcGuardCannotExpand rather than sitting unwatched; the real fix is a migration to dynamic provisioning. Proven by watching it fire, not by reading it. Against a real 1Gi probe volume filled to 82%: grew to 2Gi with FileSystemResizeSuccessful online, then at the 2Gi ceiling and 76% full refused to grow and raised PvcGuardAtMaximum, with both alerts confirmed active in Alertmanager. That run also caught a real bug — patching size and annotations as two requests raced the CSI external-resizer (VolumeResizeFailed: the object has been modified) and left the volume needing a pod restart to finish the filesystem resize. Now a single atomic patch. Alertmanager still has no receivers, so these alerts page nobody yet. That is the next link in the chain and is not addressed here. --- infra/argocd/pvc-capacity-guard.yaml | 25 ++ infra/k8s/pvc-capacity-guard/README.md | 185 +++++++++ .../k8s/pvc-capacity-guard/base/cronjob.yaml | 73 ++++ infra/k8s/pvc-capacity-guard/base/guard.py | 374 ++++++++++++++++++ .../base/kustomization.yaml | 24 ++ infra/k8s/pvc-capacity-guard/base/policy.json | 119 ++++++ infra/k8s/pvc-capacity-guard/base/rbac.yaml | 45 +++ 7 files changed, 845 insertions(+) create mode 100644 infra/argocd/pvc-capacity-guard.yaml create mode 100644 infra/k8s/pvc-capacity-guard/README.md create mode 100644 infra/k8s/pvc-capacity-guard/base/cronjob.yaml create mode 100644 infra/k8s/pvc-capacity-guard/base/guard.py create mode 100644 infra/k8s/pvc-capacity-guard/base/kustomization.yaml create mode 100644 infra/k8s/pvc-capacity-guard/base/policy.json create mode 100644 infra/k8s/pvc-capacity-guard/base/rbac.yaml diff --git a/infra/argocd/pvc-capacity-guard.yaml b/infra/argocd/pvc-capacity-guard.yaml new file mode 100644 index 000000000..2444d4665 --- /dev/null +++ b/infra/argocd/pvc-capacity-guard.yaml @@ -0,0 +1,25 @@ +# Storage capacity guard. Distinct from infra/argocd/autoscaling-stack.yaml, which is +# WORKLOAD autoscaling (KEDA/HPA — replicas in response to load). This is VOLUME +# autoscaling: bounded growth of PVCs in response to fullness. Kubernetes has no native +# PVC autoscaler, so nothing upstream covers it. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: pvc-capacity-guard + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + project: default + destination: + server: https://kubernetes.default.svc + namespace: observability + source: + repoURL: https://github.com/SocioProphet/prophet-platform.git + targetRevision: main + path: infra/k8s/pvc-capacity-guard/base + syncPolicy: + automated: { prune: true, selfHeal: true } + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/infra/k8s/pvc-capacity-guard/README.md b/infra/k8s/pvc-capacity-guard/README.md new file mode 100644 index 000000000..7a7feefb0 --- /dev/null +++ b/infra/k8s/pvc-capacity-guard/README.md @@ -0,0 +1,185 @@ +# pvc-capacity-guard + +Bounded, alerting, automatic growth for estate PersistentVolumeClaims. + +## Why + +On 2026-07-30 `workspace-minio-pvc` hit 100% (20Gi, 184M free). That volume backs both the +workspace object store **and** zot's S3 blob store, so when MinIO started answering blob +commits with `XMinioStorageFull` (HTTP 507), zot tore down its in-flight `.uploads/` session +and every push to `registry.socioprophet.ai` failed with: + +``` +unknown: blob upload unknown to registry +``` + +Four `build (...)` jobs went red on `main` — after building successfully. Only the push broke, +so the Dockerfiles looked innocent. + +Two independent failures had to line up: + +1. **Nothing ever expired.** `storage.gc: true` only reclaims *untagged* blobs, and with no + retention policy no tag ever became untagged, so GC had nothing to do. Fixed by + [PR #1091](https://github.com/SocioProphet/prophet-platform/pull/1091) (`storage.retention` + + `gcInterval: 6h` in `infra/k8s/zot/base/config.json`). +2. **Nothing warned anyone.** Fixed here. + +## The part that is not obvious + +(2) was *not* "no alert was configured". + +kube-prometheus-stack ships `KubePersistentVolumeFillingUp`, and it is loaded and healthy in +the `observability` Prometheus right now — `state=inactive, health=ok, lastError=none`. Its +query is: + +```promql +kubelet_volume_stats_available_bytes{job="kubelet", ...} / kubelet_volume_stats_capacity_bytes{job="kubelet", ...} < 0.03 +``` + +That Prometheus has **no `kubelet` scrape job at all**. Its job label values are `apiserver`, +`kube-state-metrics`, `node-exporter`, `otel-collector`, and the stack's own components — +no kubelet. `kubelet_volume_stats_*` resolves to **zero series**, so the rule evaluates an +empty set forever and sits at `inactive` regardless of what the disks are doing. On every +dashboard that is indistinguishable from "the volumes are fine." + +The reason is structural: GKE Autopilot's Warden denies `nodes/proxy` cluster-wide — + +``` +nodes "gk3-..." is forbidden: ... GKE Warden authz [denied by managed-namespaces-limitation] +``` + +— so kube-prometheus-stack cannot scrape kubelet on this cluster, and neither can any of the +off-the-shelf volume-autoscaler controllers, which all read exactly that metric. + +**A capacity guard nobody has watched fire is not a capacity guard.** See "Proving it fires". + +## Design + +### Why a CronJob and not a controller + +Kubernetes has no native PVC autoscaler. The options were: + +| Option | Verdict | +|---|---| +| A volume-autoscaler controller (e.g. DevOps-Nirvana/kubernetes-volume-autoscaler) | **Rejected.** Every one of them reads `kubelet_volume_stats_*` from an in-cluster Prometheus. On Autopilot that metric does not exist (above), so the controller would deploy cleanly, report healthy, and never act — reproducing the exact failure mode we are fixing. Making it work would mean first getting kubelet scraping past Warden, which is not in our gift. | +| A CronJob reading usage and patching the PVC | **Chosen.** ~250 lines of stdlib Python, no CRDs, no operator lifecycle, no new failure domain. The reconcile loop for "resize a disk occasionally" does not need to be level-triggered — disks fill over hours, not milliseconds. | + +The simplest thing that works, per the brief. A controller would buy continuous reconciliation +we do not need and cost a dependency we cannot satisfy. + +### Where the numbers come from + +GKE's *managed* collection **does** scrape kubelet (`monitoringConfig` enables the `KUBELET` +and `STORAGE` components) and publishes `kubelet_volume_stats_*` into Google Managed +Prometheus, queryable over a standard PromQL endpoint at +`monitoring.googleapis.com/v1/projects/

/location/global/prometheus/api/v1`. + +Verified on this cluster: 9 PVC-keyed series whose values match `df` inside the pods. + +This is a deliberate GCP dependency and worth naming: the guard reads a Google API. The +counter-argument is that these volumes *are* GCE persistent disks provisioned by +`pd.csi.storage.gke.io` — there is no sovereignty to be had in refusing to read Google's +measurement of Google's disk while running on Google's control plane. The mitigation is that +the guard queries the **same metric names** the upstream rule uses, so if in-cluster kubelet +scraping ever becomes possible, repointing it is a one-line change to `PROM_BASE`. + +### Bounds + +Growth is one-way — **a GCE persistent disk can never be shrunk** — so every expansion is +permanent spend and the bounds are what keep a runaway push loop from writing a blank cheque. + +| Bound | Value | Why | +|---|---|---| +| `thresholdPct` | 75 | PD expansion measured ~25s wall-clock on this cluster (20Gi→100Gi, online, no pod restart). At 75% even the smallest enrolled volume (2Gi) keeps 512Mi of headroom, which outlasts one cycle. 90% would be too late for a burst of image-layer writes. | +| `stepPct` / `minStepGi` / `maxStepGi` | 50% / 5Gi / 50Gi | Geometric so large volumes gain useful headroom; floored so small ones do not creep; **capped so the worst case is 50Gi per cycle = 200Gi/hour**, well inside human reaction time. | +| `maxGi` | per-PVC, **mandatory** | No default and no wildcard. A volume nobody has set a ceiling for is a volume the guard will not grow. At the ceiling it raises `PvcGuardAtMaximum` and stops. | +| `estateMaxTotalGi` | 700 | Aggregate brake. Per-PVC ceilings bound *one* runaway volume; this bounds a *correlated* one. Guarded PVCs total 402Gi today, so this caps worst case at ~1.75x current (~$70/month pd-balanced) — a known number, not an open cheque. | +| `cooldownMinutes` | 30 | Two cycles. A PD resize plus filesystem resize can briefly leave utilisation reading against the old capacity; without this the guard would double-grow on stale data. | +| schedule | `*/15 * * * *` | Paired with the 75% threshold this is the real safety margin: notice and finish expanding before the remaining 25% is consumed. | + +### It alerts; it does not silently absorb + +Every outcome reaches Alertmanager. Growing a disk is not a success to be swallowed — it is a +symptom. + +| Alert | Severity | Meaning | +|---|---|---| +| `PvcGuardExpanded` | warning | A volume was grown. Permanent spend; find out why it is growing. | +| `PvcGuardAtMaximum` | critical | Over threshold and at its ceiling. **The guard has stopped.** Raise `maxGi` deliberately or reclaim space. | +| `PvcGuardCannotExpand` | critical | Over threshold on a volume that can never be expanded (static PV / non-expanding StorageClass). Needs manual migration. | +| `PvcGuardNoData` | warning | Enrolled but reporting nothing — its consumer pod is not Running, so the kubelet publishes no stats. **The volume is unmonitored.** | +| `PvcGuardTargetMissing` | warning | Policy and cluster disagree about a PVC's existence. | +| `PvcGuardBroken` | critical | The run itself failed; no volume was evaluated this cycle. | + +`PvcGuardNoData` matters more than it looks. A PVC only produces `kubelet_volume_stats_*` +while a running pod mounts it. Two estate volumes currently produce nothing, so a guard that +*skipped* what it could not see would be silently blind in exactly the way the upstream rule +was. An empty PromQL result is treated as an error, never as "all clear": + +```python +if not rows: + raise RuntimeError("kubelet_volume_stats_* returned ZERO series. Refusing to report + 'all clear' off an empty query ...") +``` + +## Proving it fires + +A guard nobody has watched fire is the defect class this estate keeps hitting. The loop is +exercised end to end against a real volume — see `docs/` in the PR description for the run +transcript. Reproduce with: + +```sh +kubectl proxy --port=8001 & +kubectl -n observability port-forward svc/kube-prometheus-stack-alertmanager 9093:9093 & + +# 1Gi probe PVC filled to ~82%, policy maxGi=2 +GCP_TOKEN=$(gcloud auth print-access-token) \ +KUBE_PROXY=http://127.0.0.1:8001 \ +ALERTMANAGER_URL=http://127.0.0.1:9093 \ +POLICY_PATH=./probe-policy.json \ +python3 infra/k8s/pvc-capacity-guard/base/guard.py +``` + +`DRY_RUN=1` evaluates and logs without patching. `FORCE_UTIL='ns/pvc=0.93'` overrides one +PVC's reading to exercise a branch without physically filling a disk. + +## Prerequisites (not self-contained) + +Workload Identity. The `pvc-capacity-guard` KSA in `observability` is annotated for a GCP +service account that needs `roles/monitoring.viewer` to read GMP. **Not created by this +manifest set** — it is a project-level IAM change: + +```sh +gcloud iam service-accounts create pvc-capacity-guard --project socioprophet-platform +gcloud projects add-iam-policy-binding socioprophet-platform \ + --member "serviceAccount:pvc-capacity-guard@socioprophet-platform.iam.gserviceaccount.com" \ + --role roles/monitoring.viewer +gcloud iam service-accounts add-iam-policy-binding \ + pvc-capacity-guard@socioprophet-platform.iam.gserviceaccount.com \ + --role roles/iam.workloadIdentityUser \ + --member "serviceAccount:socioprophet-platform.svc.id.goog[observability/pvc-capacity-guard]" +``` + +Until that exists the CronJob will fail at `gcp_token()` and raise `PvcGuardBroken` — loudly, +which is the intended behaviour for an unconfigured guard. + +## Volumes this does NOT protect + +| PVC | Problem | +|---|---| +| `socioprophet/postgres-data-pvc` | Bound to the static PV `postgres-data-pv` with no StorageClass. **No mechanism can expand it.** The estate's primary database needs a manual migration to dynamic provisioning. Enrolled so it alerts rather than sitting unwatched. | +| `socioprophet/workspace-mail-backup` | Pending for 12 days — its consumer pod never schedules, so it never bound. Also the single highest latent risk in the estate: `mail-backup` writes a **new timestamped full tarball** of the maildir every run and deletes nothing. Fix its retention *before* fixing its scheduling. | +| `socioprophet/arcticdb-gateway-data` | Its pod has been in `CreateContainerConfigError` for ~8h, so no volume stats are published. | + +## Related debt this does not fix + +The guard buys time. These are the actual leaks: + +- **`clickhouse-data`** — the DDL partitions by month across 7 tables but declares **no `TTL` + clause anywhere**, and nothing drops old partitions. +- **`workspace-mail-pvc`** — Dovecot runs with no quota plugin and no `expire`. +- **`mesh-qdrant`** — the snapshot CronJob writes snapshots onto the same PVC it is snapshotting + and never calls the delete API. Two bugs currently stop it running; fixing them turns on + unbounded growth. +- **Alertmanager has no receivers configured**, so today these alerts land in Alertmanager and + page nobody. Wiring a receiver is the necessary next step for any of this to reach a human. diff --git a/infra/k8s/pvc-capacity-guard/base/cronjob.yaml b/infra/k8s/pvc-capacity-guard/base/cronjob.yaml new file mode 100644 index 000000000..c59ec92ab --- /dev/null +++ b/infra/k8s/pvc-capacity-guard/base/cronjob.yaml @@ -0,0 +1,73 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: pvc-capacity-guard + namespace: observability +spec: + # Every 15 minutes. Paired with thresholdPct 75 this is the real safety margin: the guard + # must notice and finish expanding before the remaining 25% is consumed. For the registry + # volume at 100Gi that is 25Gi of headroom per cycle — far more than any plausible burst + # of image-layer pushes in 15 minutes. + schedule: "*/15 * * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app: pvc-capacity-guard + spec: + serviceAccountName: pvc-capacity-guard + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: guard + # Pulled DIRECTLY from docker.io, deliberately not through + # registry.socioprophet.ai. This guard's whole job is to keep the volume that + # backs the registry from filling; routing its own image through that registry + # would mean the recovery path fails in exactly the circumstance it exists for. + # Same rule the zot README states: never make zot the only path to its own + # recovery. Stdlib-only script, so a bare python image needs no pip install. + image: python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de + command: ["python3", "/opt/guard/guard.py"] + env: + - name: GCP_PROJECT + value: socioprophet-platform + - name: POLICY_PATH + value: /etc/guard/policy.json + - name: ALERTMANAGER_URL + value: http://kube-prometheus-stack-alertmanager.observability.svc:9093 + volumeMounts: + - name: guard + mountPath: /opt/guard/guard.py + subPath: guard.py + readOnly: true + - name: guard + mountPath: /etc/guard/policy.json + subPath: policy.json + readOnly: true + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: guard + configMap: + name: pvc-capacity-guard + defaultMode: 0555 diff --git a/infra/k8s/pvc-capacity-guard/base/guard.py b/infra/k8s/pvc-capacity-guard/base/guard.py new file mode 100644 index 000000000..b6a5909e5 --- /dev/null +++ b/infra/k8s/pvc-capacity-guard/base/guard.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +"""pvc-capacity-guard — bounded, alerting, automatic growth for estate PVCs. + +WHY THIS EXISTS +--------------- +On 2026-07-30 `workspace-minio-pvc` reached 100% (20Gi, 184M free). MinIO answered blob +commits with `XMinioStorageFull` (HTTP 507), zot tore down its in-flight `.uploads/` +session, and every image push to registry.socioprophet.ai failed with the very unhelpful +`blob upload unknown to registry`. Four `build (...)` jobs went red on main. + +Two independent failures had to line up for that: + + 1. Nothing ever expired. Fixed separately by zot's retention policy (PR #1091). + 2. Nothing warned anyone. Fixed here. + +(2) is the subtle one, and it is NOT "no alert was configured". An alert WAS configured: +kube-prometheus-stack ships `KubePersistentVolumeFillingUp`, and it is loaded and healthy +in the `observability` Prometheus right now. It queries: + + kubelet_volume_stats_available_bytes{job="kubelet", ...} + +That Prometheus has **no `kubelet` scrape job at all** — GKE Autopilot's Warden denies +`nodes/proxy` cluster-wide, so kube-prometheus-stack cannot scrape kubelet here. The +metric has zero series, so the rule evaluates an empty set forever and reports +`state=inactive, health=ok, lastError=none` — which is indistinguishable, on every +dashboard, from "the volumes are fine". It was structurally incapable of firing. + +So the design constraint is not "add monitoring", it is "add monitoring that is provably +reading real data, on a cluster where the usual source is unavailable". + +WHERE THE NUMBERS COME FROM +--------------------------- +GKE's *managed* collection does scrape kubelet (monitoringConfig enables the KUBELET and +STORAGE components) and publishes `kubelet_volume_stats_*` into Google Managed Prometheus, +which is queryable over a standard PromQL endpoint. Verified on this cluster: 9 PVC-keyed +series, matching `df` inside the pods. + +Deliberately the *same metric names* upstream uses, so if kubelet scraping is ever fixed +in-cluster this can be repointed at the in-cluster Prometheus by changing PROM_BASE alone. + +BLIND SPOTS ARE FAILURES, NOT SKIPS +----------------------------------- +A PVC only produces `kubelet_volume_stats_*` while a running pod mounts it. Two estate +PVCs currently produce nothing (`arcticdb-gateway-data` — its pod is in +CreateContainerConfigError; `workspace-mail-backup` — Pending for 12 days). A guard that +silently skips what it cannot see reproduces the exact defect it was built to fix, so an +enrolled PVC with no data is an ALERT, never a skip. + +Requires no third-party Python packages: stdlib only. +""" + +from __future__ import annotations + +import json +import os +import ssl +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone + +GI = 1024 ** 3 + +# --- endpoints ------------------------------------------------------------------------- +PROJECT = os.environ.get("GCP_PROJECT", "socioprophet-platform") +PROM_BASE = os.environ.get( + "PROM_BASE", + f"https://monitoring.googleapis.com/v1/projects/{PROJECT}/location/global/prometheus/api/v1", +) +ALERTMANAGER = os.environ.get( + "ALERTMANAGER_URL", + "http://kube-prometheus-stack-alertmanager.observability.svc:9093", +) +POLICY_PATH = os.environ.get("POLICY_PATH", "/etc/guard/policy.json") + +# Talking to the API server: in-cluster by default, via `kubectl proxy` when testing. +KUBE_PROXY = os.environ.get("KUBE_PROXY") # e.g. http://127.0.0.1:8001 +SA_DIR = "/var/run/secrets/kubernetes.io/serviceaccount" + +DRY_RUN = os.environ.get("DRY_RUN", "").lower() in ("1", "true", "yes") +# Test seam: force a utilisation reading for one PVC, e.g. "socioprophet/foo=0.93". +# Used to prove the guard fires without waiting for a volume to genuinely fill. +FORCE_UTIL = os.environ.get("FORCE_UTIL", "") + + +def log(msg: str) -> None: + print(f"[{datetime.now(timezone.utc).isoformat(timespec='seconds')}] {msg}", flush=True) + + +# --- auth ------------------------------------------------------------------------------ +def gcp_token() -> str: + """Workload Identity token from the GKE metadata server (or GCP_TOKEN when testing).""" + tok = os.environ.get("GCP_TOKEN") + if tok: + return tok.strip() + req = urllib.request.Request( + "http://metadata.google.internal/computeMetadata/v1/" + "instance/service-accounts/default/token", + headers={"Metadata-Flavor": "Google"}, + ) + with urllib.request.urlopen(req, timeout=10) as r: + return json.load(r)["access_token"] + + +def kube_ctx() -> tuple[str, dict, ssl.SSLContext | None]: + if KUBE_PROXY: + return KUBE_PROXY.rstrip("/"), {}, None + with open(f"{SA_DIR}/token") as fh: + tok = fh.read().strip() + ctx = ssl.create_default_context(cafile=f"{SA_DIR}/ca.crt") + host = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc") + port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") + return f"https://{host}:{port}", {"Authorization": f"Bearer {tok}"}, ctx + + +def _http(method: str, url: str, headers: dict, body: bytes | None, + ctx: ssl.SSLContext | None, timeout: int = 30): + req = urllib.request.Request(url, method=method, data=body, headers=headers) + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r: + raw = r.read() + return json.loads(raw) if raw else {} + + +# --- signal ---------------------------------------------------------------------------- +def promql(query: str) -> list: + url = f"{PROM_BASE}/query?" + urllib.parse.urlencode({"query": query}) + out = _http("GET", url, {"Authorization": f"Bearer {gcp_token()}"}, None, None) + if out.get("status") != "success": + raise RuntimeError(f"PromQL failed: {json.dumps(out)[:300]}") + return out["data"]["result"] + + +def utilisation() -> dict[str, float]: + """{'namespace/pvc': fraction_used}. Empty result is an error, never 'all clear'.""" + rows = promql( + "kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes" + ) + if not rows: + raise RuntimeError( + "kubelet_volume_stats_* returned ZERO series. Refusing to report 'all clear' " + "off an empty query — that is precisely how KubePersistentVolumeFillingUp " + "stayed green while the registry volume hit 100%." + ) + out = {} + for r in rows: + m = r["metric"] + key = f"{m.get('namespace')}/{m.get('persistentvolumeclaim')}" + out[key] = float(r["value"][1]) + for override in filter(None, FORCE_UTIL.split(",")): + k, _, v = override.partition("=") + out[k.strip()] = float(v) + log(f"FORCE_UTIL: {k.strip()} treated as {float(v) * 100:.1f}%") + return out + + +# --- kubernetes ------------------------------------------------------------------------ +def get_pvc(base, hdrs, ctx, ns, name): + return _http("GET", f"{base}/api/v1/namespaces/{ns}/persistentvolumeclaims/{name}", + hdrs, None, ctx) + + +def grow_pvc(base, hdrs, ctx, ns, name, new_gi: int, annos: dict): + """Resize AND record bookkeeping in ONE patch. + + This must be a single request. Doing it as two back-to-back patches races the CSI + external-resizer: it reacts to the size change within milliseconds and patches the + PVC's status to mark node expansion required, and the second (annotation) patch bumps + resourceVersion underneath it. Observed exactly that during the fired-proof: + + Warning VolumeResizeFailed external-resizer pd.csi.storage.gke.io + mark PVC ... as node expansion required failed: can't patch status of PVC ... + Operation cannot be fulfilled ...: the object has been modified; please apply + your changes to the latest version and try again + + The disk still grew, but the interrupted handshake left the volume needing a pod + restart to finish the filesystem resize instead of completing online. + """ + body = json.dumps({ + "metadata": {"annotations": annos}, + "spec": {"resources": {"requests": {"storage": f"{new_gi}Gi"}}}, + }).encode() + h = dict(hdrs, **{"Content-Type": "application/merge-patch+json"}) + return _http("PATCH", f"{base}/api/v1/namespaces/{ns}/persistentvolumeclaims/{name}", + h, body, ctx) + + +def sc_expandable(base, hdrs, ctx, sc_name: str | None) -> bool: + """A PVC with no StorageClass is bound to a static PV and can never be expanded.""" + if not sc_name: + return False + try: + sc = _http("GET", f"{base}/apis/storage.k8s.io/v1/storageclasses/{sc_name}", + hdrs, None, ctx) + return bool(sc.get("allowVolumeExpansion")) + except urllib.error.HTTPError: + return False + + +def parse_gi(q: str) -> float: + units = {"Ki": 1 / (1024 ** 2), "Mi": 1 / 1024, "Gi": 1.0, "Ti": 1024.0, + "K": 1e3 / GI, "M": 1e6 / GI, "G": 1e9 / GI, "T": 1e12 / GI} + for suffix, mult in sorted(units.items(), key=lambda kv: -len(kv[0])): + if q.endswith(suffix): + return float(q[: -len(suffix)]) * mult + return float(q) / GI + + +# --- alerting -------------------------------------------------------------------------- +def alert(name: str, severity: str, summary: str, description: str, extra: dict) -> None: + payload = [{ + "labels": dict({"alertname": name, "severity": severity, + "component": "pvc-capacity-guard"}, **extra), + "annotations": {"summary": summary, "description": description}, + "startsAt": datetime.now(timezone.utc).isoformat(), + }] + try: + _http("POST", f"{ALERTMANAGER}/api/v2/alerts", + {"Content-Type": "application/json"}, json.dumps(payload).encode(), None, 10) + log(f"ALERT[{severity}] {name}: {summary}") + except Exception as exc: # never let a broken alert path stop remediation + log(f"ALERT-DELIVERY-FAILED {name}: {exc} :: {summary}") + + +# --- policy ---------------------------------------------------------------------------- +def load_policy() -> dict: + with open(POLICY_PATH) as fh: + return json.load(fh) + + +def main() -> int: + pol = load_policy() + d = pol.get("defaults", {}) + thresh = d.get("thresholdPct", 75) / 100.0 + step_pct = d.get("stepPct", 50) / 100.0 + min_step = d.get("minStepGi", 5) + max_step = d.get("maxStepGi", 50) + cooldown_s = d.get("cooldownMinutes", 30) * 60 + budget_gi = pol.get("estateMaxTotalGi", 500) + + base, hdrs, ctx = kube_ctx() + util = utilisation() + log(f"read {len(util)} PVC utilisation series from GMP") + + # Estate budget is computed over the volumes this guard is allowed to grow. + provisioned = 0.0 + for ent in pol["pvcs"]: + try: + pvc = get_pvc(base, hdrs, ctx, ent["namespace"], ent["name"]) + provisioned += parse_gi(pvc["spec"]["resources"]["requests"]["storage"]) + except Exception: + pass + log(f"estate provisioned across guarded PVCs: {provisioned:.0f}Gi / budget {budget_gi}Gi") + + grew = blind = capped = 0 + + for ent in pol["pvcs"]: + ns, name = ent["namespace"], ent["name"] + key = f"{ns}/{name}" + max_gi = ent["maxGi"] # mandatory: no ceiling, no enrolment + t = ent.get("thresholdPct", d.get("thresholdPct", 75)) / 100.0 + + try: + pvc = get_pvc(base, hdrs, ctx, ns, name) + except urllib.error.HTTPError as exc: + alert("PvcGuardTargetMissing", "warning", + f"{key} is enrolled but does not exist", + f"HTTP {exc.code} fetching the PVC. Policy and cluster disagree.", + {"namespace": ns, "persistentvolumeclaim": name}) + continue + + cur_gi = parse_gi(pvc["spec"]["resources"]["requests"]["storage"]) + annos = (pvc.get("metadata") or {}).get("annotations") or {} + + # -- blind spot: enrolled but no data. Alert; never treat as healthy. ------------ + if key not in util: + blind += 1 + alert("PvcGuardNoData", "warning", + f"{key} is enrolled but reports no utilisation data", + "kubelet_volume_stats_* has no series for this PVC — usually its " + "consumer pod is not Running (Pending/CreateContainerConfigError), so " + "the kubelet publishes nothing. The volume is UNMONITORED: this guard " + "cannot grow what it cannot measure.", + {"namespace": ns, "persistentvolumeclaim": name}) + continue + + pct = util[key] + log(f"{key}: {pct * 100:.1f}% of {cur_gi:.0f}Gi (threshold {t * 100:.0f}%, max {max_gi}Gi)") + if pct < t: + continue + + # -- non-expandable volumes cannot be helped by this guard ---------------------- + if not sc_expandable(base, hdrs, ctx, pvc["spec"].get("storageClassName")): + alert("PvcGuardCannotExpand", "critical", + f"{key} is {pct * 100:.0f}% full and CANNOT be expanded", + "Its StorageClass does not allow volume expansion (or it is bound to a " + "static PV with no StorageClass). This needs a manual migration to a " + "larger volume — automatic growth is impossible.", + {"namespace": ns, "persistentvolumeclaim": name}) + continue + + # -- cooldown: PD resize + filesystem resize is not instantaneous --------------- + last = annos.get("capacity-guard.socioprophet.ai/last-grown") + if last: + try: + age = time.time() - datetime.fromisoformat(last).timestamp() + if age < cooldown_s: + log(f" cooldown: grew {age / 60:.0f}m ago (<{cooldown_s / 60:.0f}m), skipping") + continue + except ValueError: + pass + + # -- already at its ceiling: alert loudly, do NOT grow -------------------------- + if cur_gi >= max_gi: + capped += 1 + alert("PvcGuardAtMaximum", "critical", + f"{key} is {pct * 100:.0f}% full and already at its {max_gi}Gi ceiling", + "The capacity guard will NOT grow this volume further. Raise maxGi " + "deliberately, or reclaim space (retention/GC). Writes will begin to " + "fail when it reaches 100%.", + {"namespace": ns, "persistentvolumeclaim": name}) + continue + + # -- bounded step --------------------------------------------------------------- + step = max(min_step, min(max_step, cur_gi * step_pct)) + new_gi = int(min(cur_gi + step, max_gi)) + + if provisioned - cur_gi + new_gi > budget_gi: + alert("PvcGuardBudgetExhausted", "critical", + f"{key} needs growth but the estate storage budget is exhausted", + f"Growing to {new_gi}Gi would put guarded PVCs at " + f"{provisioned - cur_gi + new_gi:.0f}Gi against a {budget_gi}Gi budget. " + "Refusing. This is the runaway-provisioning brake.", + {"namespace": ns, "persistentvolumeclaim": name}) + continue + + if DRY_RUN: + log(f" DRY_RUN: would grow {cur_gi:.0f}Gi -> {new_gi}Gi") + continue + + grow_pvc(base, hdrs, ctx, ns, name, new_gi, { + "capacity-guard.socioprophet.ai/last-grown": + datetime.now(timezone.utc).isoformat(timespec="seconds"), + "capacity-guard.socioprophet.ai/last-grown-from": f"{cur_gi:.0f}Gi", + }) + provisioned = provisioned - cur_gi + new_gi + grew += 1 + # Growth is one-way: a GCE PD can never be shrunk, so every expansion is + # permanent spend. It gets an alert, not just a log line. + alert("PvcGuardExpanded", "warning", + f"{key} grown {cur_gi:.0f}Gi -> {new_gi}Gi ({pct * 100:.0f}% full)", + f"Automatic expansion by pvc-capacity-guard; ceiling {max_gi}Gi. GCE " + "persistent disks cannot be shrunk, so this is permanent. Investigate why " + "the volume is growing — the guard buys time, it does not fix a leak.", + {"namespace": ns, "persistentvolumeclaim": name}) + + log(f"done: grew={grew} at-ceiling={capped} blind={blind} " + f"checked={len(pol['pvcs'])}") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as exc: + log(f"FATAL: {type(exc).__name__}: {exc}") + try: + alert("PvcGuardBroken", "critical", "pvc-capacity-guard run failed", + f"{type(exc).__name__}: {exc}. No volume was evaluated this cycle — " + "the estate is running without capacity protection.", {}) + except Exception: + pass + sys.exit(1) diff --git a/infra/k8s/pvc-capacity-guard/base/kustomization.yaml b/infra/k8s/pvc-capacity-guard/base/kustomization.yaml new file mode 100644 index 000000000..8e074c0a7 --- /dev/null +++ b/infra/k8s/pvc-capacity-guard/base/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability +labels: + - includeSelectors: false + pairs: + app: pvc-capacity-guard + app.kubernetes.io/part-of: platform-ops +resources: + - rbac.yaml + - cronjob.yaml + +# Generated rather than hand-written, for the same reason zot's config is (see +# infra/k8s/zot/base/kustomization.yaml): the generator's name hash changes the pod spec +# whenever the script or the policy changes. For a CronJob the stakes are lower than for a +# long-lived process — each run mounts the ConfigMap fresh — but the hash also makes an +# edit to guard.py or policy.json visible as a diff on the CronJob itself, so a policy +# change shows up in ArgoCD as a change to the workload rather than a silent ConfigMap +# mutation nobody reviews. +configMapGenerator: + - name: pvc-capacity-guard + files: + - guard.py + - policy.json diff --git a/infra/k8s/pvc-capacity-guard/base/policy.json b/infra/k8s/pvc-capacity-guard/base/policy.json new file mode 100644 index 000000000..f5f7e9d0e --- /dev/null +++ b/infra/k8s/pvc-capacity-guard/base/policy.json @@ -0,0 +1,119 @@ +{ + "_comment": [ + "Enrolment is EXPLICIT and every entry must carry a maxGi. There is no default ceiling", + "and no wildcard: a volume nobody has thought about a bound for is a volume this guard", + "will not grow. Adding a PVC here is a deliberate act that says 'I accept up to maxGi of", + "permanent spend on this volume' — GCE persistent disks cannot be shrunk, so every", + "expansion is irreversible.", + "", + "Volumes are enrolled even when the guard cannot grow them (postgres-data-pvc is on a", + "static PV; workspace-mail-backup has never bound). Enrolment is what makes the guard", + "ALERT about them each cycle instead of leaving them unwatched.", + "", + "estateMaxTotalGi is the aggregate brake. Per-PVC maxGi bounds one runaway volume; the", + "estate budget bounds a correlated one (e.g. a push loop plus a log flood plus a backup", + "job all growing at once). Guarded PVCs total 402Gi today; 700Gi caps worst-case", + "provisioned storage at ~1.75x current, roughly $70/month of pd-balanced — a known,", + "bounded number rather than an open cheque." + ], + + "estateMaxTotalGi": 700, + + "defaults": { + "_comment": [ + "thresholdPct 75: PD expansion measured ~25s wall-clock on this cluster (20Gi->100Gi,", + "online, no pod restart). At 75% even the smallest enrolled volume (2Gi) still has", + "512Mi of headroom, which comfortably outlasts one 15-minute cycle. Firing at 90%+", + "would be too late for a volume taking a burst of image-layer writes.", + "", + "stepPct 50 with minStepGi 5 / maxStepGi 50: geometric growth so large volumes gain", + "meaningful headroom, floored so tiny volumes do not creep up 1Gi at a time, and", + "CAPPED so the worst case a runaway push loop can provision is 50Gi per cycle —", + "200Gi/hour, well inside the time it takes a human to notice the alerts.", + "", + "cooldownMinutes 30: two cycles. A PD resize plus filesystem resize can leave", + "utilisation briefly reading against the old capacity; without a cooldown the guard", + "would double-grow on stale data." + ], + "thresholdPct": 75, + "stepPct": 50, + "minStepGi": 5, + "maxStepGi": 50, + "cooldownMinutes": 30 + }, + + "pvcs": [ + { + "namespace": "socioprophet", + "name": "workspace-minio-pvc", + "maxGi": 250, + "_note": "Registry blob store (zot S3 backend) AND workspace object storage. The volume that failed on 2026-07-30. zot retention (PR #1091) is the primary bound; this guard is the backstop for when retention is wrong or a push burst outruns GC." + }, + { + "namespace": "scm", + "name": "gitea-data", + "maxGi": 150, + "_note": "Sovereign SCM. 122 repos are migrating in, so growth here is expected and lumpy." + }, + { + "namespace": "serving", + "name": "mesh-xl-cache", + "maxGi": 200, + "_note": "Model/artifact cache. Largest volume in the estate at 120Gi." + }, + { + "namespace": "socioprophet", + "name": "clickhouse-data", + "maxGi": 100, + "_note": "UNBOUNDED BY DESIGN: the DDL partitions by month but declares no TTL on any of the 7 tables, so nothing ever drops old partitions. Needs a real retention policy; this ceiling only buys time." + }, + { + "namespace": "observability", + "name": "storage-loki-0", + "maxGi": 50, + "_note": "Log store. Verify Loki's own retention is set — a log volume without retention grows forever by construction." + }, + { + "namespace": "socioprophet", + "name": "workspace-mail-pvc", + "maxGi": 50, + "_note": "Dovecot maildir with no quota plugin and no expire configured. Base manifest asks for 50Gi; the p0-lab overlay shrinks it to 10Gi." + }, + { + "namespace": "socioprophet", + "name": "arcticdb-gateway-data", + "maxGi": 50, + "_note": "Currently BLIND: its pod has been in CreateContainerConfigError for ~8h, so the kubelet publishes no volume stats. Enrolled so the guard alerts on the blind spot every cycle." + }, + { + "namespace": "socioprophet", + "name": "workspace-mail-backup", + "maxGi": 60, + "_note": "Highest latent risk in the estate. The mail-backup CronJob writes a NEW timestamped full tarball of the maildir on every run and deletes nothing; successfulJobsHistoryLimit bounds Job objects, not files. It has never actually run (PVC Pending 12 days, consumer pod never schedules) so the growth is latent. Fix the retention BEFORE fixing the scheduling, or this fills within days." + }, + { + "namespace": "socioprophet", + "name": "zot-cache", + "maxGi": 20, + "_note": "zot boltdb metadata AND the sync extension's downloadDir — pull-through staging from docker.io/ghcr/gcr/registry.k8s.io lands here, which the 'stays small' comment on the manifest does not account for." + }, + { + "namespace": "socioprophet", + "name": "workspace-caldav-pvc", + "maxGi": 20, + "_note": "Radicale collections. No history/cleanup configured." + }, + { + "namespace": "socioprophet", + "name": "compute-gateway-data", + "maxGi": 10, + "_note": "Smallest enrolled volume (2Gi) — the minStepGi floor matters most here." + }, + { + "namespace": "socioprophet", + "name": "postgres-data-pvc", + "maxGi": 50, + "_note": "CANNOT BE GROWN. Bound to the static PV postgres-data-pv with no StorageClass, so no expansion is possible by any mechanism. Enrolled purely so the guard raises PvcGuardCannotExpand instead of leaving the estate's primary database silently unprotected. Remediation is a manual migration to a dynamically-provisioned volume." + } + ] +} diff --git a/infra/k8s/pvc-capacity-guard/base/rbac.yaml b/infra/k8s/pvc-capacity-guard/base/rbac.yaml new file mode 100644 index 000000000..98e6d6524 --- /dev/null +++ b/infra/k8s/pvc-capacity-guard/base/rbac.yaml @@ -0,0 +1,45 @@ +# Least privilege for the capacity guard: it reads PVCs and StorageClasses estate-wide and +# may modify exactly two things on a PVC — spec.resources.requests.storage (via patch) and +# its own bookkeeping annotations. No create, no delete. Kubernetes RBAC cannot restrict a +# patch to specific fields, so `patch` on persistentvolumeclaims is the tightest verb +# available; the ceiling that actually bounds the blast radius is policy.json's maxGi plus +# estateMaxTotalGi, not RBAC. +# +# Note what is deliberately ABSENT: nodes/proxy. GKE Autopilot's Warden denies it cluster-wide +# ("denied by managed-namespaces-limitation"), which is why the usual kubelet-scraping +# volume-autoscalers cannot work on this cluster and why the signal comes from GMP instead. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pvc-capacity-guard + namespace: observability + annotations: + # Workload Identity. The GSA needs roles/monitoring.viewer on the project to read + # kubelet_volume_stats_* out of Google Managed Prometheus. Created out of band — see + # the README's "Prerequisites" section; this is the one part that is not self-contained. + iam.gke.io/gcp-service-account: pvc-capacity-guard@socioprophet-platform.iam.gserviceaccount.com +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pvc-capacity-guard +rules: + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pvc-capacity-guard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pvc-capacity-guard +subjects: + - kind: ServiceAccount + name: pvc-capacity-guard + namespace: observability From 66cf1dc3e14f279c76b2aa5d8affb574ecd592b0 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:43:24 -0400 Subject: [PATCH 2/3] storage: move the capacity guard into the tree Argo actually watches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard was authored in infra/argocd/. The root app-of-apps (infra/tofu/environments/gcp-gke/argocd.tf, gitops_path=deploy/argocd, directory.recurse) has never watched that tree — as the header of deploy/argocd/observability-services.yaml already records, "nothing under infra/argocd/ ever reached the cluster." So the PR as authored deploys nothing: the CronJob that stops PVCs filling would itself have been a control that exists, reports healthy, and cannot fire. Same defect class as the KubePersistentVolumeFillingUp alert the PR was written to replace, one layer up. Only the Application manifest moves. The kustomize base stays where it was authored at infra/k8s/pvc-capacity-guard/base — the root app only needs the Application itself inside the watched tree. Also brought to the conventions of the live tree: - socioprophet.io/tier: reference. MANDATORY — tools/preflight_deploy_contract.py check 5 walks deploy/argocd and fails any Application without it. Verified both ways: with the annotation the gate exits 0, without it exits 1 naming this file. reference rather than foundation because the guard is a replaceable implementation running a digest-pinned stock python image, not part of the interoperability spine (check 6 would require a foundation image to be built by images.yml). - repoURL without the .git suffix, which is the form used throughout deploy/argocd; the suffixed form survives only in the two dead trees. - app.kubernetes.io/part-of: platform-ops, matching the label the kustomization already stamps on the workload. --- deploy/argocd/pvc-capacity-guard.yaml | 44 +++++++++++++++++++++++++++ infra/argocd/pvc-capacity-guard.yaml | 25 --------------- 2 files changed, 44 insertions(+), 25 deletions(-) create mode 100644 deploy/argocd/pvc-capacity-guard.yaml delete mode 100644 infra/argocd/pvc-capacity-guard.yaml diff --git a/deploy/argocd/pvc-capacity-guard.yaml b/deploy/argocd/pvc-capacity-guard.yaml new file mode 100644 index 000000000..9d435f5a5 --- /dev/null +++ b/deploy/argocd/pvc-capacity-guard.yaml @@ -0,0 +1,44 @@ +# Storage capacity guard. Distinct from infra/argocd/autoscaling-stack.yaml, which is +# WORKLOAD autoscaling (KEDA/HPA — replicas in response to load). This is VOLUME +# autoscaling: bounded growth of PVCs in response to fullness. Kubernetes has no native +# PVC autoscaler, so nothing upstream covers it. +# +# Lives in deploy/argocd/, NOT infra/argocd/. The tofu-created root app-of-apps +# (infra/tofu/environments/gcp-gke/argocd.tf) watches `deploy/argocd` with +# directory.recurse — nothing under infra/argocd/ has ever reached the cluster. An +# Application authored there deploys nothing, which for a guard whose whole purpose is +# to not silently do nothing would have been the same defect one layer up. Same move, +# and the same reason, as the Wave 0-3 observability apps (see the header of +# deploy/argocd/observability-services.yaml). The kustomize base stays where it was +# authored: only the Application manifest has to live in the watched tree. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: pvc-capacity-guard + namespace: argocd + labels: + app.kubernetes.io/part-of: platform-ops + annotations: + argocd.argoproj.io/sync-wave: "1" + # Required by tools/preflight_deploy_contract.py check 5. reference, not + # foundation: this is a replaceable implementation of volume autoscaling (the + # PR rejects the controller-shaped alternatives on Autopilot), not part of the + # interoperability spine, and it runs a digest-pinned stock python image rather + # than anything images.yml builds. + socioprophet.io/tier: "reference" +spec: + project: default + destination: + server: https://kubernetes.default.svc + namespace: observability + source: + # No .git suffix — the convention in this tree (the suffixed form appears only + # in the two dead trees). + repoURL: https://github.com/SocioProphet/prophet-platform + targetRevision: main + path: infra/k8s/pvc-capacity-guard/base + syncPolicy: + automated: { prune: true, selfHeal: true } + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/infra/argocd/pvc-capacity-guard.yaml b/infra/argocd/pvc-capacity-guard.yaml deleted file mode 100644 index 2444d4665..000000000 --- a/infra/argocd/pvc-capacity-guard.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Storage capacity guard. Distinct from infra/argocd/autoscaling-stack.yaml, which is -# WORKLOAD autoscaling (KEDA/HPA — replicas in response to load). This is VOLUME -# autoscaling: bounded growth of PVCs in response to fullness. Kubernetes has no native -# PVC autoscaler, so nothing upstream covers it. -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: pvc-capacity-guard - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "1" -spec: - project: default - destination: - server: https://kubernetes.default.svc - namespace: observability - source: - repoURL: https://github.com/SocioProphet/prophet-platform.git - targetRevision: main - path: infra/k8s/pvc-capacity-guard/base - syncPolicy: - automated: { prune: true, selfHeal: true } - syncOptions: - - CreateNamespace=true - - ServerSideApply=true From 008358f0303229327cb6d276c1903709433ad74f Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:58:27 -0400 Subject: [PATCH 3/3] pvc-capacity-guard: fail closed on an unknown budget; pin the query to this cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes to a control that could report headroom it cannot see: 1. The estate-budget accumulator did `except Exception: pass`, so an unreadable enrolled PVC was silently counted as 0Gi. That undercounts provisioned, and the aggregate brake (provisioned - cur + new > budget) would then permit a grow that actually breaches estateMaxTotalGi. Now a failed read makes the estate total UNKNOWN, raises PvcGuardBudgetUnknown, and refuses every grow this cycle — the same fail-closed posture utilisation() already takes on an empty result. 2. The utilisation query hit Google Managed Prometheus (project-wide) unscoped, so a second cluster's kubelet_volume_stats_* collide on namespace/pvc and this guard — which PATCHES disks — could resize a volume off another cluster's fill level. The query is now pinned to cluster (+location), read from env or the GKE metadata server; an undeterminable scope fails closed rather than sweeping the project. Adds a stdlib pytest proving a simulated read error near budget now refuses (was: grew) and that the query carries the cluster matchers. --- infra/k8s/pvc-capacity-guard/base/guard.py | 80 +++++++++++- .../tests/test_guard_fail_closed.py | 117 ++++++++++++++++++ 2 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 infra/k8s/pvc-capacity-guard/tests/test_guard_fail_closed.py diff --git a/infra/k8s/pvc-capacity-guard/base/guard.py b/infra/k8s/pvc-capacity-guard/base/guard.py index b6a5909e5..3e2ad3816 100644 --- a/infra/k8s/pvc-capacity-guard/base/guard.py +++ b/infra/k8s/pvc-capacity-guard/base/guard.py @@ -84,6 +84,16 @@ # Used to prove the guard fires without waiting for a volume to genuinely fill. FORCE_UTIL = os.environ.get("FORCE_UTIL", "") +# This guard PATCHES disks off a project-wide datasource, so every query MUST be pinned +# to this cluster. GMP attaches cluster/location to every series; without the matchers a +# second cluster feeding the same project returns kubelet_volume_stats_* that collide on +# the namespace/pvc key, and the guard could resize a volume here off another cluster's +# fill level. Read from env, else the GKE metadata server (same source as gcp_token), so +# it self-scopes in-cluster; unknown scope fails closed in utilisation() rather than +# querying estate-wide. +CLUSTER_NAME = os.environ.get("CLUSTER_NAME", "") +CLUSTER_LOCATION = os.environ.get("CLUSTER_LOCATION", "") + def log(msg: str) -> None: print(f"[{datetime.now(timezone.utc).isoformat(timespec='seconds')}] {msg}", flush=True) @@ -104,6 +114,33 @@ def gcp_token() -> str: return json.load(r)["access_token"] +def _metadata_attr(attr: str) -> str: + """A GKE instance attribute (cluster-name, cluster-location), or "" if the metadata + server is unreachable (i.e. not on GKE — under test the env vars supply these).""" + try: + req = urllib.request.Request( + "http://metadata.google.internal/computeMetadata/v1/instance/attributes/%s" % attr, + headers={"Metadata-Flavor": "Google"}, + ) + with urllib.request.urlopen(req, timeout=5) as r: + return r.read().decode("utf-8").strip() + except Exception: # noqa: BLE001 - absence is expected off-GKE; caller fails closed + return "" + + +def cluster_selector() -> str: + """PromQL label matchers pinning a query to THIS cluster, or "" if it cannot be + determined. cluster+location uniquely identify a GKE cluster within a project.""" + cluster = CLUSTER_NAME or _metadata_attr("cluster-name") + if not cluster: + return "" + location = CLUSTER_LOCATION or _metadata_attr("cluster-location") + matchers = ['cluster="%s"' % cluster] + if location: + matchers.append('location="%s"' % location) + return "{%s}" % ", ".join(matchers) + + def kube_ctx() -> tuple[str, dict, ssl.SSLContext | None]: if KUBE_PROXY: return KUBE_PROXY.rstrip("/"), {}, None @@ -134,8 +171,18 @@ def promql(query: str) -> list: def utilisation() -> dict[str, float]: """{'namespace/pvc': fraction_used}. Empty result is an error, never 'all clear'.""" + sel = cluster_selector() + if not sel: + # A mutating guard must never query a project-wide datasource unscoped: another + # cluster's kubelet_volume_stats_* collide on namespace/pvc and could drive a + # resize here. Fail closed, same posture as the empty-result guard below. + raise RuntimeError( + "cluster scope is unknown — set CLUSTER_NAME/CLUSTER_LOCATION or run on GKE. " + "Refusing to query kubelet_volume_stats_* estate-wide: GMP is project-wide and " + "a foreign cluster's series would collide with this cluster's namespace/pvc keys." + ) rows = promql( - "kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes" + f"kubelet_volume_stats_used_bytes{sel} / kubelet_volume_stats_capacity_bytes{sel}" ) if not rows: raise RuntimeError( @@ -244,14 +291,33 @@ def main() -> int: log(f"read {len(util)} PVC utilisation series from GMP") # Estate budget is computed over the volumes this guard is allowed to grow. + # + # An unreadable enrolled PVC does NOT get silently skipped: that would undercount + # `provisioned`, and the aggregate brake below (`provisioned - cur + new > budget`) + # would then pass a grow that actually breaches estateMaxTotalGi — a control reporting + # headroom it cannot see. So a failed read makes the estate total UNKNOWN, and an + # unknown budget refuses every grow this cycle (fail closed, same posture as + # utilisation()'s empty-result guard). Growth resumes once the inventory is readable. provisioned = 0.0 + budget_known = True + budget_blind: list[str] = [] for ent in pol["pvcs"]: try: pvc = get_pvc(base, hdrs, ctx, ent["namespace"], ent["name"]) provisioned += parse_gi(pvc["spec"]["resources"]["requests"]["storage"]) - except Exception: - pass - log(f"estate provisioned across guarded PVCs: {provisioned:.0f}Gi / budget {budget_gi}Gi") + except Exception as exc: # noqa: BLE001 - any read failure blinds the total + budget_known = False + budget_blind.append(f"{ent['namespace']}/{ent['name']} ({type(exc).__name__})") + log(f"estate provisioned across guarded PVCs: {provisioned:.0f}Gi / budget {budget_gi}Gi" + + ("" if budget_known else f" — INCOMPLETE, blind on {len(budget_blind)}")) + if not budget_known: + alert("PvcGuardBudgetUnknown", "critical", + "estate storage budget cannot be computed — refusing all growth this cycle", + "One or more enrolled PVCs could not be read (" + ", ".join(budget_blind[:5]) + + "), so the estate total is a floor, not the true figure. Growing any volume " + "now could breach estateMaxTotalGi undetected, so the guard refuses every " + "expansion until the inventory is readable again. This is the aggregate brake " + "refusing to act on a number it cannot trust.", {}) grew = blind = capped = 0 @@ -326,6 +392,12 @@ def main() -> int: step = max(min_step, min(max_step, cur_gi * step_pct)) new_gi = int(min(cur_gi + step, max_gi)) + # The aggregate brake. An unknown estate total (a PVC read failed above) means we + # cannot prove this grow stays under budget, so we refuse it — never grow on a + # number known to be incomplete. PvcGuardBudgetUnknown was already raised once. + if not budget_known: + log(f" budget unknown (incomplete inventory) — refusing to grow {key}") + continue if provisioned - cur_gi + new_gi > budget_gi: alert("PvcGuardBudgetExhausted", "critical", f"{key} needs growth but the estate storage budget is exhausted", diff --git a/infra/k8s/pvc-capacity-guard/tests/test_guard_fail_closed.py b/infra/k8s/pvc-capacity-guard/tests/test_guard_fail_closed.py new file mode 100644 index 000000000..eef081bae --- /dev/null +++ b/infra/k8s/pvc-capacity-guard/tests/test_guard_fail_closed.py @@ -0,0 +1,117 @@ +"""The capacity guard fails CLOSED on an incomplete estate inventory and never queries +a project-wide datasource unscoped. + +Two adversarial-review findings, both about a control reporting headroom it cannot see: + + 1. The estate-budget accumulator used to `except Exception: pass`, so a PVC that could + not be read was silently counted as 0Gi. That undercounts `provisioned`, and the + aggregate brake (`provisioned - cur + new > budget`) would then pass a grow that + actually breaches estateMaxTotalGi. The fix: an unreadable PVC makes the total + UNKNOWN, and an unknown budget refuses every grow this cycle. + 2. The utilisation query was unscoped against Google Managed Prometheus (project-wide), + so a second cluster's kubelet_volume_stats_* collide on namespace/pvc — a mutating + guard reading another cluster's fill level. The fix pins cluster (+location). + +Local-only (Actions are spend-capped). Requires: python3 (stdlib only) + pytest. +""" +from __future__ import annotations + +import importlib.util +import urllib.error +from pathlib import Path + +import pytest + +_GUARD = Path(__file__).resolve().parents[1] / "base" / "guard.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("pvc_guard_under_test", _GUARD) + assert spec and spec.loader, f"cannot import {_GUARD}" + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _pvc(storage: str): + return { + "spec": {"resources": {"requests": {"storage": storage}}, "storageClassName": "standard"}, + "metadata": {"annotations": {}}, + } + + +def test_a_read_error_near_budget_refuses_growth_instead_of_undercounting(monkeypatch): + """THE regression. Two enrolled PVCs; 'big' is unreadable so the estate total is a + floor, not the truth. 'small' is over threshold and WOULD grow. Pre-fix the failed + read was swallowed, `provisioned` undercounted, and the brake let 'small' grow (it + called grow_pvc). Post-fix the unknown budget refuses the grow and raises + PvcGuardBudgetUnknown.""" + g = _load() + policy = { + "estateMaxTotalGi": 300, + "defaults": {"thresholdPct": 75, "stepPct": 50, "minStepGi": 5, + "maxStepGi": 50, "cooldownMinutes": 30}, + "pvcs": [ + {"namespace": "ns", "name": "big", "maxGi": 500}, # unreadable -> total unknown + {"namespace": "ns", "name": "small", "maxGi": 500}, # 90% full -> would grow + ], + } + monkeypatch.setattr(g, "load_policy", lambda: policy) + monkeypatch.setattr(g, "kube_ctx", lambda: ("http://x", {}, None)) + monkeypatch.setattr(g, "utilisation", lambda: {"ns/small": 0.90}) + monkeypatch.setattr(g, "sc_expandable", lambda *a, **k: True) + monkeypatch.setattr(g, "DRY_RUN", False) + + def fake_get_pvc(base, hdrs, ctx, ns, name): + if name == "big": + raise urllib.error.HTTPError("http://x/big", 503, "Service Unavailable", {}, None) + return _pvc("200Gi") + monkeypatch.setattr(g, "get_pvc", fake_get_pvc) + + grows: list = [] + monkeypatch.setattr(g, "grow_pvc", lambda *a, **k: grows.append(a)) + alerts: list[str] = [] + monkeypatch.setattr(g, "alert", lambda name, *a, **k: alerts.append(name)) + + g.main() + + assert grows == [], ( + "budget was UNKNOWN (an enrolled PVC could not be read) but the guard grew a volume " + "anyway — the aggregate brake acted on an undercounted total") + assert "PvcGuardBudgetUnknown" in alerts, "the blind-budget refusal must be alerted, not silent" + assert "PvcGuardExpanded" not in alerts, "nothing should have been grown this cycle" + + +def test_utilisation_query_is_pinned_to_this_cluster(monkeypatch): + """The mutating guard must scope its project-wide GMP query to this cluster. Pre-fix + the query was bare `kubelet_volume_stats_used_bytes / ..._capacity_bytes`; post-fix + both operands carry cluster (+location) matchers.""" + g = _load() + g.CLUSTER_NAME = "prophet-gke" + g.CLUSTER_LOCATION = "us-central1" + captured: dict[str, str] = {} + + def fake_promql(q): + captured["q"] = q + return [{"metric": {"namespace": "ns", "persistentvolumeclaim": "p"}, "value": [0, "0.5"]}] + monkeypatch.setattr(g, "promql", fake_promql) + + g.utilisation() + + q = captured["q"] + assert q.count('cluster="prophet-gke"') == 2, f"both operands must be cluster-scoped: {q}" + assert 'location="us-central1"' in q, f"location matcher missing: {q}" + + +def test_unknown_cluster_scope_fails_closed(monkeypatch): + """If the cluster cannot be determined at all, refuse to query rather than sweep the + whole project — an unscoped mutating guard is the collision hazard itself.""" + g = _load() + g.CLUSTER_NAME = "" + g.CLUSTER_LOCATION = "" + monkeypatch.setattr(g, "_metadata_attr", lambda attr: "") # not on GKE + called = {"promql": False} + monkeypatch.setattr(g, "promql", lambda q: called.__setitem__("promql", True) or []) + with pytest.raises(RuntimeError, match="cluster scope is unknown"): + g.utilisation() + assert called["promql"] is False, "must refuse BEFORE issuing an unscoped query"