Skip to content

storage: three backup jobs that delete nothing, two of which never ran #1114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mdheller wants to merge 1 commit into
base: main
Choose a base branch
from
Open

storage: three backup jobs that delete nothing, two of which never ran #1114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions deploy/serving/mesh-xl-a100.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@ kind: PersistentVolumeClaim
metadata:
name: mesh-xl-cache
namespace: serving
annotations:
# `kubectl describe pvc mesh-xl-cache` shows `Used By: <none>`, which reads
# as an abandoned 120Gi volume to anyone auditing storage. It is not.
# Unmounted IS the steady state: mesh-vllm-xl runs at replicas: 0 for $0
# idle, and this volume exists precisely so that warming back up re-reads
# cached weights (~2-3 min) instead of re-downloading them (~15 min).
# Recorded here, on the object itself, so the question does not have to be
# re-answered from the manifest every time someone audits PVCs.
socioprophet.io/intentionally-unmounted: >-
Model-weight cache for the scale-to-zero mesh-vllm-xl seat. Unmounted
while that Deployment is at replicas=0, which is its default. Do not
reclaim without also accepting ~15min cold-start on the XL seat.
socioprophet.io/idle-cost: "~120Gi pd-balanced (~USD 13/mo) while unmounted"
spec:
accessModes: [ReadWriteOnce]
storageClassName: standard-rwo
Expand Down
21 changes: 14 additions & 7 deletions infra/datastores/clickhouse/cell/0001_personal_intelligence_cell_analytics.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ CREATE TABLE IF NOT EXISTS cell_signal_scores
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(observed_at)
ORDER BY (cell_id, watch_id, observed_at, signal_id);
ORDER BY (cell_id, watch_id, observed_at, signal_id)
TTL toDateTime(observed_at) + INTERVAL 12 MONTH DELETE;

CREATE TABLE IF NOT EXISTS cell_source_quality_facts
(
Expand All @@ -34,7 +35,8 @@ CREATE TABLE IF NOT EXISTS cell_source_quality_facts
)
ENGINE = SummingMergeTree
PARTITION BY toYYYYMM(event_at)
ORDER BY (cell_id, source_id, event_at);
ORDER BY (cell_id, source_id, event_at)
TTL toDateTime(event_at) + INTERVAL 24 MONTH DELETE;

CREATE TABLE IF NOT EXISTS cell_reputation_deltas
(
Expand All @@ -50,7 +52,8 @@ CREATE TABLE IF NOT EXISTS cell_reputation_deltas
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(event_at)
ORDER BY (cell_id, subject_kind, subject_ref, event_at);
ORDER BY (cell_id, subject_kind, subject_ref, event_at)
TTL toDateTime(event_at) + INTERVAL 24 MONTH DELETE;

CREATE TABLE IF NOT EXISTS cell_feedback_outcomes
(
Expand All @@ -64,7 +67,8 @@ CREATE TABLE IF NOT EXISTS cell_feedback_outcomes
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(event_at)
ORDER BY (cell_id, watch_id, action, event_at);
ORDER BY (cell_id, watch_id, action, event_at)
TTL toDateTime(event_at) + INTERVAL 24 MONTH DELETE;

CREATE TABLE IF NOT EXISTS cell_watch_pattern_metrics
(
Expand All @@ -80,7 +84,8 @@ CREATE TABLE IF NOT EXISTS cell_watch_pattern_metrics
)
ENGINE = SummingMergeTree
PARTITION BY toYYYYMM(event_at)
ORDER BY (cell_id, watch_id, pattern_id, event_at);
ORDER BY (cell_id, watch_id, pattern_id, event_at)
TTL toDateTime(event_at) + INTERVAL 24 MONTH DELETE;

CREATE TABLE IF NOT EXISTS cell_notification_metrics
(
Expand All @@ -94,7 +99,8 @@ CREATE TABLE IF NOT EXISTS cell_notification_metrics
)
ENGINE = SummingMergeTree
PARTITION BY toYYYYMM(event_at)
ORDER BY (cell_id, feed_kind, event_at);
ORDER BY (cell_id, feed_kind, event_at)
TTL toDateTime(event_at) + INTERVAL 12 MONTH DELETE;

CREATE TABLE IF NOT EXISTS cell_social_environment_snapshots
(
Expand All @@ -109,4 +115,5 @@ CREATE TABLE IF NOT EXISTS cell_social_environment_snapshots
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(snapshot_at)
ORDER BY (cell_id, snapshot_at);
ORDER BY (cell_id, snapshot_at)
TTL toDateTime(snapshot_at) + INTERVAL 12 MONTH DELETE;
90 changes: 83 additions & 7 deletions infra/k8s/mesh-qdrant/base/backup-cronjob.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,93 @@ spec:
containers:
- name: qdrant-snapshot
image: curlimages/curl:8.6.0
args:
env:
# Service is `mesh-qdrant`, not `qdrant` — the old value did not
# resolve in this namespace. memoryd already had this right
# (infra/k8s/memoryd/base/deployment.yaml: QDRANT_URL).
- name: QDRANT_URL
value: http://mesh-qdrant:6333
# Keep 7 snapshots per collection. Snapshots are full copies and
# they live on the same volume they snapshot, so retention is not
# optional here — without it the backup takes out the thing it
# is backing up.
- name: RETAIN
value: "7"
# `command:`, not `args:`. curlimages/curl sets ENTRYPOINT ["curl"],
# so `args` left the entrypoint intact and the container ran
# `curl /bin/sh -c "<script>"` — curl treating /bin/sh as a URL.
# The shell never executed. Combined with the wrong hostname and a
# `jq` that is not in this image, this CronJob has never produced a
# single snapshot since it was authored, and said nothing.
command:
- /bin/sh
- -c
- |
set -e
COLLECTIONS=$(curl -sf http://qdrant:6333/collections | jq -r '.result.collections[].name')
for col in $COLLECTIONS; do
echo "Snapshotting collection: $col"
curl -sf -X POST "http://qdrant:6333/collections/${col}/snapshots"
set -eu
log() { echo "$@" >&2; }
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
MADE=0; PRUNED=0; FAILED=0

# No jq in curlimages/curl. Extract names with grep, which
# busybox does have. Deliberately not adding a dependency to a
# backup job — fewer moving parts is the whole point.
# Whitespace-tolerant: a pretty-printed `"name" : "x"` must
# parse the same as compact `"name":"x"`. The first version of
# this was anchored to the compact form and silently extracted
# nothing against a spaced response — reporting a clean
# "0 collections" backup. Caught by running it against a mock.
names() { grep -oE '"name"[[:space:]]*:[[:space:]]*"[^"]*"' | cut -d'"' -f4; }

COLLECTIONS=$(curl -sf --max-time 30 "${QDRANT_URL}/collections" | names || true)
if [ -z "${COLLECTIONS}" ]; then
Comment on lines +58 to +61
# "zero collections" and "qdrant is unreachable" look
# identical unless checked separately. Refuse rather than
# report a successful no-op backup.
if ! curl -sf --max-time 10 "${QDRANT_URL}/readyz" >/dev/null 2>&1; then
log "REFUSED: qdrant unreachable at ${QDRANT_URL}"
printf '{"version":"0.1","receipt_id":"evr-qdrant-snapshot-unreachable-%s","created_at":"%s","service_ref":"infra/k8s/mesh-qdrant","action":"qdrant-snapshot","status":"denied","subject_ref":"pvc/qdrant-storage","hash":"","hash_algo":"sha256","metrics":{"collections":0}}\n' \
"$(date -u +%Y%m%dT%H%M%SZ)" "${NOW}"
exit 1
fi
log "qdrant reachable but reports zero collections — nothing to snapshot"
fi
Comment on lines +65 to +72

for col in ${COLLECTIONS}; do
log "snapshotting collection: ${col}"
if curl -sf --max-time 300 -X POST "${QDRANT_URL}/collections/${col}/snapshots" >/dev/null; then
MADE=$((MADE + 1))
else
log "FAILED: snapshot of ${col}"
FAILED=$((FAILED + 1))
continue
fi

# --- retention ------------------------------------------
# The old job only ever POSTed. It never called
# DELETE /collections/{c}/snapshots/{s}, so snapshots would
# have accumulated forever had it ever run. Snapshot names
# carry a timestamp and sort chronologically.
SNAPS=$(curl -sf --max-time 30 "${QDRANT_URL}/collections/${col}/snapshots" | names || true)
TOTAL=$(printf '%s\n' "${SNAPS}" | grep -c . || true)
if [ "${TOTAL}" -gt "${RETAIN}" ]; then
Comment on lines +89 to +91
DROP=$((TOTAL - RETAIN))
printf '%s\n' "${SNAPS}" | sort | head -n "${DROP}" | while read -r s; do
[ -z "${s}" ] && continue
log "retention: deleting ${col}/${s}"
curl -sf --max-time 60 -X DELETE "${QDRANT_URL}/collections/${col}/snapshots/${s}" >/dev/null \
|| log "WARN: could not delete ${col}/${s}"
done
PRUNED=$((PRUNED + DROP))
fi
done
echo "All snapshots created"

STATUS=succeeded
[ "${FAILED}" -gt 0 ] && STATUS=partial
printf '{"version":"0.1","receipt_id":"evr-qdrant-snapshot-%s","created_at":"%s","service_ref":"infra/k8s/mesh-qdrant","action":"qdrant-snapshot","status":"%s","subject_ref":"pvc/qdrant-storage","hash":"","hash_algo":"sha256","metrics":{"snapshots_made":%s,"pruned":%s,"failed":%s,"retain":%s}}\n' \
"$(date -u +%Y%m%dT%H%M%SZ)" "${NOW}" "${STATUS}" "${MADE}" "${PRUNED}" "${FAILED}" "${RETAIN}"
log "done: made=${MADE} pruned=${PRUNED} failed=${FAILED}"
[ "${FAILED}" -gt 0 ] && exit 1
exit 0
resources:
requests:
cpu: 10m
Expand Down
13 changes: 13 additions & 0 deletions infra/k8s/mesh-qdrant/base/statefulset.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,19 @@ spec:
containers:
- name: qdrant
image: qdrant/qdrant:v1.9.4
env:
# Without this, qdrant defaults snapshots_path to ./snapshots
# relative to /qdrant — i.e. /qdrant/snapshots, which is NOT on the
# PVC. Snapshots then accumulate on the pod's ephemeral writable
# layer: they eat node ephemeral storage until eviction and vanish
# on every restart. A backup that does not survive a restart is not
# a backup, and nothing said so.
#
# Putting them under the mounted volume makes them durable AND makes
# them count against the same 20Gi the guard watches, so unbounded
# snapshot growth becomes visible instead of invisible.
- name: QDRANT__STORAGE__SNAPSHOTS_PATH
value: /qdrant/storage/snapshots
ports:
- name: http
containerPort: 6333
Expand Down
106 changes: 103 additions & 3 deletions infra/k8s/workspace-mail/base/backup-cronjob.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,114 @@ spec:
# exact digest, uid 1000, caps dropped, seccomp RuntimeDefault): the pull
# succeeds and the one-liner produces a real, listable .tar.gz.
image: registry.socioprophet.ai/library/alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b
env:
# Keep 7 dailies. The source is a 10Gi mail volume against a 20Gi
# target, so 7 compressed fulls is the most that fits with room to
# write the next one before pruning.
- name: RETAIN
value: "7"
# Hard ceiling on what backups may occupy, independent of RETAIN.
# Whichever binds first wins.
- name: MAX_TOTAL_MB
value: "14000"
# Refuse to start a backup that cannot fit. See below.
- name: MIN_FREE_MB
value: "3000"
command:
- /bin/sh
- -c
- |
set -e
# Retention, verification and a receipt.
#
# This job previously wrote a new full tarball every night and
# deleted NOTHING. It never actually ran (the image did not
# exist), which is the only reason the 20Gi volume is not
# already full — the retention bug was latent, not fixed. It is
# fixed here BEFORE the job is allowed to schedule.
#
# Order matters: prune first, then check free space, then write,
# then verify, then receipt. Writing first and pruning after
# would need headroom for N+1 backups, which is exactly the
# margin that does not exist.
set -eu
STAMP=$(date -u +%Y%m%dT%H%M%SZ)
tar czf /backup/mail-${STAMP}.tar.gz /maildata
echo "Mail backup complete: mail-${STAMP}.tar.gz"
OUT="/backup/mail-${STAMP}.tar.gz"
RC=0

log() { echo "$@" >&2; }

# --- 1. prune by count -------------------------------------
# Oldest first; names are lexically sortable by construction.
COUNT=$(ls -1 /backup/mail-*.tar.gz 2>/dev/null | wc -l | tr -d ' ')
log "retention: ${COUNT} existing backup(s), keeping ${RETAIN}"
if [ "${COUNT}" -ge "${RETAIN}" ]; then
DROP=$((COUNT - RETAIN + 1))
ls -1 /backup/mail-*.tar.gz 2>/dev/null | sort | head -n "${DROP}" | \
while read -r f; do
log "retention: pruning $(basename "$f")"
rm -f "$f"
done
fi

# --- 2. prune by total size --------------------------------
while :; do
USED=$(du -sm /backup 2>/dev/null | awk '{print $1}')
[ -z "${USED}" ] && USED=0
[ "${USED}" -le "${MAX_TOTAL_MB}" ] && break
OLDEST=$(ls -1 /backup/mail-*.tar.gz 2>/dev/null | sort | head -n1)
[ -z "${OLDEST}" ] && break
log "retention: ${USED}MB over ${MAX_TOTAL_MB}MB budget, pruning $(basename "${OLDEST}")"
rm -f "${OLDEST}"
done

# --- 3. refuse rather than fill the disk -------------------
# A backup that fills the volume takes out the next backup AND
# anything else on it. Refusing is the safe failure, and it is
# loud: exit 1 -> Job failure -> KubeJobFailed (a live alert).
FREE=$(df -Pm /backup | awk 'NR==2 {print $4}')
log "preflight: ${FREE}MB free, need >= ${MIN_FREE_MB}MB"
if [ "${FREE}" -lt "${MIN_FREE_MB}" ]; then
log "REFUSED: only ${FREE}MB free on /backup, below MIN_FREE_MB=${MIN_FREE_MB}"
printf '{"version":"0.1","receipt_id":"evr-mail-backup-refused-%s","created_at":"%s","service_ref":"infra/k8s/workspace-mail","action":"mail-backup","status":"denied","subject_ref":"pvc/workspace-mail-backup","hash":"","hash_algo":"sha256","metrics":{"free_mb":%s,"min_free_mb":%s,"retained":%s}}\n' \
"${STAMP}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${FREE}" "${MIN_FREE_MB}" "${COUNT}"
exit 1
fi
Comment on lines +105 to +112

# --- 4. write ----------------------------------------------
if ! tar czf "${OUT}" /maildata; then
log "FAILED: tar returned non-zero"
rm -f "${OUT}"
RC=1
fi

# --- 5. verify ---------------------------------------------
# An unreadable archive is not a backup. Listing it is cheap
# and catches a truncated write, which is the failure mode a
# nearly-full volume actually produces.
if [ "${RC}" -eq 0 ] && ! tar tzf "${OUT}" >/dev/null 2>&1; then
log "FAILED: ${OUT} is not a readable gzip archive — removing"
rm -f "${OUT}"
RC=1
fi

# --- 6. receipt --------------------------------------------
# One EvidenceReceipt line to stdout -> fluentbit-gke -> Cloud
# Logging. `hash` is the archive digest, so the receipt proves
# WHICH bytes were produced, not merely that something ran.
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
KEPT=$(ls -1 /backup/mail-*.tar.gz 2>/dev/null | wc -l | tr -d ' ')
USED=$(du -sm /backup 2>/dev/null | awk '{print $1}')
if [ "${RC}" -eq 0 ]; then
SIZE=$(du -m "${OUT}" | awk '{print $1}')
SHA=$(sha256sum "${OUT}" | awk '{print $1}')
printf '{"version":"0.1","receipt_id":"evr-mail-backup-%s","created_at":"%s","service_ref":"infra/k8s/workspace-mail","action":"mail-backup","status":"succeeded","subject_ref":"pvc/workspace-mail-backup","hash":"%s","hash_algo":"sha256","output_refs":["%s"],"metrics":{"size_mb":%s,"retained":%s,"used_mb":%s,"retain_policy":%s,"max_total_mb":%s}}\n' \
"${STAMP}" "${NOW}" "${SHA}" "${OUT}" "${SIZE}" "${KEPT}" "${USED}" "${RETAIN}" "${MAX_TOTAL_MB}"
log "Mail backup complete: $(basename "${OUT}") ${SIZE}MB; ${KEPT} retained, ${USED}MB used"
else
printf '{"version":"0.1","receipt_id":"evr-mail-backup-failed-%s","created_at":"%s","service_ref":"infra/k8s/workspace-mail","action":"mail-backup","status":"failed","subject_ref":"pvc/workspace-mail-backup","hash":"","hash_algo":"sha256","metrics":{"retained":%s,"used_mb":%s}}\n' \
"${STAMP}" "${NOW}" "${KEPT}" "${USED}"
fi
exit "${RC}"
volumeMounts:
- name: mail-data
mountPath: /maildata
Expand Down
Loading