diff --git a/deploy/serving/mesh-xl-a100.yaml b/deploy/serving/mesh-xl-a100.yaml index 2a9b48314..616dfbf27 100644 --- a/deploy/serving/mesh-xl-a100.yaml +++ b/deploy/serving/mesh-xl-a100.yaml @@ -15,6 +15,19 @@ kind: PersistentVolumeClaim metadata: name: mesh-xl-cache namespace: serving + annotations: + # `kubectl describe pvc mesh-xl-cache` shows `Used By: `, which reads + # as an abandoned 120Gi volume to anyone auditing storage. It is not. + # Unmounted IS the steady state: mesh-vllm-xl runs at replicas: 0 for $0 + # idle, and this volume exists precisely so that warming back up re-reads + # cached weights (~2-3 min) instead of re-downloading them (~15 min). + # Recorded here, on the object itself, so the question does not have to be + # re-answered from the manifest every time someone audits PVCs. + socioprophet.io/intentionally-unmounted: >- + Model-weight cache for the scale-to-zero mesh-vllm-xl seat. Unmounted + while that Deployment is at replicas=0, which is its default. Do not + reclaim without also accepting ~15min cold-start on the XL seat. + socioprophet.io/idle-cost: "~120Gi pd-balanced (~USD 13/mo) while unmounted" spec: accessModes: [ReadWriteOnce] storageClassName: standard-rwo diff --git a/infra/datastores/clickhouse/cell/0001_personal_intelligence_cell_analytics.sql b/infra/datastores/clickhouse/cell/0001_personal_intelligence_cell_analytics.sql index 3c6ec823f..e7c3e0302 100644 --- a/infra/datastores/clickhouse/cell/0001_personal_intelligence_cell_analytics.sql +++ b/infra/datastores/clickhouse/cell/0001_personal_intelligence_cell_analytics.sql @@ -17,7 +17,8 @@ CREATE TABLE IF NOT EXISTS cell_signal_scores ) ENGINE = MergeTree PARTITION BY toYYYYMM(observed_at) -ORDER BY (cell_id, watch_id, observed_at, signal_id); +ORDER BY (cell_id, watch_id, observed_at, signal_id) +TTL toDateTime(observed_at) + INTERVAL 12 MONTH DELETE; CREATE TABLE IF NOT EXISTS cell_source_quality_facts ( @@ -34,7 +35,8 @@ CREATE TABLE IF NOT EXISTS cell_source_quality_facts ) ENGINE = SummingMergeTree PARTITION BY toYYYYMM(event_at) -ORDER BY (cell_id, source_id, event_at); +ORDER BY (cell_id, source_id, event_at) +TTL toDateTime(event_at) + INTERVAL 24 MONTH DELETE; CREATE TABLE IF NOT EXISTS cell_reputation_deltas ( @@ -50,7 +52,8 @@ CREATE TABLE IF NOT EXISTS cell_reputation_deltas ) ENGINE = MergeTree PARTITION BY toYYYYMM(event_at) -ORDER BY (cell_id, subject_kind, subject_ref, event_at); +ORDER BY (cell_id, subject_kind, subject_ref, event_at) +TTL toDateTime(event_at) + INTERVAL 24 MONTH DELETE; CREATE TABLE IF NOT EXISTS cell_feedback_outcomes ( @@ -64,7 +67,8 @@ CREATE TABLE IF NOT EXISTS cell_feedback_outcomes ) ENGINE = MergeTree PARTITION BY toYYYYMM(event_at) -ORDER BY (cell_id, watch_id, action, event_at); +ORDER BY (cell_id, watch_id, action, event_at) +TTL toDateTime(event_at) + INTERVAL 24 MONTH DELETE; CREATE TABLE IF NOT EXISTS cell_watch_pattern_metrics ( @@ -80,7 +84,8 @@ CREATE TABLE IF NOT EXISTS cell_watch_pattern_metrics ) ENGINE = SummingMergeTree PARTITION BY toYYYYMM(event_at) -ORDER BY (cell_id, watch_id, pattern_id, event_at); +ORDER BY (cell_id, watch_id, pattern_id, event_at) +TTL toDateTime(event_at) + INTERVAL 24 MONTH DELETE; CREATE TABLE IF NOT EXISTS cell_notification_metrics ( @@ -94,7 +99,8 @@ CREATE TABLE IF NOT EXISTS cell_notification_metrics ) ENGINE = SummingMergeTree PARTITION BY toYYYYMM(event_at) -ORDER BY (cell_id, feed_kind, event_at); +ORDER BY (cell_id, feed_kind, event_at) +TTL toDateTime(event_at) + INTERVAL 12 MONTH DELETE; CREATE TABLE IF NOT EXISTS cell_social_environment_snapshots ( @@ -109,4 +115,5 @@ CREATE TABLE IF NOT EXISTS cell_social_environment_snapshots ) ENGINE = MergeTree PARTITION BY toYYYYMM(snapshot_at) -ORDER BY (cell_id, snapshot_at); +ORDER BY (cell_id, snapshot_at) +TTL toDateTime(snapshot_at) + INTERVAL 12 MONTH DELETE; diff --git a/infra/k8s/mesh-qdrant/base/backup-cronjob.yaml b/infra/k8s/mesh-qdrant/base/backup-cronjob.yaml index 7ccd5c9f1..5ca534bc1 100644 --- a/infra/k8s/mesh-qdrant/base/backup-cronjob.yaml +++ b/infra/k8s/mesh-qdrant/base/backup-cronjob.yaml @@ -20,17 +20,93 @@ spec: containers: - name: qdrant-snapshot image: curlimages/curl:8.6.0 - args: + env: + # Service is `mesh-qdrant`, not `qdrant` — the old value did not + # resolve in this namespace. memoryd already had this right + # (infra/k8s/memoryd/base/deployment.yaml: QDRANT_URL). + - name: QDRANT_URL + value: http://mesh-qdrant:6333 + # Keep 7 snapshots per collection. Snapshots are full copies and + # they live on the same volume they snapshot, so retention is not + # optional here — without it the backup takes out the thing it + # is backing up. + - name: RETAIN + value: "7" + # `command:`, not `args:`. curlimages/curl sets ENTRYPOINT ["curl"], + # so `args` left the entrypoint intact and the container ran + # `curl /bin/sh -c "