From 72bd127948db2029abc940caee523064c57d62df Mon Sep 17 00:00:00 2001 From: thatbagu Date: Sat, 21 Nov 2026 16:47:41 +0500 Subject: [PATCH] fix(backups): keep every backup of the last week, so as-of restores find them restic's forget kept only the newest backup of each day. A --new-cluster rebuild backs up, then the new cluster backs up its fresh volumes the same day: that backup's forget deleted the rebuild's, and restore-all --as-of silently restored an older snapshot (on a live cluster, Pi-hole and WireGuard came back as they were a day earlier, freshly installed). The default per-app retention now also keeps everything within 7 days. vm-app-backups gets the scenario: two backups the same day, a restore as of the first must return the first (without the fix it returns an older one). Co-Authored-By: Claude Opus 5.5 --- docs/backups.md | 2 +- system/k8s/vars.nix | 5 ++++- tests/vm/app-backups.nix | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/backups.md b/docs/backups.md index d0e62ca..8b26229 100644 --- a/docs/backups.md +++ b/docs/backups.md @@ -134,7 +134,7 @@ backups = { apps = { enable = true; schedule = "15 2 * * *"; # UTC - retain = { daily = 7; weekly = 4; monthly = 6; }; + retain = { within = "7d"; daily = 7; weekly = 4; monthly = 6; }; volumes = [ "myns/my-claim" ]; # optional: volumes of apps that aren't webApps }; }; diff --git a/system/k8s/vars.nix b/system/k8s/vars.nix index f4ce60f..4e8f53c 100644 --- a/system/k8s/vars.nix +++ b/system/k8s/vars.nix @@ -228,7 +228,10 @@ rec { apps = { enable = false; schedule = "15 2 * * *"; - retain = { daily = 7; weekly = 4; monthly = 6; }; + # within: every backup of the last 7 days too, not just the newest + # of each day. A restore "as of" an earlier one the same day (a + # rebuild's, before the new cluster's first) needs it still there. + retain = { within = "7d"; daily = 7; weekly = 4; monthly = 6; }; volumes = [ ]; # more volumes than webApp's: [ "/" ] }; }; diff --git a/tests/vm/app-backups.nix b/tests/vm/app-backups.nix index 0bd069f..c99790b 100644 --- a/tests/vm/app-backups.nix +++ b/tests/vm/app-backups.nix @@ -91,6 +91,22 @@ testLib.mkVmTest { succeed = "[ \"$(${k} -n notes get replicationsource notes-data -o jsonpath='{.status.latestMoverStatus.result}')\" = Successful ] && [ -n \"$(${k} -n notes get replicationsource notes-data -o jsonpath='{.status.restic.lastUnlocked}')\" ]"; } ]; } + { + # A rebuild backs up, then the new cluster backs up its fresh volumes + # the same day: restoring "as of" the first must still find it + # (restic's forget keeps only the newest per day without retain.within). + name = "a restore as of an earlier backup the same day"; + steps = [ + { succeed = "${notes} sh -c 'echo first > /data/hello'"; } + { succeed = "nixlab-backup-now notes/notes-data && date -u +%Y-%m-%dT%H:%M:%SZ > /tmp/as-of"; timeout = 900; } + { succeed = "sleep 2; ${notes} sh -c 'echo second > /data/hello'"; } + { succeed = "nixlab-backup-now notes/notes-data"; timeout = 900; } + { succeed = "nixlab-restore notes/notes-data --as-of $(cat /tmp/as-of) --yes"; timeout = 900; } + { waitUntil = "${k} -n notes rollout status deploy/notes --timeout=10s"; timeout = 300; } + { desc = "the first one's contents"; + succeed = "c=$(${notes} cat /data/hello); echo \"got: $c\" >&2; [ \"$c\" = first ]"; } + ]; + } ]; onFailure = [