From 12cd5723e794ac8c67ce0daca79552c2f9812aed Mon Sep 17 00:00:00 2001 From: thatbagu Date: Sat, 21 Nov 2026 15:48:28 +0500 Subject: [PATCH 1/5] test: a fresh cluster bootstrapping in a VM, and upgrading from the last release - vm-bootstrap: Argo CD syncs every layer (MetalLB, the gateway and edge proxy, cert-manager with the internal CA, Longhorn, Pi-hole, the VPN, a web app) from the rendered branch, which the VM serves itself over git's HTTP protocol; passes only when every Application is Synced and Healthy, the app answers through the gateway, and Longhorn's first sync went through by itself. The three bootstrap bugs found on real hardware (AppProject race, gateway health, the sync operation's options) are the kind it catches. - vm-upgrade: the previous release's render applied for real, then the current one over it (server-side dry run, CRDs for real): catches fields that can't change on existing objects and dropped CRD versions. Checked by sabotage: a changed Deployment selector is rejected ("field is immutable"). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 +- CONTRIBUTING.md | 4 + tests/default.nix | 29 ++++++- tests/vm/bootstrap.nix | 109 +++++++++++++++++++++++++ tests/vm/images.json | 170 +++++++++++++++++++++++++++++++++++++++ tests/vm/upgrade.nix | 95 ++++++++++++++++++++++ tools/update.nix | 6 ++ 7 files changed, 414 insertions(+), 3 deletions(-) create mode 100644 tests/vm/bootstrap.nix create mode 100644 tests/vm/upgrade.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35e40c6..195ff0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - group: [quick, quick-eval, quick-render, hosts, vm-cluster, vm-minimal, vm-sso, vm-gateway, vm-ha, vm-longhorn, vm-install, vm-app-backups, vm-single, vm-install-encrypted, vm-nas, vm-power, vm-tailscale, vm-monitoring, vm-monitoring-full] + group: [quick, quick-eval, quick-render, hosts, vm-cluster, vm-minimal, vm-sso, vm-gateway, vm-bootstrap, vm-upgrade, vm-ha, vm-longhorn, vm-install, vm-app-backups, vm-single, vm-install-encrypted, vm-nas, vm-power, vm-tailscale, vm-monitoring, vm-monitoring-full] name: checks (${{ matrix.group }}) steps: - uses: actions/checkout@v4 @@ -61,7 +61,7 @@ jobs: # everything at once: together they exhaust a runner's memory. filter='n: let g = builtins.getEnv "GROUP"; - vmsAndHosts = [ "host-master" "host-worker" "vm-cluster" "vm-minimal" "vm-sso" "vm-gateway" "vm-ha" "vm-longhorn" "vm-install" "vm-app-backups" "vm-single" "vm-install-encrypted" "vm-nas" "vm-power" "vm-tailscale" "vm-monitoring" "vm-monitoring-full"]; + vmsAndHosts = [ "host-master" "host-worker" "vm-cluster" "vm-minimal" "vm-sso" "vm-gateway" "vm-bootstrap" "vm-upgrade" "vm-ha" "vm-longhorn" "vm-install" "vm-app-backups" "vm-single" "vm-install-encrypted" "vm-nas" "vm-power" "vm-tailscale" "vm-monitoring" "vm-monitoring-full"]; prefix = p: builtins.substring 0 (builtins.stringLength p) n == p; in if g == "quick" then !(builtins.elem n vmsAndHosts) && !(prefix "eval-") && !(prefix "render-") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74d67d8..eb53aef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,6 +93,10 @@ chore: bump pihole to 2025.11.1 Subject line: imperative mood, no period, 72 characters max. Body: explain the *why*, not the *what*, when it's not obvious. +## Releasing + +After tagging a release, point the upgrade test at it, so the next release is checked against this one: in `tests/vm/upgrade.nix`, set `previousVersion` to the new tag and `sha256` to the output of `nix-prefetch-url --unpack https://github.com/thatbagu/nixlab/archive/refs/tags/.tar.gz`. + ## License MIT. By contributing you agree your changes are released under the same license. diff --git a/tests/default.nix b/tests/default.nix index e2a5d3e..60441fd 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -53,7 +53,8 @@ let # overrides merge into baseVars; replace swaps whole top-level settings # (e.g. nodes, to drop one). - mkTestLab = { overrides ? { }, replace ? { }, extraAppModules ? [ ], extraNixosModules ? [ ], secretsFile ? testSecrets, extraSecrets ? [ ] }: mkNixlab { + # (testLabArgs: what it passes to mkNixlab, for another version's mkNixlab.) + testLabArgs = { overrides ? { }, replace ? { }, extraAppModules ? [ ], extraNixosModules ? [ ], secretsFile ? testSecrets, extraSecrets ? [ ] }: { vars = lib.recursiveUpdate baseVars overrides // replace; inherit secretsFile; # The unit tests' secrets are fake and plaintext; the VM tests' are encrypted. @@ -61,6 +62,7 @@ let hardwareConfig = _: ../template/hosts/example/hardware-configuration.nix; inherit extraAppModules extraNixosModules extraSecrets; }; + mkTestLab = args: mkNixlab (testLabArgs args); # One machine with the whole platform (profile = "single"). singleNode = { @@ -143,6 +145,26 @@ let backups = { enable = true; target = "nfs://192.168.1.1:/export/backups"; }; }; + # The fresh-bootstrap VM test: Argo CD syncs every layer from a git + # repository the machine serves itself (the rendered branch; the test + # network numbers its one machine .1). No internet: the internal CA, no + # Nextcloud. + bootstrapVm = { + overrides = allLayersOff // { + gitops.enable = true; + metallb.enable = true; + ingress.enable = true; + edge.enable = true; + certManager.enable = true; + cloudflare.enable = false; + longhorn.enable = true; + pihole.enable = true; + vpn.enable = true; + gitRepository = "http://192.168.1.1:8000/rendered.git"; + }; + extraAppModules = [ gatewayVmApp ]; + }; + # The per-app backup VM test: k3s' local storage (VolSync reads the volume # itself), MinIO on another machine (the test network numbers machines in # name order: s3 .1, tmaster .2) as the S3 target, and an app with a volume. @@ -204,6 +226,7 @@ let longhorn-vm.overrides = longhornVmOverrides; # The gateway VM test's setup. gateway-vm = { overrides = gatewayVmOverrides; extraAppModules = [ gatewayVmApp ]; }; + bootstrap-vm = bootstrapVm; # Every layer off: k3s on NixOS and nothing else. minimal.overrides = allLayersOff; # Everything on one machine (profile = "single"). @@ -1041,6 +1064,10 @@ lib.mapAttrs' (name: lab: lib.nameValuePair "render-${name}" lab.nixidyEnv.envir # NixOS VM test: HTTPS through the gateway (Traefik, Gateway API), with a # cert-manager certificate, behind the edge proxy (tests/vm/gateway.nix). vm-gateway = import ./vm/gateway.nix { inherit pkgs mkTestLab testLib; overrides = gatewayVmOverrides; extraAppModules = [ gatewayVmApp ]; }; + # A fresh cluster bootstrapping every layer through Argo CD, from git. + vm-bootstrap = import ./vm/bootstrap.nix { inherit pkgs mkTestLab testLib; inherit (bootstrapVm) overrides extraAppModules; }; + # Upgrading from the previous release: its render, then the current one over it. + vm-upgrade = import ./vm/upgrade.nix { inherit pkgs mkTestLab testLib inputs allLayersOff; labArgs = testLabArgs { }; }; # NixOS VM test: monitoring on the real master, alerts delivered to a # stand-in webhook (tests/vm/monitoring.nix). # NixOS VM test: the UPS shuts the cluster down in order (tests/vm/power.nix). diff --git a/tests/vm/bootstrap.nix b/tests/vm/bootstrap.nix new file mode 100644 index 0000000..3d2e4c2 --- /dev/null +++ b/tests/vm/bootstrap.nix @@ -0,0 +1,109 @@ +# NixOS VM test of a fresh cluster bootstrapping by itself, the way a real +# one does: Argo CD syncs every layer (MetalLB, the gateway, cert-manager with +# the internal CA, Longhorn, Pi-hole, the VPN, a web app) from the rendered +# branch, which the machine serves itself as a git repository. Checks that, +# with nobody touching anything: +# - every Application ends up Synced and Healthy (sync waves, Longhorn's +# one-time manual sync, a gateway whose certificates come later, Secrets +# for namespaces another app creates) +# - the web app answers HTTPS through the edge proxy and the gateway +# +# The VM has no internet: images are preloaded (images.nix), and the served +# branch drops their registry digests (preloaded images keep only the tag). +{ pkgs, mkTestLab, testLib, overrides, extraAppModules }: + +let + images = (import ./images.nix { inherit pkgs; }).vm-bootstrap; + lab = mkTestLab { secretsFile = ./secrets.yaml; inherit overrides extraAppModules; }; + env = lab.nixidyEnv.environmentPackage; + k = "k3s kubectl"; + apps = "${k} -n argocd get applications -o custom-columns=A:.metadata.name,S:.status.sync.status,H:.status.health.status --no-headers"; +in +testLib.mkVmTest { + name = "nixlab-bootstrap"; + nodeName = "tmaster"; + specialArgs = lab.specialArgsFor "tmaster"; + + node = { + imports = lab.modulesFor "tmaster"; + virtualisation = { + memorySize = 8192; + cores = 6; + diskSize = 8192; + emptyDiskImages = [ 24576 ]; + fileSystems."/persist" = { device = "/dev/vdb"; fsType = "ext4"; autoFormat = true; neededForBoot = true; }; + }; + system.activationScripts = { + vmTestAgeKey.text = '' + install -D -m 0400 ${./age-key.txt} /persist/etc/sops-nix/keys.txt + ''; + setupSecrets.deps = [ "vmTestAgeKey" ]; + setupSecretsForUsers.deps = [ "vmTestAgeKey" ]; + }; + networking.firewall.allowedTCPPorts = [ 7946 ]; + networking.firewall.allowedUDPPorts = [ 7946 ]; + services.k3s.images = [ pkgs.k3s.passthru.airgap-images ] ++ images; + environment.systemPackages = [ pkgs.curl ]; + + # The rendered branch, as CI pushes it, over git's plain HTTP protocol + # (gitRepository in the lab points here). + systemd.services.rendered-repo = { + wantedBy = [ "multi-user.target" ]; + before = [ "argocd-bootstrap.service" ]; + path = with pkgs; [ git gnused python3 ]; + script = '' + rm -rf /run/rendered /srv/git; mkdir -p /run/rendered /srv/git + cp -r ${env}/. /run/rendered/ && chmod -R u+w /run/rendered + find /run/rendered -name '*.yaml' -exec sed -i -E 's/@sha256:[0-9a-f]{64}//' {} + + cd /run/rendered + git init -q -b k8s-rendered && git add -A + git -c user.email=ci@test -c user.name=ci commit -qm render + git clone -q --bare /run/rendered /srv/git/rendered.git + git -C /srv/git/rendered.git update-server-info + exec python3 -m http.server 8000 --directory /srv/git + ''; + }; + }; + + subtests = [ + { + name = "the cluster comes up and Argo CD bootstraps it"; + steps = [ + { waitForUnit = "k3s.service"; } + { waitUntil = "${k} get nodes | grep -w Ready"; } + { waitForUnit = "rendered-repo.service"; } + { waitUntil = "curl -sf http://192.168.1.1:8000/rendered.git/info/refs | grep -c k8s-rendered"; } + { waitForUnit = "argocd-bootstrap.service"; timeout = 600; } + ]; + } + { + name = "every app ends up Synced and Healthy, by itself"; + steps = [ + { desc = "all of them exist"; + waitUntil = "[ $(${apps} | wc -l) -ge ${toString (builtins.length (builtins.attrNames lab.nixidyEnv.config.applications) - 1)} ]"; timeout = 1800; } + { desc = "and all of them are Synced and Healthy"; + waitUntil = "${apps} >&2; [ -z \"$(${apps} | grep -v 'Synced *Healthy')\" ]"; timeout = 2400; } + ]; + } + { + name = "the web app answers through the edge proxy and the gateway"; + steps = [ + { waitUntil = "curl -skf --resolve whoami.yourdomain.example.com:443:192.168.1.193 https://whoami.yourdomain.example.com/ | grep -c Hostname"; timeout = 600; } + ]; + } + { + name = "Longhorn's first sync went through without help"; + steps = [ + { succeed = "[ \"$(${k} -n argocd get application longhorn -o jsonpath='{.status.operationState.phase}')\" = Succeeded ]"; } + { succeed = "${k} -n longhorn-system get nodes.longhorn.io tmaster"; } + ]; + } + ]; + + onFailure = [ + "${apps}" + "${k} -n argocd get applications -o json | ${pkgs.jq}/bin/jq -r '.items[] | select(.status.sync.status != \"Synced\" or .status.health.status != \"Healthy\") | \"\\(.metadata.name): \\(.status.operationState.phase // \"-\") \\(.status.operationState.message // \"\" | .[0:300])\"'" + "${k} get pods -A | grep -vE 'Running|Completed'" + "journalctl -u argocd-bootstrap --no-pager | tail -30" + ]; +} diff --git a/tests/vm/images.json b/tests/vm/images.json index d0eab69..86d76ee 100644 --- a/tests/vm/images.json +++ b/tests/vm/images.json @@ -31,6 +31,176 @@ "imageName": "quay.io/backube/volsync" } }, + "vm-bootstrap": { + "alpine": { + "finalImageTag": "3.22", + "hash": "sha256-tH8dnuPM5eClAzXUtr1sxf2trSLsFsuyYh97O2qLNAc=", + "imageDigest": "sha256:2e0b81d60597f449baa56dc25d355367869d2dca4cd57e866f1e2b52af3eb55f", + "imageName": "docker.io/library/alpine" + }, + "argocd": { + "finalImageTag": "v3.4.1", + "hash": "sha256-/NDpUXALQw6UMHLEzAEZAQIVW2l85HT+KpYsbwh3MCA=", + "imageDigest": "sha256:097c922fb910e6bb0278b0e26382f88de4fa5c966564111bddd334051b0e63d7", + "imageName": "quay.io/argoproj/argocd" + }, + "cert-manager-cainjector": { + "finalImageTag": "v1.20.2", + "hash": "sha256-k7KNy3kmJS9qQft0sNYC3UnhVPA9CS/nDDbNDdp4emg=", + "imageDigest": "sha256:6f5a644135887b2aa7d5cc145072fa56421560e3586ff1f184358022d490f4e1", + "imageName": "quay.io/jetstack/cert-manager-cainjector" + }, + "cert-manager-controller": { + "finalImageTag": "v1.20.2", + "hash": "sha256-lcjoakYzDhQiCb5Bu5ZLFf8exWYqhzFRjHEi3jpZpaA=", + "imageDigest": "sha256:fe0623d7d04a382c888f03343a3a2da716e0d96ad3d5d790c0ebcbcb2a4329a5", + "imageName": "quay.io/jetstack/cert-manager-controller" + }, + "cert-manager-startupapicheck": { + "finalImageTag": "v1.20.2", + "hash": "sha256-qqs3zPuv+8PkcgMAUATH15/r7Wppq+OyhAgvseeKIhM=", + "imageDigest": "sha256:4e2a69b4a0cc9627905bbeecf720f95d5153ca39cacdab923d2748e73556792b", + "imageName": "quay.io/jetstack/cert-manager-startupapicheck" + }, + "cert-manager-webhook": { + "finalImageTag": "v1.20.2", + "hash": "sha256-Tq+dBEIAV1ButQCgZ7fKfRie8B2VwnDC4GUVTDc1CQA=", + "imageDigest": "sha256:baf651128b9f05c426cbd5e60e2036bf382c99ca270f49d0757d6f7d2452f4e5", + "imageName": "quay.io/jetstack/cert-manager-webhook" + }, + "controller": { + "finalImageTag": "v0.15.3", + "hash": "sha256-j71loxJPSdOfqSGX3b5/4X7OtwhQlslAsSGnhpDlcZ8=", + "imageDigest": "sha256:6698ccc54c380913816ed1fd0758637ec87dd79da419c4ab170a2c26c158ab89", + "imageName": "quay.io/metallb/controller" + }, + "csi-attacher": { + "finalImageTag": "v4.11.0-20260428", + "hash": "sha256-+defbWikOWKmZvwOX8vAy5cbwQa6EDiJOnfOLe4fzuo=", + "imageDigest": "sha256:fe417c28a6b86f8e7e5d49fc223e22e9ab457f894d2c4a321932d136dc2c2530", + "imageName": "docker.io/longhornio/csi-attacher" + }, + "csi-node-driver-registrar": { + "finalImageTag": "v2.16.0-20260428", + "hash": "sha256-tcltO1AEnHotTxyJ+N7FbzvBS2U1DgOrikGBB8lOA2k=", + "imageDigest": "sha256:e82a8c8f800d7fbb3c1edf3f90b557768091821a44d52280093394f7918ccb68", + "imageName": "docker.io/longhornio/csi-node-driver-registrar" + }, + "csi-provisioner": { + "finalImageTag": "v5.3.0-20260428", + "hash": "sha256-nUkForHWNHisq1cMNTjzJ3wuy7dmWBHSi/H2Zaa6ewA=", + "imageDigest": "sha256:9e519a21a77c060104716e1f98222bb46ab617778a3bfcd861c87119a8256764", + "imageName": "docker.io/longhornio/csi-provisioner" + }, + "csi-resizer": { + "finalImageTag": "v2.1.0-20260428", + "hash": "sha256-BPXAG3NIacqmf3f8V4kjWZIfYw3Becdn8oCuNStGulE=", + "imageDigest": "sha256:41cb674d1154e798aa2c20f53f72ee2a5597f1369bcad5878d1708aee47f6663", + "imageName": "docker.io/longhornio/csi-resizer" + }, + "csi-snapshotter": { + "finalImageTag": "v8.5.0-20260428", + "hash": "sha256-tDLxZZUsFzerg1q0ofOD5IrdtxH2ywFFjJ3u/rG79xc=", + "imageDigest": "sha256:1975fac3890f4e08b98792881cb597502112ce0eeeaaef383e52458c96db94c5", + "imageName": "docker.io/longhornio/csi-snapshotter" + }, + "external-dns": { + "finalImageTag": "v0.21.0", + "hash": "sha256-a7Z3K/H0ExAebR8meaPtVDH2/RbqxlhFNH60WfJqoaA=", + "imageDigest": "sha256:f53faaf71cb270d1ca9dce6ea0c94bfebf1a18696263487f0fbc74b9bf2bd7ff", + "imageName": "registry.k8s.io/external-dns/external-dns" + }, + "haproxy": { + "finalImageTag": "3.0-alpine", + "hash": "sha256-FKOu0U4s+ZPAW9QFEDz2WEH7HkfM3IOMjJ8yy6A4Zng=", + "imageDigest": "sha256:1c15903c50d5510dcc01de755049e7015a6dd3f35bb2dc4582e888116cb71d8f", + "imageName": "docker.io/library/haproxy" + }, + "kubectl": { + "finalImageTag": "v1.35.10", + "hash": "sha256-KoBupv9E6TsoXG3WalK6DacansbXB3pkBSkRZ/gdAkY=", + "imageDigest": "sha256:4e1d541a3d62f74637974e4c71f3cea91b9a36ffd059ab45e355648898783837", + "imageName": "registry.k8s.io/kubectl" + }, + "livenessprobe": { + "finalImageTag": "v2.18.0-20260428", + "hash": "sha256-qZqaIl3LqLd0og1DEM5KwKh5P0lMlRZyUDIhyrFJ8L8=", + "imageDigest": "sha256:eae162f7e70fb981f90d9206f299dddaf590c0c896cfb67acceca12cef526a44", + "imageName": "docker.io/longhornio/livenessprobe" + }, + "longhorn-engine": { + "finalImageTag": "v1.11.2", + "hash": "sha256-3ceS6YUy/h0W1Ofi0ZT9lafGf3MRWr6JZ1H/MqgM8t0=", + "imageDigest": "sha256:7482e0437fbf475e1e32696fab22f47bf99b1ef8d067ffce9e34028347722628", + "imageName": "docker.io/longhornio/longhorn-engine" + }, + "longhorn-instance-manager": { + "finalImageTag": "v1.11.2", + "hash": "sha256-gElWLr5Mk6ZPkUgsIDNHvuS53WXTC9Rn2YMUKdmQydI=", + "imageDigest": "sha256:16dac125ef30bd3a375bc8ff7d10636ea0302d22d208c0cfb1be37ebb93ca30b", + "imageName": "docker.io/longhornio/longhorn-instance-manager" + }, + "longhorn-manager": { + "finalImageTag": "v1.11.2", + "hash": "sha256-w2BvuR0oPN9H1fY3DYGd+5+pj0ECOmoEfuAMOnW7vjo=", + "imageDigest": "sha256:0f80ca11ac4eb7522f4e6e801a7afc9909ea8d3041575f3d029964c46590f096", + "imageName": "docker.io/longhornio/longhorn-manager" + }, + "longhorn-share-manager": { + "finalImageTag": "v1.11.2", + "hash": "sha256-1JByktvTg0MWqF4qGn74zmfX/az/GjMux0RrYxMRIE8=", + "imageDigest": "sha256:c11559e998ea982e6bac1637d66cc2aaab662a6b546709f2e54e2bfa50ffb0c3", + "imageName": "docker.io/longhornio/longhorn-share-manager" + }, + "longhorn-ui": { + "finalImageTag": "v1.11.2", + "hash": "sha256-z8/YJtkoVM2CKnoO1yoJdl+OfKQWuiw8VNbipj2ZOUs=", + "imageDigest": "sha256:885bc78f99f31da0d9b0fd8f533a53558a3aa81f9719c62e0d3c69ed8456d5b7", + "imageName": "docker.io/longhornio/longhorn-ui" + }, + "pihole": { + "finalImageTag": "2025.11.1", + "hash": "sha256-0CYHbyO4MgCwmr2vpaIurQhweK8IkiM61uiWIrJFumM=", + "imageDigest": "sha256:91dc91ddd413bf461c283204558f8f21839851e9824799075a7ceff7c77eea40", + "imageName": "docker.io/pihole/pihole" + }, + "redis": { + "finalImageTag": "8.2.3-alpine", + "hash": "sha256-uao+VHcBa3kuM46wLFHUbP71iAAEl/sg79xicQx3zp0=", + "imageDigest": "sha256:08ad0b1d280850169a790dba1393ff7a90aef951fc19632cf4d3ce4f78e679ba", + "imageName": "ecr-public.aws.com/docker/library/redis" + }, + "sops": { + "finalImageTag": "v3.13.3-alpine", + "hash": "sha256-LYnxTOLZBhdmlYFySLZLGemnNmHjI+cxLfiPtjWciOc=", + "imageDigest": "sha256:ae501277bf742f1662e0f881f43dd8fd6798b489a8058e921dbf6cda597140ea", + "imageName": "ghcr.io/getsops/sops" + }, + "speaker": { + "finalImageTag": "v0.15.3", + "hash": "sha256-t2epMeOG1rfWG3juOMfEhb9bfzksvQ0SuwOLJKB1ob4=", + "imageDigest": "sha256:c6a5b25b2e1fba610a57b2db4bb8141d7c133569d561a8cc29e38ca5113efbc4", + "imageName": "quay.io/metallb/speaker" + }, + "traefik": { + "finalImageTag": "v3.7.0", + "hash": "sha256-vrr2L/QJYNZOzAcrIBHFGbIDHC0j4wkqUXw8XQkF5/8=", + "imageDigest": "sha256:eb328e2c806c53aafbbace6c451fa54d268961261a85452fcf0fb752a30c17be", + "imageName": "docker.io/library/traefik" + }, + "whoami": { + "finalImageTag": "v1.11.0", + "hash": "sha256-qBTCdI2Hf7Rd8SCWOge0ZtJ+gK5C1Ztq/XfeU5clMew=", + "imageDigest": "sha256:200689790a0a0ea48ca45992e0450bc26ccab5307375b41c84dfc4f2475937ab", + "imageName": "docker.io/traefik/whoami" + }, + "wireguard": { + "finalImageTag": "latest", + "hash": "sha256-mBzayAz7RneKTuBVLOMCfKk7YTrgxCc5V2RjA18A4xM=", + "imageDigest": "sha256:b677c3f46d524921c8a7c91873479eb7bab190d098fddbbd887ce333ec078c90", + "imageName": "lscr.io/linuxserver/wireguard" + } + }, "vm-cluster": { "argocd": { "finalImageTag": "v3.4.1", diff --git a/tests/vm/upgrade.nix b/tests/vm/upgrade.nix new file mode 100644 index 0000000..fd96a24 --- /dev/null +++ b/tests/vm/upgrade.nix @@ -0,0 +1,95 @@ +# NixOS VM test of upgrading from the previous release: its whole render is +# applied to a cluster for real, then the current render over it, the API +# server judging (server-side dry run, CRDs for real, like Argo CD applies +# them). Catches what only breaks on upgrade: a field that can't change on an +# existing object (a selector, a StatefulSet's volume templates, a Job's +# template), a CRD version still stored but no longer served. +# +# Webhook configurations are left out: their services don't run here. +{ pkgs, mkTestLab, testLib, inputs, allLayersOff, labArgs }: + +let + previousVersion = "v1.24.2"; + previous = builtins.fetchTarball { + url = "https://github.com/thatbagu/nixlab/archive/refs/tags/${previousVersion}.tar.gz"; + sha256 = "0djsxqmih6q8dpdn1j768fzw9wkbr20ar0shlv4q7xcp934y59py"; + }; + # The previous release's code, with today's inputs (what changes between + # releases is nixlab's own render; inputs have their own update checks). + previousMkNixlab = import "${previous}/system/mk-nixlab.nix" { inherit inputs; }; + before = (previousMkNixlab labArgs).nixidyEnv.environmentPackage; + after = (mkTestLab { }).nixidyEnv.environmentPackage; + + # The cluster itself: k3s and nothing else. + cluster = mkTestLab { secretsFile = ./secrets.yaml; overrides = allLayersOff; }; + k = "k3s kubectl"; + apply = pkgs.writeShellApplication { + name = "apply-render"; + runtimeInputs = with pkgs; [ kubectl findutils gnugrep gawk ]; + text = '' + # apply-render + ENV=$1 real=$2 + mapfile -t crds < <(find -L "$ENV" -name 'CustomResourceDefinition-*.yaml' | sort) + # CRDs for real either way (Argo CD applies them first). + for f in "''${crds[@]}"; do kubectl apply --server-side --force-conflicts -f "$f" >/dev/null || { echo "CRD ''${f#"$ENV"/} rejected" >&2; exit 1; }; done + kubectl wait --for=condition=Established --timeout=120s "''${crds[@]/#/-f}" >/dev/null + mapfile -t namespaces < <(grep -RhoE '^ namespace: [a-z0-9-]+' "$ENV" | awk '{print $2}' | sort -u) + for ns in "''${namespaces[@]}"; do + kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + done + dry=(); [[ $real == yes ]] || dry=(--dry-run=server) + errors=0 count=0 + while IFS= read -r f; do + count=$((count + 1)) + if ! out=$(kubectl apply "''${dry[@]}" --server-side --force-conflicts -f "$f" 2>&1); then + echo "''${f#"$ENV"/}: $out" >&2 + errors=$((errors + 1)) + fi + done < <(find -L "$ENV" -name '*.yaml' ! -name 'CustomResourceDefinition-*' ! -name '*WebhookConfiguration-*' | sort) + echo "''${#crds[@]} CRDs, $count objects, $errors rejected" + [[ $errors -eq 0 ]] + ''; + }; +in +testLib.mkVmTest { + name = "nixlab-upgrade"; + nodeName = "tmaster"; + specialArgs = cluster.specialArgsFor "tmaster"; + + node = { + imports = cluster.modulesFor "tmaster"; + virtualisation = { + memorySize = 3072; + cores = 2; + diskSize = 4096; + emptyDiskImages = [ 8192 ]; + fileSystems."/persist" = { device = "/dev/vdb"; fsType = "ext4"; autoFormat = true; neededForBoot = true; }; + }; + system.activationScripts = { + vmTestAgeKey.text = '' + install -D -m 0400 ${./age-key.txt} /persist/etc/sops-nix/keys.txt + ''; + setupSecrets.deps = [ "vmTestAgeKey" ]; + setupSecretsForUsers.deps = [ "vmTestAgeKey" ]; + }; + services.k3s.images = [ pkgs.k3s.passthru.airgap-images ]; + }; + + subtests = [ + { + name = "k3s comes up"; + steps = [ + { waitForUnit = "k3s.service"; } + { waitUntil = "${k} get nodes | grep -w Ready"; } + ]; + } + { + name = "the previous release's render (${previousVersion}), applied for real"; + steps = [ { succeed = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml ${apply}/bin/apply-render ${before} yes"; timeout = 900; } ]; + } + { + name = "the current render applies over it"; + steps = [ { succeed = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml ${apply}/bin/apply-render ${after} no"; timeout = 900; } ]; + } + ]; +} diff --git a/tools/update.nix b/tools/update.nix index 8f9972b..adf3b70 100644 --- a/tools/update.nix +++ b/tools/update.nix @@ -42,6 +42,12 @@ let vm-app-backups = { render = "render-app-backups-vm"; apps = [ "snapshot-controller" "volsync" "nixlab-secrets" "notes" ]; extra = [ ]; }; vm-single = { render = "render-single-vm"; apps = [ "traefik" "cert-manager" "cert-manager-issuers" "pihole" "externaldns-pihole" "wireguard" "nixlab-secrets" "whoami" ]; extra = [ ]; }; vm-gateway = { render = "render-gateway-vm"; apps = [ "metallb" "metallb-config" "traefik" "cert-manager" "cert-manager-issuers" "haproxy" "whoami" ]; extra = [ ]; }; + vm-bootstrap = { + render = "render-bootstrap-vm"; + apps = [ "argocd" "nixlab-secrets" "metallb" "metallb-config" "traefik" "haproxy" "cert-manager" "cert-manager-issuers" "longhorn" "pihole" "externaldns-pihole" "wireguard" "whoami" ]; + extra = [ ]; + also = "(docker\\.io/)?longhornio/(longhorn-(manager|engine|instance-manager)|csi-[a-z-]+|livenessprobe):[A-Za-z0-9._-]+"; + }; }; build = check: "nix build --no-link --print-out-paths .#checks.x86_64-linux.${check}"; renders = lib.unique (map (t: t.render) (lib.attrValues vmTests)); From 0c9915d885def59bbf64ba621527bf29f6ba1d70 Mon Sep 17 00:00:00 2001 From: thatbagu Date: Sat, 21 Nov 2026 16:48:39 +0500 Subject: [PATCH 2/5] test(vm-bootstrap): give k3s time to import its images k3s imports every preloaded image (29 here) before the kubelet starts, which took over 5 minutes on CI. Co-Authored-By: Claude Opus 5.5 --- tests/vm/bootstrap.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/vm/bootstrap.nix b/tests/vm/bootstrap.nix index 3d2e4c2..ca03ade 100644 --- a/tests/vm/bootstrap.nix +++ b/tests/vm/bootstrap.nix @@ -70,7 +70,8 @@ testLib.mkVmTest { name = "the cluster comes up and Argo CD bootstraps it"; steps = [ { waitForUnit = "k3s.service"; } - { waitUntil = "${k} get nodes | grep -w Ready"; } + # k3s imports every preloaded image before its kubelet starts: minutes. + { waitUntil = "${k} get nodes | grep -w Ready"; timeout = 1800; } { waitForUnit = "rendered-repo.service"; } { waitUntil = "curl -sf http://192.168.1.1:8000/rendered.git/info/refs | grep -c k8s-rendered"; } { waitForUnit = "argocd-bootstrap.service"; timeout = 600; } From 4d2b1d6457415f3ab0a9af6770538f76bfa3393a Mon Sep 17 00:00:00 2001 From: thatbagu Date: Sat, 21 Nov 2026 17:32:59 +0500 Subject: [PATCH 3/5] test(vm-bootstrap): serve the rendered branch over git's smart HTTP Argo CD lists refs with go-git, which doesn't speak git's plain HTTP protocol ("failed to list refs: unexpected EOF"). A small bridge to git http-backend serves the smart one. Co-Authored-By: Claude Opus 5.5 --- tests/vm/bootstrap.nix | 8 ++++---- tests/vm/git-http.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/vm/git-http.py diff --git a/tests/vm/bootstrap.nix b/tests/vm/bootstrap.nix index ca03ade..2bd5c9b 100644 --- a/tests/vm/bootstrap.nix +++ b/tests/vm/bootstrap.nix @@ -45,8 +45,9 @@ testLib.mkVmTest { services.k3s.images = [ pkgs.k3s.passthru.airgap-images ] ++ images; environment.systemPackages = [ pkgs.curl ]; - # The rendered branch, as CI pushes it, over git's plain HTTP protocol - # (gitRepository in the lab points here). + # The rendered branch, as CI pushes it, over git's smart HTTP protocol + # (Argo CD's go-git doesn't speak the plain one; gitRepository in the lab + # points here). systemd.services.rendered-repo = { wantedBy = [ "multi-user.target" ]; before = [ "argocd-bootstrap.service" ]; @@ -59,8 +60,7 @@ testLib.mkVmTest { git init -q -b k8s-rendered && git add -A git -c user.email=ci@test -c user.name=ci commit -qm render git clone -q --bare /run/rendered /srv/git/rendered.git - git -C /srv/git/rendered.git update-server-info - exec python3 -m http.server 8000 --directory /srv/git + exec python3 ${./git-http.py} /srv/git 8000 ''; }; }; diff --git a/tests/vm/git-http.py b/tests/vm/git-http.py new file mode 100644 index 0000000..fcc7d1e --- /dev/null +++ b/tests/vm/git-http.py @@ -0,0 +1,29 @@ +# git's smart HTTP protocol (what Argo CD's go-git speaks), via git http-backend. +import http.server, os, subprocess, sys + +root = sys.argv[1] + +class Handler(http.server.BaseHTTPRequestHandler): + def run(self): + path, _, query = self.path.partition("?") + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) if length else b"" + env = dict(os.environ, GIT_PROJECT_ROOT=root, GIT_HTTP_EXPORT_ALL="1", + PATH_INFO=path, QUERY_STRING=query, REQUEST_METHOD=self.command, + CONTENT_TYPE=self.headers.get("Content-Type", ""), CONTENT_LENGTH=str(len(body)), + REMOTE_ADDR=self.client_address[0]) + out = subprocess.run(["git", "http-backend"], input=body, env=env, capture_output=True).stdout + head, _, rest = out.partition(b"\r\n\r\n") + status, headers = 200, [] + for line in head.decode().split("\r\n"): + k, _, v = line.partition(": ") + if k.lower() == "status": status = int(v.split()[0]) + elif k: headers.append((k, v)) + self.send_response(status) + for k, v in headers: self.send_header(k, v) + self.send_header("Content-Length", str(len(rest))) + self.end_headers() + self.wfile.write(rest) + do_GET = do_POST = run + +http.server.ThreadingHTTPServer(("0.0.0.0", int(sys.argv[2])), Handler).serve_forever() From b4af5ddceb050534aa5704608adba97b13902de6 Mon Sep 17 00:00:00 2001 From: thatbagu Date: Sat, 21 Nov 2026 17:46:02 +0500 Subject: [PATCH 4/5] refactor(update): an image group without an app list covers its whole render vm-bootstrap listed its 13 apps by hand, duplicating the render: a core app added later would have been missing from its preloaded images. Groups may now leave apps out, and get every app in their render (and are rerun for any change to it). Co-Authored-By: Claude Opus 5.5 --- tools/update.nix | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tools/update.nix b/tools/update.nix index adf3b70..43721ee 100644 --- a/tools/update.nix +++ b/tools/update.nix @@ -23,9 +23,10 @@ let # Where pinned images and charts live: the platform's, or your repo's. digestDirs = if library then "system" else "apps"; chartsDir = if library then "system/k8s/charts" else "charts"; - # Each VM test's preloaded images: those of a rendered app, plus the test's - # own fixtures (not part of the render). A chart or image update runs the - # VM tests whose apps it changes. + # Each VM test's preloaded images: those of its rendered apps (apps; left + # out: every app in its render), plus the test's own fixtures (not part of + # the render). A chart or image update runs the VM tests whose apps it + # changes. vmTests = { vm-cluster = { render = "render-default"; apps = [ "argocd" "nixlab-secrets" ]; extra = [ "docker.io/traefik/whoami:v1.11.0" ]; }; vm-monitoring = { render = "render-monitoring"; apps = [ "monitoring" "nixlab-secrets" ]; extra = [ "docker.io/mendhak/http-https-echo:43" ]; }; @@ -44,7 +45,6 @@ let vm-gateway = { render = "render-gateway-vm"; apps = [ "metallb" "metallb-config" "traefik" "cert-manager" "cert-manager-issuers" "haproxy" "whoami" ]; extra = [ ]; }; vm-bootstrap = { render = "render-bootstrap-vm"; - apps = [ "argocd" "nixlab-secrets" "metallb" "metallb-config" "traefik" "haproxy" "cert-manager" "cert-manager-issuers" "longhorn" "pihole" "externaldns-pihole" "wireguard" "whoami" ]; extra = [ ]; also = "(docker\\.io/)?longhornio/(longhorn-(manager|engine|instance-manager)|csi-[a-z-]+|livenessprobe):[A-Za-z0-9._-]+"; }; @@ -58,17 +58,16 @@ let next=$(mktemp); echo '{}' > "$next" ${lib.concatStrings (lib.mapAttrsToList (test: t: '' env=$(${build t.render}) + ${if t ? apps then "vm_apps=(${lib.escapeShellArgs t.apps})" else ''vm_apps=(); for d in "$env"/*/; do d=''${d%/}; d=''${d##*/}; [[ $d == apps ]] || vm_apps+=("$d"); done''} wanted=$( { - # Charts' own test pods (helm test) never run in the VM tests. - ${lib.concatMapStrings (app: '' + for vm_app in "''${vm_apps[@]}"; do + # Charts' own test pods (helm test) never run in the VM tests. # Images without a registry (pihole/pihole:...) are Docker Hub's. - grep -RhoE --exclude='Pod-*-test*.yaml' '(image: |prometheus-config-reloader=)"?[a-z0-9][a-z0-9./_-]*(:[A-Za-z0-9._-]+)?(@sha256:[0-9a-f]{64})?' "$env/${app}" \ + grep -RhoE --exclude='Pod-*-test*.yaml' '(image: |prometheus-config-reloader=)"?[a-z0-9][a-z0-9./_-]*(:[A-Za-z0-9._-]+)?(@sha256:[0-9a-f]{64})?' "$env/$vm_app" \ | sed -E 's/^(image: |prometheus-config-reloader=)"?//' \ | sed -E '/^[^/]+\.[a-z]+\//! s|^|docker.io/|; s|^docker\.io/([^/]+)(:[^/]*)?$|docker.io/library/\1\2|' - '') t.apps} - ${lib.optionalString (t ? also) (lib.concatMapStrings (app: '' - grep -RhoE ${lib.escapeShellArg t.also} "$env/${app}" | sed -E 's|^longhornio/|docker.io/longhornio/|' - '') t.apps)} + ${lib.optionalString (t ? also) ''grep -RhoE ${lib.escapeShellArg t.also} "$env/$vm_app" | sed -E 's|^longhornio/|docker.io/longhornio/|' ''} + done ${lib.optionalString (t.extra != [ ]) "printf '%s\\n' ${lib.escapeShellArgs t.extra}"} } | sort -u) for ref in $wanted; do @@ -109,7 +108,9 @@ mkTool { if [[ "$kind" == input ]]; then echo "$all"; return; fi grep -vxE 'host-.*|vm-.*' <<<"$all" ${lib.concatStrings (lib.mapAttrsToList (test: t: '' - if ${if t.apps == [ ] then "false" else lib.concatMapStringsSep " || " (app: ''! diff -rq "''${before[${t.render}]}/${app}" "''${after[${t.render}]}/${app}" >/dev/null 2>&1'') t.apps}; then echo ${test}; fi + if ${if !(t ? apps) then ''! diff -rq "''${before[${t.render}]}" "''${after[${t.render}]}" >/dev/null 2>&1'' + else if t.apps == [ ] then "false" + else lib.concatMapStringsSep " || " (app: ''! diff -rq "''${before[${t.render}]}/${app}" "''${after[${t.render}]}/${app}" >/dev/null 2>&1'') t.apps}; then echo ${test}; fi '') vmTests)} # vm-cluster checks the whole default render against the API server. if ! diff -rq "''${before[render-default]}" "''${after[render-default]}" >/dev/null 2>&1; then echo vm-cluster; fi From 4cede2549447a26432470afd371e3d91f75c638e Mon Sep 17 00:00:00 2001 From: thatbagu Date: Sat, 21 Nov 2026 20:19:56 +0500 Subject: [PATCH 5/5] test(vm-bootstrap): commit the render's files, not its symlinks The render's directories are symlinks into the Nix store; committed as such, Argo CD refuses the repository ("out-of-bounds symlinks"). Co-Authored-By: Claude Opus 5.5 --- tests/vm/bootstrap.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vm/bootstrap.nix b/tests/vm/bootstrap.nix index 2bd5c9b..399b9a5 100644 --- a/tests/vm/bootstrap.nix +++ b/tests/vm/bootstrap.nix @@ -54,7 +54,7 @@ testLib.mkVmTest { path = with pkgs; [ git gnused python3 ]; script = '' rm -rf /run/rendered /srv/git; mkdir -p /run/rendered /srv/git - cp -r ${env}/. /run/rendered/ && chmod -R u+w /run/rendered + cp -rL ${env}/. /run/rendered/ && chmod -R u+w /run/rendered find /run/rendered -name '*.yaml' -exec sed -i -E 's/@sha256:[0-9a-f]{64}//' {} + cd /run/rendered git init -q -b k8s-rendered && git add -A