diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35e40c6..195ff0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - group: [quick, quick-eval, quick-render, hosts, vm-cluster, vm-minimal, vm-sso, vm-gateway, vm-ha, vm-longhorn, vm-install, vm-app-backups, vm-single, vm-install-encrypted, vm-nas, vm-power, vm-tailscale, vm-monitoring, vm-monitoring-full] + group: [quick, quick-eval, quick-render, hosts, vm-cluster, vm-minimal, vm-sso, vm-gateway, vm-bootstrap, vm-upgrade, vm-ha, vm-longhorn, vm-install, vm-app-backups, vm-single, vm-install-encrypted, vm-nas, vm-power, vm-tailscale, vm-monitoring, vm-monitoring-full] name: checks (${{ matrix.group }}) steps: - uses: actions/checkout@v4 @@ -61,7 +61,7 @@ jobs: # everything at once: together they exhaust a runner's memory. filter='n: let g = builtins.getEnv "GROUP"; - vmsAndHosts = [ "host-master" "host-worker" "vm-cluster" "vm-minimal" "vm-sso" "vm-gateway" "vm-ha" "vm-longhorn" "vm-install" "vm-app-backups" "vm-single" "vm-install-encrypted" "vm-nas" "vm-power" "vm-tailscale" "vm-monitoring" "vm-monitoring-full"]; + vmsAndHosts = [ "host-master" "host-worker" "vm-cluster" "vm-minimal" "vm-sso" "vm-gateway" "vm-bootstrap" "vm-upgrade" "vm-ha" "vm-longhorn" "vm-install" "vm-app-backups" "vm-single" "vm-install-encrypted" "vm-nas" "vm-power" "vm-tailscale" "vm-monitoring" "vm-monitoring-full"]; prefix = p: builtins.substring 0 (builtins.stringLength p) n == p; in if g == "quick" then !(builtins.elem n vmsAndHosts) && !(prefix "eval-") && !(prefix "render-") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74d67d8..eb53aef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,6 +93,10 @@ chore: bump pihole to 2025.11.1 Subject line: imperative mood, no period, 72 characters max. Body: explain the *why*, not the *what*, when it's not obvious. +## Releasing + +After tagging a release, point the upgrade test at it, so the next release is checked against this one: in `tests/vm/upgrade.nix`, set `previousVersion` to the new tag and `sha256` to the output of `nix-prefetch-url --unpack https://github.com/thatbagu/nixlab/archive/refs/tags/.tar.gz`. + ## License MIT. By contributing you agree your changes are released under the same license. diff --git a/tests/default.nix b/tests/default.nix index e2a5d3e..60441fd 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -53,7 +53,8 @@ let # overrides merge into baseVars; replace swaps whole top-level settings # (e.g. nodes, to drop one). - mkTestLab = { overrides ? { }, replace ? { }, extraAppModules ? [ ], extraNixosModules ? [ ], secretsFile ? testSecrets, extraSecrets ? [ ] }: mkNixlab { + # (testLabArgs: what it passes to mkNixlab, for another version's mkNixlab.) + testLabArgs = { overrides ? { }, replace ? { }, extraAppModules ? [ ], extraNixosModules ? [ ], secretsFile ? testSecrets, extraSecrets ? [ ] }: { vars = lib.recursiveUpdate baseVars overrides // replace; inherit secretsFile; # The unit tests' secrets are fake and plaintext; the VM tests' are encrypted. @@ -61,6 +62,7 @@ let hardwareConfig = _: ../template/hosts/example/hardware-configuration.nix; inherit extraAppModules extraNixosModules extraSecrets; }; + mkTestLab = args: mkNixlab (testLabArgs args); # One machine with the whole platform (profile = "single"). singleNode = { @@ -143,6 +145,26 @@ let backups = { enable = true; target = "nfs://192.168.1.1:/export/backups"; }; }; + # The fresh-bootstrap VM test: Argo CD syncs every layer from a git + # repository the machine serves itself (the rendered branch; the test + # network numbers its one machine .1). No internet: the internal CA, no + # Nextcloud. + bootstrapVm = { + overrides = allLayersOff // { + gitops.enable = true; + metallb.enable = true; + ingress.enable = true; + edge.enable = true; + certManager.enable = true; + cloudflare.enable = false; + longhorn.enable = true; + pihole.enable = true; + vpn.enable = true; + gitRepository = "http://192.168.1.1:8000/rendered.git"; + }; + extraAppModules = [ gatewayVmApp ]; + }; + # The per-app backup VM test: k3s' local storage (VolSync reads the volume # itself), MinIO on another machine (the test network numbers machines in # name order: s3 .1, tmaster .2) as the S3 target, and an app with a volume. @@ -204,6 +226,7 @@ let longhorn-vm.overrides = longhornVmOverrides; # The gateway VM test's setup. gateway-vm = { overrides = gatewayVmOverrides; extraAppModules = [ gatewayVmApp ]; }; + bootstrap-vm = bootstrapVm; # Every layer off: k3s on NixOS and nothing else. minimal.overrides = allLayersOff; # Everything on one machine (profile = "single"). @@ -1041,6 +1064,10 @@ lib.mapAttrs' (name: lab: lib.nameValuePair "render-${name}" lab.nixidyEnv.envir # NixOS VM test: HTTPS through the gateway (Traefik, Gateway API), with a # cert-manager certificate, behind the edge proxy (tests/vm/gateway.nix). vm-gateway = import ./vm/gateway.nix { inherit pkgs mkTestLab testLib; overrides = gatewayVmOverrides; extraAppModules = [ gatewayVmApp ]; }; + # A fresh cluster bootstrapping every layer through Argo CD, from git. + vm-bootstrap = import ./vm/bootstrap.nix { inherit pkgs mkTestLab testLib; inherit (bootstrapVm) overrides extraAppModules; }; + # Upgrading from the previous release: its render, then the current one over it. + vm-upgrade = import ./vm/upgrade.nix { inherit pkgs mkTestLab testLib inputs allLayersOff; labArgs = testLabArgs { }; }; # NixOS VM test: monitoring on the real master, alerts delivered to a # stand-in webhook (tests/vm/monitoring.nix). # NixOS VM test: the UPS shuts the cluster down in order (tests/vm/power.nix). diff --git a/tests/vm/bootstrap.nix b/tests/vm/bootstrap.nix new file mode 100644 index 0000000..399b9a5 --- /dev/null +++ b/tests/vm/bootstrap.nix @@ -0,0 +1,110 @@ +# NixOS VM test of a fresh cluster bootstrapping by itself, the way a real +# one does: Argo CD syncs every layer (MetalLB, the gateway, cert-manager with +# the internal CA, Longhorn, Pi-hole, the VPN, a web app) from the rendered +# branch, which the machine serves itself as a git repository. Checks that, +# with nobody touching anything: +# - every Application ends up Synced and Healthy (sync waves, Longhorn's +# one-time manual sync, a gateway whose certificates come later, Secrets +# for namespaces another app creates) +# - the web app answers HTTPS through the edge proxy and the gateway +# +# The VM has no internet: images are preloaded (images.nix), and the served +# branch drops their registry digests (preloaded images keep only the tag). +{ pkgs, mkTestLab, testLib, overrides, extraAppModules }: + +let + images = (import ./images.nix { inherit pkgs; }).vm-bootstrap; + lab = mkTestLab { secretsFile = ./secrets.yaml; inherit overrides extraAppModules; }; + env = lab.nixidyEnv.environmentPackage; + k = "k3s kubectl"; + apps = "${k} -n argocd get applications -o custom-columns=A:.metadata.name,S:.status.sync.status,H:.status.health.status --no-headers"; +in +testLib.mkVmTest { + name = "nixlab-bootstrap"; + nodeName = "tmaster"; + specialArgs = lab.specialArgsFor "tmaster"; + + node = { + imports = lab.modulesFor "tmaster"; + virtualisation = { + memorySize = 8192; + cores = 6; + diskSize = 8192; + emptyDiskImages = [ 24576 ]; + fileSystems."/persist" = { device = "/dev/vdb"; fsType = "ext4"; autoFormat = true; neededForBoot = true; }; + }; + system.activationScripts = { + vmTestAgeKey.text = '' + install -D -m 0400 ${./age-key.txt} /persist/etc/sops-nix/keys.txt + ''; + setupSecrets.deps = [ "vmTestAgeKey" ]; + setupSecretsForUsers.deps = [ "vmTestAgeKey" ]; + }; + networking.firewall.allowedTCPPorts = [ 7946 ]; + networking.firewall.allowedUDPPorts = [ 7946 ]; + services.k3s.images = [ pkgs.k3s.passthru.airgap-images ] ++ images; + environment.systemPackages = [ pkgs.curl ]; + + # The rendered branch, as CI pushes it, over git's smart HTTP protocol + # (Argo CD's go-git doesn't speak the plain one; gitRepository in the lab + # points here). + systemd.services.rendered-repo = { + wantedBy = [ "multi-user.target" ]; + before = [ "argocd-bootstrap.service" ]; + path = with pkgs; [ git gnused python3 ]; + script = '' + rm -rf /run/rendered /srv/git; mkdir -p /run/rendered /srv/git + cp -rL ${env}/. /run/rendered/ && chmod -R u+w /run/rendered + find /run/rendered -name '*.yaml' -exec sed -i -E 's/@sha256:[0-9a-f]{64}//' {} + + cd /run/rendered + git init -q -b k8s-rendered && git add -A + git -c user.email=ci@test -c user.name=ci commit -qm render + git clone -q --bare /run/rendered /srv/git/rendered.git + exec python3 ${./git-http.py} /srv/git 8000 + ''; + }; + }; + + subtests = [ + { + name = "the cluster comes up and Argo CD bootstraps it"; + steps = [ + { waitForUnit = "k3s.service"; } + # k3s imports every preloaded image before its kubelet starts: minutes. + { waitUntil = "${k} get nodes | grep -w Ready"; timeout = 1800; } + { waitForUnit = "rendered-repo.service"; } + { waitUntil = "curl -sf http://192.168.1.1:8000/rendered.git/info/refs | grep -c k8s-rendered"; } + { waitForUnit = "argocd-bootstrap.service"; timeout = 600; } + ]; + } + { + name = "every app ends up Synced and Healthy, by itself"; + steps = [ + { desc = "all of them exist"; + waitUntil = "[ $(${apps} | wc -l) -ge ${toString (builtins.length (builtins.attrNames lab.nixidyEnv.config.applications) - 1)} ]"; timeout = 1800; } + { desc = "and all of them are Synced and Healthy"; + waitUntil = "${apps} >&2; [ -z \"$(${apps} | grep -v 'Synced *Healthy')\" ]"; timeout = 2400; } + ]; + } + { + name = "the web app answers through the edge proxy and the gateway"; + steps = [ + { waitUntil = "curl -skf --resolve whoami.yourdomain.example.com:443:192.168.1.193 https://whoami.yourdomain.example.com/ | grep -c Hostname"; timeout = 600; } + ]; + } + { + name = "Longhorn's first sync went through without help"; + steps = [ + { succeed = "[ \"$(${k} -n argocd get application longhorn -o jsonpath='{.status.operationState.phase}')\" = Succeeded ]"; } + { succeed = "${k} -n longhorn-system get nodes.longhorn.io tmaster"; } + ]; + } + ]; + + onFailure = [ + "${apps}" + "${k} -n argocd get applications -o json | ${pkgs.jq}/bin/jq -r '.items[] | select(.status.sync.status != \"Synced\" or .status.health.status != \"Healthy\") | \"\\(.metadata.name): \\(.status.operationState.phase // \"-\") \\(.status.operationState.message // \"\" | .[0:300])\"'" + "${k} get pods -A | grep -vE 'Running|Completed'" + "journalctl -u argocd-bootstrap --no-pager | tail -30" + ]; +} diff --git a/tests/vm/git-http.py b/tests/vm/git-http.py new file mode 100644 index 0000000..fcc7d1e --- /dev/null +++ b/tests/vm/git-http.py @@ -0,0 +1,29 @@ +# git's smart HTTP protocol (what Argo CD's go-git speaks), via git http-backend. +import http.server, os, subprocess, sys + +root = sys.argv[1] + +class Handler(http.server.BaseHTTPRequestHandler): + def run(self): + path, _, query = self.path.partition("?") + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) if length else b"" + env = dict(os.environ, GIT_PROJECT_ROOT=root, GIT_HTTP_EXPORT_ALL="1", + PATH_INFO=path, QUERY_STRING=query, REQUEST_METHOD=self.command, + CONTENT_TYPE=self.headers.get("Content-Type", ""), CONTENT_LENGTH=str(len(body)), + REMOTE_ADDR=self.client_address[0]) + out = subprocess.run(["git", "http-backend"], input=body, env=env, capture_output=True).stdout + head, _, rest = out.partition(b"\r\n\r\n") + status, headers = 200, [] + for line in head.decode().split("\r\n"): + k, _, v = line.partition(": ") + if k.lower() == "status": status = int(v.split()[0]) + elif k: headers.append((k, v)) + self.send_response(status) + for k, v in headers: self.send_header(k, v) + self.send_header("Content-Length", str(len(rest))) + self.end_headers() + self.wfile.write(rest) + do_GET = do_POST = run + +http.server.ThreadingHTTPServer(("0.0.0.0", int(sys.argv[2])), Handler).serve_forever() diff --git a/tests/vm/images.json b/tests/vm/images.json index d0eab69..86d76ee 100644 --- a/tests/vm/images.json +++ b/tests/vm/images.json @@ -31,6 +31,176 @@ "imageName": "quay.io/backube/volsync" } }, + "vm-bootstrap": { + "alpine": { + "finalImageTag": "3.22", + "hash": "sha256-tH8dnuPM5eClAzXUtr1sxf2trSLsFsuyYh97O2qLNAc=", + "imageDigest": "sha256:2e0b81d60597f449baa56dc25d355367869d2dca4cd57e866f1e2b52af3eb55f", + "imageName": "docker.io/library/alpine" + }, + "argocd": { + "finalImageTag": "v3.4.1", + "hash": "sha256-/NDpUXALQw6UMHLEzAEZAQIVW2l85HT+KpYsbwh3MCA=", + "imageDigest": "sha256:097c922fb910e6bb0278b0e26382f88de4fa5c966564111bddd334051b0e63d7", + "imageName": "quay.io/argoproj/argocd" + }, + "cert-manager-cainjector": { + "finalImageTag": "v1.20.2", + "hash": "sha256-k7KNy3kmJS9qQft0sNYC3UnhVPA9CS/nDDbNDdp4emg=", + "imageDigest": "sha256:6f5a644135887b2aa7d5cc145072fa56421560e3586ff1f184358022d490f4e1", + "imageName": "quay.io/jetstack/cert-manager-cainjector" + }, + "cert-manager-controller": { + "finalImageTag": "v1.20.2", + "hash": "sha256-lcjoakYzDhQiCb5Bu5ZLFf8exWYqhzFRjHEi3jpZpaA=", + "imageDigest": "sha256:fe0623d7d04a382c888f03343a3a2da716e0d96ad3d5d790c0ebcbcb2a4329a5", + "imageName": "quay.io/jetstack/cert-manager-controller" + }, + "cert-manager-startupapicheck": { + "finalImageTag": "v1.20.2", + "hash": "sha256-qqs3zPuv+8PkcgMAUATH15/r7Wppq+OyhAgvseeKIhM=", + "imageDigest": "sha256:4e2a69b4a0cc9627905bbeecf720f95d5153ca39cacdab923d2748e73556792b", + "imageName": "quay.io/jetstack/cert-manager-startupapicheck" + }, + "cert-manager-webhook": { + "finalImageTag": "v1.20.2", + "hash": "sha256-Tq+dBEIAV1ButQCgZ7fKfRie8B2VwnDC4GUVTDc1CQA=", + "imageDigest": "sha256:baf651128b9f05c426cbd5e60e2036bf382c99ca270f49d0757d6f7d2452f4e5", + "imageName": "quay.io/jetstack/cert-manager-webhook" + }, + "controller": { + "finalImageTag": "v0.15.3", + "hash": "sha256-j71loxJPSdOfqSGX3b5/4X7OtwhQlslAsSGnhpDlcZ8=", + "imageDigest": "sha256:6698ccc54c380913816ed1fd0758637ec87dd79da419c4ab170a2c26c158ab89", + "imageName": "quay.io/metallb/controller" + }, + "csi-attacher": { + "finalImageTag": "v4.11.0-20260428", + "hash": "sha256-+defbWikOWKmZvwOX8vAy5cbwQa6EDiJOnfOLe4fzuo=", + "imageDigest": "sha256:fe417c28a6b86f8e7e5d49fc223e22e9ab457f894d2c4a321932d136dc2c2530", + "imageName": "docker.io/longhornio/csi-attacher" + }, + "csi-node-driver-registrar": { + "finalImageTag": "v2.16.0-20260428", + "hash": "sha256-tcltO1AEnHotTxyJ+N7FbzvBS2U1DgOrikGBB8lOA2k=", + "imageDigest": "sha256:e82a8c8f800d7fbb3c1edf3f90b557768091821a44d52280093394f7918ccb68", + "imageName": "docker.io/longhornio/csi-node-driver-registrar" + }, + "csi-provisioner": { + "finalImageTag": "v5.3.0-20260428", + "hash": "sha256-nUkForHWNHisq1cMNTjzJ3wuy7dmWBHSi/H2Zaa6ewA=", + "imageDigest": "sha256:9e519a21a77c060104716e1f98222bb46ab617778a3bfcd861c87119a8256764", + "imageName": "docker.io/longhornio/csi-provisioner" + }, + "csi-resizer": { + "finalImageTag": "v2.1.0-20260428", + "hash": "sha256-BPXAG3NIacqmf3f8V4kjWZIfYw3Becdn8oCuNStGulE=", + "imageDigest": "sha256:41cb674d1154e798aa2c20f53f72ee2a5597f1369bcad5878d1708aee47f6663", + "imageName": "docker.io/longhornio/csi-resizer" + }, + "csi-snapshotter": { + "finalImageTag": "v8.5.0-20260428", + "hash": "sha256-tDLxZZUsFzerg1q0ofOD5IrdtxH2ywFFjJ3u/rG79xc=", + "imageDigest": "sha256:1975fac3890f4e08b98792881cb597502112ce0eeeaaef383e52458c96db94c5", + "imageName": "docker.io/longhornio/csi-snapshotter" + }, + "external-dns": { + "finalImageTag": "v0.21.0", + "hash": "sha256-a7Z3K/H0ExAebR8meaPtVDH2/RbqxlhFNH60WfJqoaA=", + "imageDigest": "sha256:f53faaf71cb270d1ca9dce6ea0c94bfebf1a18696263487f0fbc74b9bf2bd7ff", + "imageName": "registry.k8s.io/external-dns/external-dns" + }, + "haproxy": { + "finalImageTag": "3.0-alpine", + "hash": "sha256-FKOu0U4s+ZPAW9QFEDz2WEH7HkfM3IOMjJ8yy6A4Zng=", + "imageDigest": "sha256:1c15903c50d5510dcc01de755049e7015a6dd3f35bb2dc4582e888116cb71d8f", + "imageName": "docker.io/library/haproxy" + }, + "kubectl": { + "finalImageTag": "v1.35.10", + "hash": "sha256-KoBupv9E6TsoXG3WalK6DacansbXB3pkBSkRZ/gdAkY=", + "imageDigest": "sha256:4e1d541a3d62f74637974e4c71f3cea91b9a36ffd059ab45e355648898783837", + "imageName": "registry.k8s.io/kubectl" + }, + "livenessprobe": { + "finalImageTag": "v2.18.0-20260428", + "hash": "sha256-qZqaIl3LqLd0og1DEM5KwKh5P0lMlRZyUDIhyrFJ8L8=", + "imageDigest": "sha256:eae162f7e70fb981f90d9206f299dddaf590c0c896cfb67acceca12cef526a44", + "imageName": "docker.io/longhornio/livenessprobe" + }, + "longhorn-engine": { + "finalImageTag": "v1.11.2", + "hash": "sha256-3ceS6YUy/h0W1Ofi0ZT9lafGf3MRWr6JZ1H/MqgM8t0=", + "imageDigest": "sha256:7482e0437fbf475e1e32696fab22f47bf99b1ef8d067ffce9e34028347722628", + "imageName": "docker.io/longhornio/longhorn-engine" + }, + "longhorn-instance-manager": { + "finalImageTag": "v1.11.2", + "hash": "sha256-gElWLr5Mk6ZPkUgsIDNHvuS53WXTC9Rn2YMUKdmQydI=", + "imageDigest": "sha256:16dac125ef30bd3a375bc8ff7d10636ea0302d22d208c0cfb1be37ebb93ca30b", + "imageName": "docker.io/longhornio/longhorn-instance-manager" + }, + "longhorn-manager": { + "finalImageTag": "v1.11.2", + "hash": "sha256-w2BvuR0oPN9H1fY3DYGd+5+pj0ECOmoEfuAMOnW7vjo=", + "imageDigest": "sha256:0f80ca11ac4eb7522f4e6e801a7afc9909ea8d3041575f3d029964c46590f096", + "imageName": "docker.io/longhornio/longhorn-manager" + }, + "longhorn-share-manager": { + "finalImageTag": "v1.11.2", + "hash": "sha256-1JByktvTg0MWqF4qGn74zmfX/az/GjMux0RrYxMRIE8=", + "imageDigest": "sha256:c11559e998ea982e6bac1637d66cc2aaab662a6b546709f2e54e2bfa50ffb0c3", + "imageName": "docker.io/longhornio/longhorn-share-manager" + }, + "longhorn-ui": { + "finalImageTag": "v1.11.2", + "hash": "sha256-z8/YJtkoVM2CKnoO1yoJdl+OfKQWuiw8VNbipj2ZOUs=", + "imageDigest": "sha256:885bc78f99f31da0d9b0fd8f533a53558a3aa81f9719c62e0d3c69ed8456d5b7", + "imageName": "docker.io/longhornio/longhorn-ui" + }, + "pihole": { + "finalImageTag": "2025.11.1", + "hash": "sha256-0CYHbyO4MgCwmr2vpaIurQhweK8IkiM61uiWIrJFumM=", + "imageDigest": "sha256:91dc91ddd413bf461c283204558f8f21839851e9824799075a7ceff7c77eea40", + "imageName": "docker.io/pihole/pihole" + }, + "redis": { + "finalImageTag": "8.2.3-alpine", + "hash": "sha256-uao+VHcBa3kuM46wLFHUbP71iAAEl/sg79xicQx3zp0=", + "imageDigest": "sha256:08ad0b1d280850169a790dba1393ff7a90aef951fc19632cf4d3ce4f78e679ba", + "imageName": "ecr-public.aws.com/docker/library/redis" + }, + "sops": { + "finalImageTag": "v3.13.3-alpine", + "hash": "sha256-LYnxTOLZBhdmlYFySLZLGemnNmHjI+cxLfiPtjWciOc=", + "imageDigest": "sha256:ae501277bf742f1662e0f881f43dd8fd6798b489a8058e921dbf6cda597140ea", + "imageName": "ghcr.io/getsops/sops" + }, + "speaker": { + "finalImageTag": "v0.15.3", + "hash": "sha256-t2epMeOG1rfWG3juOMfEhb9bfzksvQ0SuwOLJKB1ob4=", + "imageDigest": "sha256:c6a5b25b2e1fba610a57b2db4bb8141d7c133569d561a8cc29e38ca5113efbc4", + "imageName": "quay.io/metallb/speaker" + }, + "traefik": { + "finalImageTag": "v3.7.0", + "hash": "sha256-vrr2L/QJYNZOzAcrIBHFGbIDHC0j4wkqUXw8XQkF5/8=", + "imageDigest": "sha256:eb328e2c806c53aafbbace6c451fa54d268961261a85452fcf0fb752a30c17be", + "imageName": "docker.io/library/traefik" + }, + "whoami": { + "finalImageTag": "v1.11.0", + "hash": "sha256-qBTCdI2Hf7Rd8SCWOge0ZtJ+gK5C1Ztq/XfeU5clMew=", + "imageDigest": "sha256:200689790a0a0ea48ca45992e0450bc26ccab5307375b41c84dfc4f2475937ab", + "imageName": "docker.io/traefik/whoami" + }, + "wireguard": { + "finalImageTag": "latest", + "hash": "sha256-mBzayAz7RneKTuBVLOMCfKk7YTrgxCc5V2RjA18A4xM=", + "imageDigest": "sha256:b677c3f46d524921c8a7c91873479eb7bab190d098fddbbd887ce333ec078c90", + "imageName": "lscr.io/linuxserver/wireguard" + } + }, "vm-cluster": { "argocd": { "finalImageTag": "v3.4.1", diff --git a/tests/vm/upgrade.nix b/tests/vm/upgrade.nix new file mode 100644 index 0000000..fd96a24 --- /dev/null +++ b/tests/vm/upgrade.nix @@ -0,0 +1,95 @@ +# NixOS VM test of upgrading from the previous release: its whole render is +# applied to a cluster for real, then the current render over it, the API +# server judging (server-side dry run, CRDs for real, like Argo CD applies +# them). Catches what only breaks on upgrade: a field that can't change on an +# existing object (a selector, a StatefulSet's volume templates, a Job's +# template), a CRD version still stored but no longer served. +# +# Webhook configurations are left out: their services don't run here. +{ pkgs, mkTestLab, testLib, inputs, allLayersOff, labArgs }: + +let + previousVersion = "v1.24.2"; + previous = builtins.fetchTarball { + url = "https://github.com/thatbagu/nixlab/archive/refs/tags/${previousVersion}.tar.gz"; + sha256 = "0djsxqmih6q8dpdn1j768fzw9wkbr20ar0shlv4q7xcp934y59py"; + }; + # The previous release's code, with today's inputs (what changes between + # releases is nixlab's own render; inputs have their own update checks). + previousMkNixlab = import "${previous}/system/mk-nixlab.nix" { inherit inputs; }; + before = (previousMkNixlab labArgs).nixidyEnv.environmentPackage; + after = (mkTestLab { }).nixidyEnv.environmentPackage; + + # The cluster itself: k3s and nothing else. + cluster = mkTestLab { secretsFile = ./secrets.yaml; overrides = allLayersOff; }; + k = "k3s kubectl"; + apply = pkgs.writeShellApplication { + name = "apply-render"; + runtimeInputs = with pkgs; [ kubectl findutils gnugrep gawk ]; + text = '' + # apply-render + ENV=$1 real=$2 + mapfile -t crds < <(find -L "$ENV" -name 'CustomResourceDefinition-*.yaml' | sort) + # CRDs for real either way (Argo CD applies them first). + for f in "''${crds[@]}"; do kubectl apply --server-side --force-conflicts -f "$f" >/dev/null || { echo "CRD ''${f#"$ENV"/} rejected" >&2; exit 1; }; done + kubectl wait --for=condition=Established --timeout=120s "''${crds[@]/#/-f}" >/dev/null + mapfile -t namespaces < <(grep -RhoE '^ namespace: [a-z0-9-]+' "$ENV" | awk '{print $2}' | sort -u) + for ns in "''${namespaces[@]}"; do + kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + done + dry=(); [[ $real == yes ]] || dry=(--dry-run=server) + errors=0 count=0 + while IFS= read -r f; do + count=$((count + 1)) + if ! out=$(kubectl apply "''${dry[@]}" --server-side --force-conflicts -f "$f" 2>&1); then + echo "''${f#"$ENV"/}: $out" >&2 + errors=$((errors + 1)) + fi + done < <(find -L "$ENV" -name '*.yaml' ! -name 'CustomResourceDefinition-*' ! -name '*WebhookConfiguration-*' | sort) + echo "''${#crds[@]} CRDs, $count objects, $errors rejected" + [[ $errors -eq 0 ]] + ''; + }; +in +testLib.mkVmTest { + name = "nixlab-upgrade"; + nodeName = "tmaster"; + specialArgs = cluster.specialArgsFor "tmaster"; + + node = { + imports = cluster.modulesFor "tmaster"; + virtualisation = { + memorySize = 3072; + cores = 2; + diskSize = 4096; + emptyDiskImages = [ 8192 ]; + fileSystems."/persist" = { device = "/dev/vdb"; fsType = "ext4"; autoFormat = true; neededForBoot = true; }; + }; + system.activationScripts = { + vmTestAgeKey.text = '' + install -D -m 0400 ${./age-key.txt} /persist/etc/sops-nix/keys.txt + ''; + setupSecrets.deps = [ "vmTestAgeKey" ]; + setupSecretsForUsers.deps = [ "vmTestAgeKey" ]; + }; + services.k3s.images = [ pkgs.k3s.passthru.airgap-images ]; + }; + + subtests = [ + { + name = "k3s comes up"; + steps = [ + { waitForUnit = "k3s.service"; } + { waitUntil = "${k} get nodes | grep -w Ready"; } + ]; + } + { + name = "the previous release's render (${previousVersion}), applied for real"; + steps = [ { succeed = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml ${apply}/bin/apply-render ${before} yes"; timeout = 900; } ]; + } + { + name = "the current render applies over it"; + steps = [ { succeed = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml ${apply}/bin/apply-render ${after} no"; timeout = 900; } ]; + } + ]; +} diff --git a/tools/update.nix b/tools/update.nix index 8f9972b..43721ee 100644 --- a/tools/update.nix +++ b/tools/update.nix @@ -23,9 +23,10 @@ let # Where pinned images and charts live: the platform's, or your repo's. digestDirs = if library then "system" else "apps"; chartsDir = if library then "system/k8s/charts" else "charts"; - # Each VM test's preloaded images: those of a rendered app, plus the test's - # own fixtures (not part of the render). A chart or image update runs the - # VM tests whose apps it changes. + # Each VM test's preloaded images: those of its rendered apps (apps; left + # out: every app in its render), plus the test's own fixtures (not part of + # the render). A chart or image update runs the VM tests whose apps it + # changes. vmTests = { vm-cluster = { render = "render-default"; apps = [ "argocd" "nixlab-secrets" ]; extra = [ "docker.io/traefik/whoami:v1.11.0" ]; }; vm-monitoring = { render = "render-monitoring"; apps = [ "monitoring" "nixlab-secrets" ]; extra = [ "docker.io/mendhak/http-https-echo:43" ]; }; @@ -42,6 +43,11 @@ let vm-app-backups = { render = "render-app-backups-vm"; apps = [ "snapshot-controller" "volsync" "nixlab-secrets" "notes" ]; extra = [ ]; }; vm-single = { render = "render-single-vm"; apps = [ "traefik" "cert-manager" "cert-manager-issuers" "pihole" "externaldns-pihole" "wireguard" "nixlab-secrets" "whoami" ]; extra = [ ]; }; vm-gateway = { render = "render-gateway-vm"; apps = [ "metallb" "metallb-config" "traefik" "cert-manager" "cert-manager-issuers" "haproxy" "whoami" ]; extra = [ ]; }; + vm-bootstrap = { + render = "render-bootstrap-vm"; + extra = [ ]; + also = "(docker\\.io/)?longhornio/(longhorn-(manager|engine|instance-manager)|csi-[a-z-]+|livenessprobe):[A-Za-z0-9._-]+"; + }; }; build = check: "nix build --no-link --print-out-paths .#checks.x86_64-linux.${check}"; renders = lib.unique (map (t: t.render) (lib.attrValues vmTests)); @@ -52,17 +58,16 @@ let next=$(mktemp); echo '{}' > "$next" ${lib.concatStrings (lib.mapAttrsToList (test: t: '' env=$(${build t.render}) + ${if t ? apps then "vm_apps=(${lib.escapeShellArgs t.apps})" else ''vm_apps=(); for d in "$env"/*/; do d=''${d%/}; d=''${d##*/}; [[ $d == apps ]] || vm_apps+=("$d"); done''} wanted=$( { - # Charts' own test pods (helm test) never run in the VM tests. - ${lib.concatMapStrings (app: '' + for vm_app in "''${vm_apps[@]}"; do + # Charts' own test pods (helm test) never run in the VM tests. # Images without a registry (pihole/pihole:...) are Docker Hub's. - grep -RhoE --exclude='Pod-*-test*.yaml' '(image: |prometheus-config-reloader=)"?[a-z0-9][a-z0-9./_-]*(:[A-Za-z0-9._-]+)?(@sha256:[0-9a-f]{64})?' "$env/${app}" \ + grep -RhoE --exclude='Pod-*-test*.yaml' '(image: |prometheus-config-reloader=)"?[a-z0-9][a-z0-9./_-]*(:[A-Za-z0-9._-]+)?(@sha256:[0-9a-f]{64})?' "$env/$vm_app" \ | sed -E 's/^(image: |prometheus-config-reloader=)"?//' \ | sed -E '/^[^/]+\.[a-z]+\//! s|^|docker.io/|; s|^docker\.io/([^/]+)(:[^/]*)?$|docker.io/library/\1\2|' - '') t.apps} - ${lib.optionalString (t ? also) (lib.concatMapStrings (app: '' - grep -RhoE ${lib.escapeShellArg t.also} "$env/${app}" | sed -E 's|^longhornio/|docker.io/longhornio/|' - '') t.apps)} + ${lib.optionalString (t ? also) ''grep -RhoE ${lib.escapeShellArg t.also} "$env/$vm_app" | sed -E 's|^longhornio/|docker.io/longhornio/|' ''} + done ${lib.optionalString (t.extra != [ ]) "printf '%s\\n' ${lib.escapeShellArgs t.extra}"} } | sort -u) for ref in $wanted; do @@ -103,7 +108,9 @@ mkTool { if [[ "$kind" == input ]]; then echo "$all"; return; fi grep -vxE 'host-.*|vm-.*' <<<"$all" ${lib.concatStrings (lib.mapAttrsToList (test: t: '' - if ${if t.apps == [ ] then "false" else lib.concatMapStringsSep " || " (app: ''! diff -rq "''${before[${t.render}]}/${app}" "''${after[${t.render}]}/${app}" >/dev/null 2>&1'') t.apps}; then echo ${test}; fi + if ${if !(t ? apps) then ''! diff -rq "''${before[${t.render}]}" "''${after[${t.render}]}" >/dev/null 2>&1'' + else if t.apps == [ ] then "false" + else lib.concatMapStringsSep " || " (app: ''! diff -rq "''${before[${t.render}]}/${app}" "''${after[${t.render}]}/${app}" >/dev/null 2>&1'') t.apps}; then echo ${test}; fi '') vmTests)} # vm-cluster checks the whole default render against the API server. if ! diff -rq "''${before[render-default]}" "''${after[render-default]}" >/dev/null 2>&1; then echo vm-cluster; fi