Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
strategy:
fail-fast: false
matrix:
group: [quick, quick-eval, quick-render, hosts, vm-cluster, vm-minimal, vm-sso, vm-gateway, vm-ha, vm-longhorn, vm-install, vm-app-backups, vm-single, vm-install-encrypted, vm-nas, vm-power, vm-tailscale, vm-monitoring, vm-monitoring-full]
group: [quick, quick-eval, quick-render, hosts, vm-cluster, vm-minimal, vm-sso, vm-gateway, vm-bootstrap, vm-upgrade, vm-ha, vm-longhorn, vm-install, vm-app-backups, vm-single, vm-install-encrypted, vm-nas, vm-power, vm-tailscale, vm-monitoring, vm-monitoring-full]
name: checks (${{ matrix.group }})
steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -61,7 +61,7 @@ jobs:
# everything at once: together they exhaust a runner's memory.
filter='n: let
g = builtins.getEnv "GROUP";
vmsAndHosts = [ "host-master" "host-worker" "vm-cluster" "vm-minimal" "vm-sso" "vm-gateway" "vm-ha" "vm-longhorn" "vm-install" "vm-app-backups" "vm-single" "vm-install-encrypted" "vm-nas" "vm-power" "vm-tailscale" "vm-monitoring" "vm-monitoring-full"];
vmsAndHosts = [ "host-master" "host-worker" "vm-cluster" "vm-minimal" "vm-sso" "vm-gateway" "vm-bootstrap" "vm-upgrade" "vm-ha" "vm-longhorn" "vm-install" "vm-app-backups" "vm-single" "vm-install-encrypted" "vm-nas" "vm-power" "vm-tailscale" "vm-monitoring" "vm-monitoring-full"];
prefix = p: builtins.substring 0 (builtins.stringLength p) n == p;
in
if g == "quick" then !(builtins.elem n vmsAndHosts) && !(prefix "eval-") && !(prefix "render-")
Expand Down
4 changes: 4 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ chore: bump pihole to 2025.11.1
Subject line: imperative mood, no period, 72 characters max.
Body: explain the *why*, not the *what*, when it's not obvious.

## Releasing

After tagging a release, point the upgrade test at it, so the next release is checked against this one: in `tests/vm/upgrade.nix`, set `previousVersion` to the new tag and `sha256` to the output of `nix-prefetch-url --unpack https://github.com/thatbagu/nixlab/archive/refs/tags/<tag>.tar.gz`.

## License

MIT. By contributing you agree your changes are released under the same license.
29 changes: 28 additions & 1 deletion tests/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,16 @@ let

# overrides merge into baseVars; replace swaps whole top-level settings
# (e.g. nodes, to drop one).
mkTestLab = { overrides ? { }, replace ? { }, extraAppModules ? [ ], extraNixosModules ? [ ], secretsFile ? testSecrets, extraSecrets ? [ ] }: mkNixlab {
# (testLabArgs: what it passes to mkNixlab, for another version's mkNixlab.)
testLabArgs = { overrides ? { }, replace ? { }, extraAppModules ? [ ], extraNixosModules ? [ ], secretsFile ? testSecrets, extraSecrets ? [ ] }: {
vars = lib.recursiveUpdate baseVars overrides // replace;
inherit secretsFile;
# The unit tests' secrets are fake and plaintext; the VM tests' are encrypted.
allowUnencryptedSecrets = secretsFile == testSecrets;
hardwareConfig = _: ../template/hosts/example/hardware-configuration.nix;
inherit extraAppModules extraNixosModules extraSecrets;
};
mkTestLab = args: mkNixlab (testLabArgs args);

# One machine with the whole platform (profile = "single").
singleNode = {
Expand Down Expand Up @@ -143,6 +145,26 @@ let
backups = { enable = true; target = "nfs://192.168.1.1:/export/backups"; };
};

# The fresh-bootstrap VM test: Argo CD syncs every layer from a git
# repository the machine serves itself (the rendered branch; the test
# network numbers its one machine .1). No internet: the internal CA, no
# Nextcloud.
bootstrapVm = {
overrides = allLayersOff // {
gitops.enable = true;
metallb.enable = true;
ingress.enable = true;
edge.enable = true;
certManager.enable = true;
cloudflare.enable = false;
longhorn.enable = true;
pihole.enable = true;
vpn.enable = true;
gitRepository = "http://192.168.1.1:8000/rendered.git";
};
extraAppModules = [ gatewayVmApp ];
};

# The per-app backup VM test: k3s' local storage (VolSync reads the volume
# itself), MinIO on another machine (the test network numbers machines in
# name order: s3 .1, tmaster .2) as the S3 target, and an app with a volume.
Expand Down Expand Up @@ -204,6 +226,7 @@ let
longhorn-vm.overrides = longhornVmOverrides;
# The gateway VM test's setup.
gateway-vm = { overrides = gatewayVmOverrides; extraAppModules = [ gatewayVmApp ]; };
bootstrap-vm = bootstrapVm;
# Every layer off: k3s on NixOS and nothing else.
minimal.overrides = allLayersOff;
# Everything on one machine (profile = "single").
Expand Down Expand Up @@ -1041,6 +1064,10 @@ lib.mapAttrs' (name: lab: lib.nameValuePair "render-${name}" lab.nixidyEnv.envir
# NixOS VM test: HTTPS through the gateway (Traefik, Gateway API), with a
# cert-manager certificate, behind the edge proxy (tests/vm/gateway.nix).
vm-gateway = import ./vm/gateway.nix { inherit pkgs mkTestLab testLib; overrides = gatewayVmOverrides; extraAppModules = [ gatewayVmApp ]; };
# A fresh cluster bootstrapping every layer through Argo CD, from git.
vm-bootstrap = import ./vm/bootstrap.nix { inherit pkgs mkTestLab testLib; inherit (bootstrapVm) overrides extraAppModules; };
# Upgrading from the previous release: its render, then the current one over it.
vm-upgrade = import ./vm/upgrade.nix { inherit pkgs mkTestLab testLib inputs allLayersOff; labArgs = testLabArgs { }; };
# NixOS VM test: monitoring on the real master, alerts delivered to a
# stand-in webhook (tests/vm/monitoring.nix).
# NixOS VM test: the UPS shuts the cluster down in order (tests/vm/power.nix).
Expand Down
110 changes: 110 additions & 0 deletions tests/vm/bootstrap.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
# NixOS VM test of a fresh cluster bootstrapping by itself, the way a real
# one does: Argo CD syncs every layer (MetalLB, the gateway, cert-manager with
# the internal CA, Longhorn, Pi-hole, the VPN, a web app) from the rendered
# branch, which the machine serves itself as a git repository. Checks that,
# with nobody touching anything:
# - every Application ends up Synced and Healthy (sync waves, Longhorn's
# one-time manual sync, a gateway whose certificates come later, Secrets
# for namespaces another app creates)
# - the web app answers HTTPS through the edge proxy and the gateway
#
# The VM has no internet: images are preloaded (images.nix), and the served
# branch drops their registry digests (preloaded images keep only the tag).
{ pkgs, mkTestLab, testLib, overrides, extraAppModules }:

let
images = (import ./images.nix { inherit pkgs; }).vm-bootstrap;
lab = mkTestLab { secretsFile = ./secrets.yaml; inherit overrides extraAppModules; };
env = lab.nixidyEnv.environmentPackage;
k = "k3s kubectl";
apps = "${k} -n argocd get applications -o custom-columns=A:.metadata.name,S:.status.sync.status,H:.status.health.status --no-headers";
in
testLib.mkVmTest {
name = "nixlab-bootstrap";
nodeName = "tmaster";
specialArgs = lab.specialArgsFor "tmaster";

node = {
imports = lab.modulesFor "tmaster";
virtualisation = {
memorySize = 8192;
cores = 6;
diskSize = 8192;
emptyDiskImages = [ 24576 ];
fileSystems."/persist" = { device = "/dev/vdb"; fsType = "ext4"; autoFormat = true; neededForBoot = true; };
};
system.activationScripts = {
vmTestAgeKey.text = ''
install -D -m 0400 ${./age-key.txt} /persist/etc/sops-nix/keys.txt
'';
setupSecrets.deps = [ "vmTestAgeKey" ];
setupSecretsForUsers.deps = [ "vmTestAgeKey" ];
};
networking.firewall.allowedTCPPorts = [ 7946 ];
networking.firewall.allowedUDPPorts = [ 7946 ];
services.k3s.images = [ pkgs.k3s.passthru.airgap-images ] ++ images;
environment.systemPackages = [ pkgs.curl ];

# The rendered branch, as CI pushes it, over git's smart HTTP protocol
# (Argo CD's go-git doesn't speak the plain one; gitRepository in the lab
# points here).
systemd.services.rendered-repo = {
wantedBy = [ "multi-user.target" ];
before = [ "argocd-bootstrap.service" ];
path = with pkgs; [ git gnused python3 ];
script = ''
rm -rf /run/rendered /srv/git; mkdir -p /run/rendered /srv/git
cp -rL ${env}/. /run/rendered/ && chmod -R u+w /run/rendered
find /run/rendered -name '*.yaml' -exec sed -i -E 's/@sha256:[0-9a-f]{64}//' {} +
cd /run/rendered
git init -q -b k8s-rendered && git add -A
git -c user.email=ci@test -c user.name=ci commit -qm render
git clone -q --bare /run/rendered /srv/git/rendered.git
exec python3 ${./git-http.py} /srv/git 8000
'';
};
};

subtests = [
{
name = "the cluster comes up and Argo CD bootstraps it";
steps = [
{ waitForUnit = "k3s.service"; }
# k3s imports every preloaded image before its kubelet starts: minutes.
{ waitUntil = "${k} get nodes | grep -w Ready"; timeout = 1800; }
{ waitForUnit = "rendered-repo.service"; }
{ waitUntil = "curl -sf http://192.168.1.1:8000/rendered.git/info/refs | grep -c k8s-rendered"; }
{ waitForUnit = "argocd-bootstrap.service"; timeout = 600; }
];
}
{
name = "every app ends up Synced and Healthy, by itself";
steps = [
{ desc = "all of them exist";
waitUntil = "[ $(${apps} | wc -l) -ge ${toString (builtins.length (builtins.attrNames lab.nixidyEnv.config.applications) - 1)} ]"; timeout = 1800; }
{ desc = "and all of them are Synced and Healthy";
waitUntil = "${apps} >&2; [ -z \"$(${apps} | grep -v 'Synced *Healthy')\" ]"; timeout = 2400; }
];
}
{
name = "the web app answers through the edge proxy and the gateway";
steps = [
{ waitUntil = "curl -skf --resolve whoami.yourdomain.example.com:443:192.168.1.193 https://whoami.yourdomain.example.com/ | grep -c Hostname"; timeout = 600; }
];
}
{
name = "Longhorn's first sync went through without help";
steps = [
{ succeed = "[ \"$(${k} -n argocd get application longhorn -o jsonpath='{.status.operationState.phase}')\" = Succeeded ]"; }
{ succeed = "${k} -n longhorn-system get nodes.longhorn.io tmaster"; }
];
}
];

onFailure = [
"${apps}"
"${k} -n argocd get applications -o json | ${pkgs.jq}/bin/jq -r '.items[] | select(.status.sync.status != \"Synced\" or .status.health.status != \"Healthy\") | \"\\(.metadata.name): \\(.status.operationState.phase // \"-\") \\(.status.operationState.message // \"\" | .[0:300])\"'"
"${k} get pods -A | grep -vE 'Running|Completed'"
"journalctl -u argocd-bootstrap --no-pager | tail -30"
];
}
29 changes: 29 additions & 0 deletions tests/vm/git-http.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# git's smart HTTP protocol (what Argo CD's go-git speaks), via git http-backend.
import http.server, os, subprocess, sys

root = sys.argv[1]

class Handler(http.server.BaseHTTPRequestHandler):
def run(self):
path, _, query = self.path.partition("?")
length = int(self.headers.get("Content-Length") or 0)
body = self.rfile.read(length) if length else b""
env = dict(os.environ, GIT_PROJECT_ROOT=root, GIT_HTTP_EXPORT_ALL="1",
PATH_INFO=path, QUERY_STRING=query, REQUEST_METHOD=self.command,
CONTENT_TYPE=self.headers.get("Content-Type", ""), CONTENT_LENGTH=str(len(body)),
REMOTE_ADDR=self.client_address[0])
out = subprocess.run(["git", "http-backend"], input=body, env=env, capture_output=True).stdout
head, _, rest = out.partition(b"\r\n\r\n")
status, headers = 200, []
for line in head.decode().split("\r\n"):
k, _, v = line.partition(": ")
if k.lower() == "status": status = int(v.split()[0])
elif k: headers.append((k, v))
self.send_response(status)
for k, v in headers: self.send_header(k, v)
self.send_header("Content-Length", str(len(rest)))
self.end_headers()
self.wfile.write(rest)
do_GET = do_POST = run

http.server.ThreadingHTTPServer(("0.0.0.0", int(sys.argv[2])), Handler).serve_forever()
Loading
Loading