From 09f9317389e98a177631de50e8bb5063c2fb6f98 Mon Sep 17 00:00:00 2001 From: thatbagu Date: Sat, 21 Nov 2026 00:30:16 +0500 Subject: [PATCH] fix(k8s): the first Longhorn sync creates its namespace and retries The bootstrap started Longhorn's first sync with a bare operation, which doesn't inherit the app's sync options or retry: on a fresh cluster it failed (namespaces "longhorn-system" not found), for good, and every later sync wave waited on it. Found by an unattended --new-cluster rebuild of a live cluster on v1.24.0 (the earlier run's "app project not found" error hid it). The operation now carries the app's syncOptions and retry. Co-Authored-By: Claude Opus 5.5 --- system/k8s/default.nix | 12 +++++++++--- tests/default.nix | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/system/k8s/default.nix b/system/k8s/default.nix index b96c66a..0068347 100644 --- a/system/k8s/default.nix +++ b/system/k8s/default.nix @@ -118,7 +118,7 @@ in { after = [ "k3s.service" "nixlab-age-key.service" ]; wants = [ "k3s.service" "nixlab-age-key.service" ]; wantedBy = [ "multi-user.target" ]; - path = [ pkgs.kubectl ]; + path = [ pkgs.kubectl pkgs.jq ]; environment.KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; serviceConfig = { Type = "oneshot"; @@ -167,8 +167,14 @@ in { kubectl -n argocd get appprojects.argoproj.io default >/dev/null 2>&1 && break sleep 5 done - kubectl -n argocd patch applications.argoproj.io longhorn --type merge \ - -p '{"operation":{"initiatedBy":{"username":"nixlab-bootstrap"},"sync":{"revision":"${k8sVars.renderedBranch}"}}}' + # An operation doesn't inherit the app's sync options or retry: + # without them the namespace isn't created ("namespaces + # \"longhorn-system\" not found") and a failure is final. + kubectl -n argocd get applications.argoproj.io longhorn -o json \ + | jq -c '{operation: {initiatedBy: {username: "nixlab-bootstrap"}, + sync: {revision: "${k8sVars.renderedBranch}", syncOptions: (.spec.syncPolicy.syncOptions // [])}, + retry: (.spec.syncPolicy.retry // {})}}' \ + | kubectl -n argocd patch applications.argoproj.io longhorn --type merge --patch-file /dev/stdin ''} echo "Done. Watch it converge: kubectl -n argocd get applications" ''; diff --git a/tests/default.nix b/tests/default.nix index 6ad5045..e2a5d3e 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -918,6 +918,25 @@ lib.mapAttrs' (name: lab: lib.nameValuePair "render-${name}" lab.nixidyEnv.envir { desc = "a pod without a controller stays"; fail = "grep -q bare deleted"; } ]; }; + # The bootstrap's first Longhorn sync, its jq program run on the rendered + # Application: the operation must carry the app's sync options (the + # namespace gets created) and its retry (a failure isn't final). + bootstrap-longhorn-sync = let + script = (master labs.default).systemd.services.argocd-bootstrap.script; + program = lib.head (lib.splitString "' \\" (lib.last (lib.splitString "jq -c '" script))); + in testLib.mkSandboxTest { + name = "bootstrap-longhorn-sync"; + runtimeInputs = with pkgs; [ jq yq-go ]; + setup = '' + yq -o json ${labs.default.nixidyEnv.environmentPackage}/apps/Application-longhorn.yaml > app.json + jq -c ${lib.escapeShellArg program} app.json > op.json + ''; + steps = [ + { desc = "it creates the namespace"; succeed = "jq -e '.operation.sync.syncOptions | index(\"CreateNamespace=true\")' op.json"; } + { desc = "it retries without giving up"; succeed = "jq -e '.operation.retry.limit == -1' op.json"; } + { desc = "it syncs the rendered branch"; succeed = "jq -e '.operation.sync.revision == \"k8s-rendered\"' op.json"; } + ]; + }; tool-install = import ./install.nix { inherit pkgs testLib; install = (mkTools { vars = baseVars; }).install; }; # The template's example names nodes apart from their hostnames, and a # node may get encrypt = true after init ran: the tools take either name,