From 863723ac8870b0f4c9d85061ccda4d4ce21cf557 Mon Sep 17 00:00:00 2001 From: thatbagu Date: Sat, 21 Nov 2026 15:16:22 +0500 Subject: [PATCH] feat(tools): nix run .#rekey, replacing the age key safely The documented steps were manual and in a risky order: push a re-encrypted secrets.yaml before every node had the new key, and their upgrades (and the in-cluster secrets job) can't decrypt it. rekey does it in two steps: the new key joins the old one on the workstation, every node and the cluster's copy, and secrets.yaml is encrypted for it (checked to open with the new key alone); --finish, refused until that's committed and pushed, removes the old key everywhere and sets the workstation's copy aside. Co-Authored-By: Claude Opus 5.5 --- docs/secrets.md | 17 +++--- tests/default.nix | 2 + tests/rekey.nix | 48 +++++++++++++++++ tools/default.nix | 1 + tools/rekey.nix | 133 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 190 insertions(+), 11 deletions(-) create mode 100644 tests/rekey.nix create mode 100644 tools/rekey.nix diff --git a/docs/secrets.md b/docs/secrets.md index dc143a3..a6af169 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -125,17 +125,12 @@ Decrypted values exist only in memory and in the Secrets themselves: ## Re-keying (replacing the age key) -If you need to rotate the age key itself: - ```bash -# Generate a new key -age-keygen -o ~/.config/sops/age/keys.txt.new - -# Update .sops.yaml with the new public key, then re-encrypt -sops updatekeys secrets.yaml - -# Remove the old key -mv ~/.config/sops/age/keys.txt.new ~/.config/sops/age/keys.txt +nix run .#rekey # a new key joins the old one everywhere; secrets.yaml is encrypted for it +git commit .sops.yaml secrets.yaml && git push +nix run .#rekey -- --finish # once pushed: the old key goes, everywhere ``` -Update `/persist/etc/sops-nix/keys.txt` on each node with the new private key, then redeploy. +The first step adds the new key on your workstation, on every node, and in the cluster's copy, then re-encrypts `secrets.yaml` for it and checks that the new key alone opens it. Both keys work until `--finish`, so the nodes can still build what's pushed, whichever version that is. `--finish` refuses until that's committed and pushed. Your old key is set aside in `.retired-`; delete it once you're sure. Back up the new key offline. + +Every node must be reachable, since each one gets the key over SSH. A node that's down: run the step again once it's back. diff --git a/tests/default.nix b/tests/default.nix index e2a5d3e..9dafcbb 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -937,6 +937,8 @@ lib.mapAttrs' (name: lab: lib.nameValuePair "render-${name}" lab.nixidyEnv.envir { desc = "it syncs the rendered branch"; succeed = "jq -e '.operation.sync.revision == \"k8s-rendered\"' op.json"; } ]; }; + # Replacing the age key (two steps; the old one goes only once pushed). + tool-rekey = import ./rekey.nix { inherit pkgs testLib; tools = { inherit (tools) init; inherit ((mkTools { vars = baseVars; })) rekey; }; }; tool-install = import ./install.nix { inherit pkgs testLib; install = (mkTools { vars = baseVars; }).install; }; # The template's example names nodes apart from their hostnames, and a # node may get encrypt = true after init ran: the tools take either name, diff --git a/tests/rekey.nix b/tests/rekey.nix new file mode 100644 index 0000000..f543a69 --- /dev/null +++ b/tests/rekey.nix @@ -0,0 +1,48 @@ +# nix run .#rekey, for real (age, sops, git) in the build sandbox, with an +# ssh stub standing in for the nodes (each one's /persist is a directory +# here): both keys work in between, the old one goes only once pushed. +{ pkgs, testLib, tools }: + +let + K = "$HOME/.config/sops/age/keys.txt"; + ssh = pkgs.writeShellScriptBin "fake-ssh" '' + host=''${*: -2:1}; host=''${host#*@}; cmd=''${*: -1} + cmd=''${cmd#sudo }; cmd=''${cmd//\/persist/$NODES/$host/persist} + exec bash -c "$cmd" + ''; + systemctl = pkgs.writeShellScriptBin "systemctl" ''echo "$*" >> "$NODES/systemctl.log"''; + keysIn = f: "grep -c AGE-SECRET-KEY ${f}"; +in +testLib.mkSandboxTest { + name = "rekey"; + runtimeInputs = [ tools.init tools.rekey ssh systemctl ] ++ (with pkgs; [ sops git gnugrep ]); + setup = '' + cp -r ${../template} repo && chmod -R u+w repo && cd repo + printf 'vpn.example.org\npw-test-1\npw-test-1\ncf-token\nme@example.org\n' | nixlab-init >/dev/null + export NIXLAB_SSH=fake-ssh NODES=$PWD/../nodes + for n in tmaster.local tworker.local; do install -D -m 400 ${K} $NODES/$n/persist/etc/sops-nix/keys.txt; done + cp ${K} ../old.key + git init -q && git config user.email t@t && git config user.name t && git add -A && git commit -qm init + git init -q --bare ../remote.git && git remote add origin ../remote.git && git push -q origin HEAD:main + ''; + steps = [ + { desc = "dry run"; succeed = "nixlab-rekey --dry-run | grep -c 'Would create a new age key' >/dev/null"; } + { desc = "rekey runs"; succeed = "nixlab-rekey >out 2>&1 || { cat out >&2; false; }"; } + { desc = "both keys on the workstation"; succeed = "[ $(${keysIn K}) = 2 ]"; } + { desc = "both keys on every node"; succeed = "for n in tmaster.local tworker.local; do [ $(${keysIn "$NODES/$n/persist/etc/sops-nix/keys.txt"}) = 2 ] || exit 1; done"; } + { desc = "the cluster's copy refreshed, from the master only"; + succeed = "[ \"$(cat $NODES/systemctl.log)\" = 'restart nixlab-age-key' ]"; } + { desc = "secrets.yaml opens with both keys on file (the nodes' view)"; succeed = "SOPS_AGE_KEY_FILE=$NODES/tworker.local/persist/etc/sops-nix/keys.txt sops -d secrets.yaml >/dev/null"; } + # (HOME: sops would also try ~/.config/sops/age/keys.txt.) + { desc = "but not with the old key alone"; fail = "HOME=/nonexistent XDG_CONFIG_HOME=/nonexistent SOPS_AGE_KEY_FILE=../old.key sops -d secrets.yaml >/dev/null 2>&1"; } + { desc = "a second rekey before --finish is refused"; fail = "nixlab-rekey >/dev/null 2>&1"; } + { desc = "--finish refuses before the commit"; succeed = "! nixlab-rekey --finish 2>err && grep -c 'Commit and push' err >/dev/null"; } + { desc = "--finish refuses before the push"; + succeed = "git commit -qam rekey && ! nixlab-rekey --finish 2>err && grep -c 'Push first' err >/dev/null"; } + { desc = "--finish, once pushed"; succeed = "git push -q origin HEAD:main && git fetch -q origin && nixlab-rekey --finish >/dev/null"; } + { desc = "one key left everywhere"; succeed = "[ $(${keysIn K}) = 1 ] && for n in tmaster.local tworker.local; do [ $(${keysIn "$NODES/$n/persist/etc/sops-nix/keys.txt"}) = 1 ] || exit 1; done"; } + { desc = "and it opens secrets.yaml"; succeed = "sops -d secrets.yaml >/dev/null && SOPS_AGE_KEY_FILE=$NODES/tmaster.local/persist/etc/sops-nix/keys.txt sops -d secrets.yaml >/dev/null"; } + { desc = "the old key is set aside, not lost"; succeed = "diff <(grep AGE-SECRET-KEY ../old.key) <(grep AGE-SECRET-KEY ${K}.retired-*)"; } + { desc = "--finish again: nothing to do"; succeed = "nixlab-rekey --finish | grep -c 'nothing to finish' >/dev/null"; } + ]; +} diff --git a/tools/default.nix b/tools/default.nix index fc8da99..71fd2ac 100644 --- a/tools/default.nix +++ b/tools/default.nix @@ -50,6 +50,7 @@ if library then { update = import ./update.nix args; wake = import ./wake.nix args; unlock = import ./unlock.nix args; + rekey = import ./rekey.nix args; render = mkTool { name = "render"; runtimeInputs = [ nixidyCli ]; diff --git a/tools/rekey.nix b/tools/rekey.nix new file mode 100644 index 0000000..c550c68 --- /dev/null +++ b/tools/rekey.nix @@ -0,0 +1,133 @@ +# nix run .#rekey [--dry-run]: replace the age key that encrypts secrets.yaml, +# in two steps so nothing is ever unable to decrypt: +# 1. rekey: a new key joins the old one on your workstation and every node +# (and the cluster's copy), and secrets.yaml is encrypted for the new +# key. You commit and push. +# 2. rekey --finish: once that's pushed, the old key goes everywhere. Your +# workstation's copy is set aside in .retired-. +{ pkgs, lib, vars, mkTool, secretsPath, ... }: + +let + addressOf = n: n.address or "${n.hostname}.local"; + nodes = lib.attrValues vars.nodes; + nodeKeyFile = "/persist/etc/sops-nix/keys.txt"; +in +mkTool { + name = "rekey"; + runtimeInputs = with pkgs; [ age sops openssh gnused gnugrep coreutils git ]; + text = '' + finish=0 dry=0 + for a in "$@"; do + case "$a" in + --finish) finish=1 ;; + --dry-run) dry=1 ;; + *) echo "usage: nix run .#rekey [-- --finish] [--dry-run]" >&2; exit 2 ;; + esac + done + SECRETS=${secretsPath} + AGE_KEY=''${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt} + SSH=''${NIXLAB_SSH:-ssh} + user=${lib.escapeShellArg vars.username} + addresses=(${lib.concatMapStringsSep " " (n: lib.escapeShellArg (addressOf n)) nodes}) + masters=(${lib.concatMapStringsSep " " (n: lib.escapeShellArg (addressOf n)) (lib.filter (n: n.master or false) nodes)}) + [[ -r "$AGE_KEY" ]] || { echo "Can't read your age key ($AGE_KEY); point SOPS_AGE_KEY_FILE at it." >&2; exit 1; } + [[ -f .sops.yaml && -f "$SECRETS" ]] || { echo "Run this in your nixlab repository (.sops.yaml, $SECRETS)." >&2; exit 1; } + + tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT; chmod 700 "$tmp" + # decrypts_with : secrets.yaml opens with just that key (sops + # also tries ~/.config/sops/age/keys.txt: hidden here). + mkdir "$tmp/nohome" + decrypts_with() { env -u SOPS_AGE_KEY HOME="$tmp/nohome" XDG_CONFIG_HOME="$tmp/nohome" SOPS_AGE_KEY_FILE="$1" sops -d "$SECRETS" >/dev/null 2>&1; } + # One identity per file in $tmp/id.N, with its public key in $tmp/pub.N. + split_keys() { + local n=0 + while IFS= read -r line; do + [[ $line == AGE-SECRET-KEY-* ]] || continue + printf '%s\n' "$line" > "$tmp/id.$n" + age-keygen -y "$tmp/id.$n" > "$tmp/pub.$n" + n=$((n + 1)) + done < "$1" + echo "$n" + } + # on
: as root on a node; stdin passes through. + on() { "$SSH" -o BatchMode=yes -o ConnectTimeout=10 "$user@$1" "sudo bash -c '$2'"; } + refresh_cluster() { + for m in "''${masters[@]}"; do + on "$m" "systemctl restart nixlab-age-key" && echo " the cluster's copy (from $m)" + done + } + + recipient=$(grep -oE 'age1[0-9a-z]+' .sops.yaml | head -1) + [[ -n $recipient ]] || { echo ".sops.yaml has no age recipient." >&2; exit 1; } + + if [[ $finish == 0 ]]; then + count=$(split_keys "$AGE_KEY") + [[ $count == 1 ]] || { echo "$AGE_KEY holds $count keys: finish the last rekey first (nix run .#rekey -- --finish)." >&2; exit 1; } + [[ "$(cat "$tmp/pub.0")" == "$recipient" ]] || { echo "Your key isn't the one in .sops.yaml ($recipient)." >&2; exit 1; } + decrypts_with "$AGE_KEY" || { echo "Your key doesn't decrypt $SECRETS." >&2; exit 1; } + if [[ $dry == 1 ]]; then + echo "Would create a new age key and add it (next to the old one) to $AGE_KEY and to ${nodeKeyFile} on: ''${addresses[*]}" + echo "then refresh the cluster's copy, encrypt $SECRETS for the new key only, and check the new key alone decrypts it." + exit 0 + fi + age-keygen -o "$tmp/new" 2>/dev/null + new_pub=$(age-keygen -y "$tmp/new") + echo "==> New key: $new_pub" + echo "==> Adding it next to the old one" + for a in "''${addresses[@]}"; do + # Into a new file, then swapped in (read-only, like install leaves it). + # Twice-run safe: added only if missing. + on "$a" "grep -q $new_pub ${nodeKeyFile} || { umask 077; { cat ${nodeKeyFile}; echo; cat; } > ${nodeKeyFile}.new && chmod 400 ${nodeKeyFile}.new && mv ${nodeKeyFile}.new ${nodeKeyFile}; }" < "$tmp/new" \ + || { echo "Couldn't add it on $a; nothing is encrypted for it yet, and secrets.yaml is unchanged. Run rekey again once $a is reachable." >&2; exit 1; } + echo " $a" + done + { echo; cat "$tmp/new"; } >> "$AGE_KEY"; echo " $AGE_KEY" + refresh_cluster + echo "==> Encrypting $SECRETS for the new key" + sed -i "s/$recipient/$new_pub/" .sops.yaml + SOPS_AGE_KEY_FILE="$AGE_KEY" sops updatekeys -y "$SECRETS" >/dev/null 2>&1 + decrypts_with "$tmp/new" || { echo "The new key doesn't decrypt $SECRETS; .sops.yaml and $SECRETS left as they are now, old key still everywhere." >&2; exit 1; } + ! decrypts_with "$tmp/id.0" || { echo "$SECRETS still opens with the old key." >&2; exit 1; } + cat <&2; exit 1; } + [[ $count -gt 1 ]] || { echo "Only the current key is left; nothing to finish."; exit 0; } + decrypts_with "$tmp/id.$keep" || { echo "$SECRETS doesn't open with the new key alone; not removing anything." >&2; exit 1; } + if git rev-parse --git-dir >/dev/null 2>&1; then + git diff --quiet HEAD -- .sops.yaml "$SECRETS" \ + || { echo "Commit and push .sops.yaml and $SECRETS first: the nodes build what's pushed." >&2; exit 1; } + [[ -z "$(git branch -r --contains HEAD 2>/dev/null)" ]] \ + && { echo "Push first: the nodes build what's pushed, and need the old key until then." >&2; exit 1; } + fi + if [[ $dry == 1 ]]; then + echo "Would leave only $recipient in ${nodeKeyFile} on: ''${addresses[*]}, refresh the cluster's copy, and set the old key(s) aside from $AGE_KEY." + exit 0 + fi + echo "==> Removing the old key" + for a in "''${addresses[@]}"; do + on "$a" "umask 077; cat > ${nodeKeyFile}.new && chmod 400 ${nodeKeyFile}.new && mv ${nodeKeyFile}.new ${nodeKeyFile}" < "$tmp/id.$keep" \ + || { echo "Couldn't on $a; run --finish again once it's reachable." >&2; exit 1; } + echo " $a" + done + refresh_cluster + retired="$AGE_KEY.retired-$(date +%Y%m%d)" + for ((i = 0; i < count; i++)); do [[ $i == "$keep" ]] || cat "$tmp/id.$i"; done >> "$retired" + chmod 600 "$retired" + { echo "# public key: $recipient"; cat "$tmp/id.$keep"; } > "$AGE_KEY.new" && chmod 600 "$AGE_KEY.new" && mv "$AGE_KEY.new" "$AGE_KEY" + echo " $AGE_KEY (the old key is in $retired: delete it once you're sure)" + echo "Done: only $recipient is left." + ''; +}