From c88f8788a4c3c3b03c958fba740e81721f629cd3 Mon Sep 17 00:00:00 2001 From: Alper Gundogdu Date: Thu, 24 Sep 2026 13:26:25 +0100 Subject: [PATCH 1/2] fix(roles): daemon-reload inside the restart handlers "Install decdn-node systemd unit" notifies "Reload systemd" and then "Restart decdn-node", relying on the reload running first. On site.yml it does not: baseline loads devsec.hardening.os_hardening with include_role, which appends that role's own "Reload systemd" handler to the play at run time. Being loaded last, it shadows the role's handler of the same name and runs after the restart, so systemd restarts the node from its cached copy of the unit it just rewrote. Seen on a real fleet deploy: WatchdogSec=60 on disk, NeedDaemonReload=no afterwards, and the running node had no watchdog until a manual restart. "Restart decdn-node" and "Restart alloy" now set daemon_reload: true, so handler order no longer matters. The default scenario reproduces it: prepare.yml leaves a stale unit loaded and running, converge.yml include_role's a test role with a same-named "Reload systemd" handler and arms the watchdog, and verify.yml asserts the running process is the stub with WATCHDOG_USEC set. The stub heartbeats WATCHDOG=1 like the real daemon. Without the fix, verify finds /bin/sleep from the stale unit. Co-Authored-By: Claude Opus 5.5 --- ansible/galaxy/CHANGELOG.md | 8 +++++ ansible/molecule/default/converge.yml | 11 ++++++ .../molecule/default/files/decdn-node-stub | 29 +++++++++++++++ ansible/molecule/default/molecule.yml | 3 +- ansible/molecule/default/prepare.yml | 19 ++++++++++ .../shadow_reload_systemd/handlers/main.yml | 10 ++++++ ansible/molecule/default/verify.yml | 35 +++++++++++++++++++ ansible/molecule/os-matrix/molecule.yml | 3 +- ansible/roles/decdn_node/handlers/main.yml | 5 +++ ansible/roles/grafana_alloy/handlers/main.yml | 5 +++ 10 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 ansible/molecule/default/roles/shadow_reload_systemd/handlers/main.yml diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index ed81c55..c2bf647 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -78,6 +78,14 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- A changed `decdn-node` or `alloy` unit now takes effect on the converge that + writes it. The restart handlers relied on a separate `Reload systemd` + handler running first, but `devsec.hardening.os_hardening` (loaded by + `baseline` through `include_role`) defines a handler of the same name. Being + loaded last, it shadows the roles' own and runs after the restarts, so + systemd restarted the service from its cached unit: a new `WatchdogSec` was + on disk while the running node had no watchdog until the next restart. The + restart handlers now `daemon_reload` themselves. - The OTLP gateway is authenticated with its own instance ID. The role reused the Prometheus one, so a Grafana Cloud org whose stack ID differs 401s every trace while metrics and logs keep flowing. New `grafana_alloy_otlp_username` / diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml index 568f194..24f1538 100644 --- a/ansible/molecule/default/converge.yml +++ b/ansible/molecule/default/converge.yml @@ -33,6 +33,10 @@ # Makes the stub compare the environment received by `config validate` # against the exact literal above (not merely assert that no command ran). DECDN_MOLECULE_ASSERT_RPC_LITERAL: "1" + # Arms the unit's watchdog (the stub heartbeats), so verify.yml can prove the + # running process was started from the unit this converge wrote: prepare.yml + # leaves a stale unit loaded that has no WatchdogSec. + decdn_node_watchdog_sec: 60 decdn_chain_id: 421614 decdn_region: "US" decdn_payment_pool_address: "0x1111111111111111111111111111111111111111" @@ -124,6 +128,13 @@ ansible.builtin.include_role: name: baseline tasks_from: sudo_users + # Reproduces site.yml's handler order: baseline include_role's + # devsec.hardening.os_hardening, whose same-named "Reload systemd" handler is + # loaded last, shadows the roles' own and runs after "Restart decdn-node". + # verify.yml asserts the node still runs on the unit this converge wrote. + - name: Load a same-named "Reload systemd" handler, as devsec.hardening does + ansible.builtin.include_role: + name: shadow_reload_systemd # molecule/default/roles, on ANSIBLE_ROLES_PATH roles: - role: grafana_alloy tags: [observability, decdn_node, node] diff --git a/ansible/molecule/default/files/decdn-node-stub b/ansible/molecule/default/files/decdn-node-stub index a8b7344..a944f97 100755 --- a/ansible/molecule/default/files/decdn-node-stub +++ b/ansible/molecule/default/files/decdn-node-stub @@ -10,6 +10,9 @@ can be exercised end-to-end without a published release or a live chain: the role's `/metrics` readiness probe succeeds, and block forever under systemd (Type=simple). CLI args (`--config`, `--keystore-password-file`) are accepted and ignored. + Under a unit with WatchdogSec it heartbeats WATCHDOG=1 to + NOTIFY_SOCKET every WATCHDOG_USEC/2, as the real daemon + does, so systemd does not kill it at the deadline. If the marker file NO_METRICS_MARKER exists, it instead blocks forever WITHOUT binding metrics — standing in for a healthy node whose metrics listener binds late (behind the @@ -36,8 +39,10 @@ It intentionally implements no protocol behaviour — the scenario verifies host prep, config/unit rendering, hardening, and loopback binding, not node logic. """ import os +import socket import tomllib import sys +import threading import time from http.server import BaseHTTPRequestHandler, HTTPServer @@ -156,6 +161,29 @@ def config_validate(args): return 0 +def _start_watchdog_heartbeat(): + """Send WATCHDOG=1 every WATCHDOG_USEC/2, if systemd armed a watchdog. + + systemd sets WATCHDOG_USEC (and NOTIFY_SOCKET) only when the unit that started + this process carries WatchdogSec, so a unit without one gets no thread. + """ + usec = os.environ.get("WATCHDOG_USEC") + addr = os.environ.get("NOTIFY_SOCKET") + if not usec or not addr: + return + if addr.startswith("@"): # abstract namespace + addr = "\0" + addr[1:] + interval = int(usec) / 2 / 1_000_000 + + def beat(): + with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock: + while True: + sock.sendto(b"WATCHDOG=1", addr) + time.sleep(interval) + + threading.Thread(target=beat, daemon=True).start() + + def main(): args = sys.argv[1:] if "--version" in args: @@ -166,6 +194,7 @@ def main(): if args[:2] == ["config", "validate"]: return config_validate(args[2:]) if args and args[0] == "run": + _start_watchdog_heartbeat() # Late-bind simulation: stay alive (Type=simple => unit stays `running`) # but never bind metrics, so the role's advisory probe times out. if os.path.isfile(NO_METRICS_MARKER): diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml index 1fb90c9..5b8b13a 100644 --- a/ansible/molecule/default/molecule.yml +++ b/ansible/molecule/default/molecule.yml @@ -34,7 +34,8 @@ platforms: provisioner: name: ansible env: - ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + # molecule/default/roles holds converge.yml's test-only shadow_reload_systemd. + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles:${MOLECULE_PROJECT_DIRECTORY}/molecule/default/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" # The docker connection plugin supports pipelining, which drops a `docker exec` # round trip per remote task — 105s -> 82s measured on this scenario. Safe here: diff --git a/ansible/molecule/default/prepare.yml b/ansible/molecule/default/prepare.yml index 0f467c6..d2e197d 100644 --- a/ansible/molecule/default/prepare.yml +++ b/ansible/molecule/default/prepare.yml @@ -42,3 +42,22 @@ dest: "{{ decdn_home }}/node.secret" content: "molecule-placeholder-node-secret\n" mode: "0644" + + # A stale decdn-node unit, loaded and running, so converge CHANGES the unit + # rather than creating it. systemd loads a new unit from disk on first use, but + # restarts a loaded one from its cached copy until a daemon-reload, which is + # the case the restart handler has to get right. verify.yml asserts the + # running process came from the role's unit, not this one. + - name: Stage a stale decdn-node unit + ansible.builtin.copy: + dest: /etc/systemd/system/decdn-node.service + content: | + [Service] + ExecStart=/bin/sleep infinity + mode: "0644" + + - name: Load and start the stale unit + ansible.builtin.systemd: + name: decdn-node + state: started + daemon_reload: true diff --git a/ansible/molecule/default/roles/shadow_reload_systemd/handlers/main.yml b/ansible/molecule/default/roles/shadow_reload_systemd/handlers/main.yml new file mode 100644 index 0000000..128c705 --- /dev/null +++ b/ansible/molecule/default/roles/shadow_reload_systemd/handlers/main.yml @@ -0,0 +1,10 @@ +--- +# Stands in for devsec.hardening.os_hardening's handler of the same name. baseline +# loads that role with include_role, which appends its handlers to the play at run +# time, so on a real site.yml run it is the LAST "Reload systemd" loaded: it wins the +# name lookup for every role's `notify: Reload systemd`, and runs after the static +# roles' handlers — after "Restart decdn-node". converge.yml loads this role the same +# way to reproduce that ordering in the container. +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 815052a..5e9d200 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -322,6 +322,41 @@ vars: unit: "{{ decdn_unit.content | b64decode }}" + # The unit on disk is not what runs: systemd restarts a loaded unit from its + # cached copy until a daemon-reload. prepare.yml left a stale unit (sleep, no + # WatchdogSec) loaded and converge.yml shadows the roles' "Reload systemd" the + # way devsec.hardening does, so this passes only if the restart itself reloads. + # WATCHDOG_USEC is set by systemd in the started process's environment, so it + # proves the watchdog was armed for THIS process, not merely configured. + - name: Read the running decdn-node's command line and environment + ansible.builtin.shell: + cmd: | + set -euo pipefail + pid=$(systemctl show -p MainPID --value decdn-node) + tr '\0' '\n' < "/proc/$pid/cmdline" + echo --- + # WATCHDOG_* only: the environment also carries DECDN_RPC_URL. + tr '\0' '\n' < "/proc/$pid/environ" | grep '^WATCHDOG_' || true + executable: /bin/bash + register: decdn_running + changed_when: false + + - name: Assert decdn-node runs from the role's unit with the watchdog armed + ansible.builtin.assert: + that: + - "'/usr/local/bin/decdn-node' in running_cmd" + - "'WATCHDOG_USEC=60000000' in running_env" + - "'WatchdogSec=60' in unit and 'NotifyAccess=main' in unit" + - "'AF_NETLINK AF_UNIX' in unit" + fail_msg: >- + decdn-node is running from a stale unit or without its watchdog: a + restart handler ran before any daemon-reload. Command line: + {{ running_cmd }}; environment: {{ running_env }} + vars: + running_cmd: "{{ decdn_running.stdout.split('---')[0] }}" + running_env: "{{ decdn_running.stdout.split('---')[1] | default('') }}" + unit: "{{ decdn_unit.content | b64decode }}" + - name: Collect listening TCP sockets ansible.builtin.command: cmd: ss -ltn diff --git a/ansible/molecule/os-matrix/molecule.yml b/ansible/molecule/os-matrix/molecule.yml index 7a08fc9..e920951 100644 --- a/ansible/molecule/os-matrix/molecule.yml +++ b/ansible/molecule/os-matrix/molecule.yml @@ -49,7 +49,8 @@ platforms: provisioner: name: ansible env: - ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + # molecule/default/roles holds converge.yml's test-only shadow_reload_systemd. + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles:${MOLECULE_PROJECT_DIRECTORY}/molecule/default/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" # See default/molecule.yml for why pipelining is on. ANSIBLE_PIPELINING: "true" diff --git a/ansible/roles/decdn_node/handlers/main.yml b/ansible/roles/decdn_node/handlers/main.yml index fd30903..64fae43 100644 --- a/ansible/roles/decdn_node/handlers/main.yml +++ b/ansible/roles/decdn_node/handlers/main.yml @@ -3,10 +3,15 @@ ansible.builtin.systemd: daemon_reload: true +# Reloads systemd itself rather than relying on "Reload systemd" running first. +# A same-named handler loaded later shadows this role's — baseline include_role's +# devsec.hardening.os_hardening defines one — and runs after this restart, which +# then starts decdn-node from systemd's cached copy of the unit it just rewrote. - name: Restart decdn-node ansible.builtin.systemd: name: decdn-node state: restarted + daemon_reload: true # Applies the five hot-reloadable config sections without dropping connections. # Nothing in this role notifies it — the node.toml handler restarts, because a diff --git a/ansible/roles/grafana_alloy/handlers/main.yml b/ansible/roles/grafana_alloy/handlers/main.yml index 73c5354..091ae41 100644 --- a/ansible/roles/grafana_alloy/handlers/main.yml +++ b/ansible/roles/grafana_alloy/handlers/main.yml @@ -3,7 +3,12 @@ ansible.builtin.systemd: daemon_reload: true +# Reloads systemd itself rather than relying on "Reload systemd" running first. +# A same-named handler loaded later shadows this role's — baseline include_role's +# devsec.hardening.os_hardening defines one — and runs after this restart, which +# then starts alloy from systemd's cached copy of the unit it just rewrote. - name: Restart alloy ansible.builtin.systemd: name: alloy state: restarted + daemon_reload: true From d9d1f83775f0e997a315e3899601738286b6ab3f Mon Sep 17 00:00:00 2001 From: Alper Gundogdu Date: Thu, 24 Sep 2026 15:53:22 +0100 Subject: [PATCH 2/2] test(molecule): fail clearly when decdn-node has no main process Co-Authored-By: Claude Opus 5.5 --- ansible/molecule/default/verify.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 5e9d200..988f237 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -333,6 +333,13 @@ cmd: | set -euo pipefail pid=$(systemctl show -p MainPID --value decdn-node) + # MainPID is 0 when the unit is not running; say so rather than fail + # on a confusing read of /proc/0. + if [ -z "$pid" ] || [ "$pid" = 0 ]; then + echo "decdn-node has no main process (MainPID='$pid');" \ + "check: journalctl -u decdn-node -e" >&2 + exit 1 + fi tr '\0' '\n' < "/proc/$pid/cmdline" echo --- # WATCHDOG_* only: the environment also carries DECDN_RPC_URL.