fix(roles): daemon-reload inside the restart handlers

ansible/galaxy/CHANGELOG.md

@@ -78,6 +78,14 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
 
 ### Fixed
 
+- A changed `decdn-node` or `alloy` unit now takes effect on the converge that
+  writes it. The restart handlers relied on a separate `Reload systemd`
+  handler running first, but `devsec.hardening.os_hardening` (loaded by
+  `baseline` through `include_role`) defines a handler of the same name. Being
+  loaded last, it shadows the roles' own and runs after the restarts, so
+  systemd restarted the service from its cached unit: a new `WatchdogSec` was
+  on disk while the running node had no watchdog until the next restart. The
+  restart handlers now `daemon_reload` themselves.
 - The OTLP gateway is authenticated with its own instance ID. The role reused
   the Prometheus one, so a Grafana Cloud org whose stack ID differs 401s every
   trace while metrics and logs keep flowing. New `grafana_alloy_otlp_username` /

ansible/molecule/default/converge.yml

@@ -33,6 +33,10 @@
       # Makes the stub compare the environment received by `config validate`
       # against the exact literal above (not merely assert that no command ran).
       DECDN_MOLECULE_ASSERT_RPC_LITERAL: "1"
+    # Arms the unit's watchdog (the stub heartbeats), so verify.yml can prove the
+    # running process was started from the unit this converge wrote: prepare.yml
+    # leaves a stale unit loaded that has no WatchdogSec.
+    decdn_node_watchdog_sec: 60
     decdn_chain_id: 421614
     decdn_region: "US"
     decdn_payment_pool_address: "0x1111111111111111111111111111111111111111"
@@ -124,6 +128,13 @@
       ansible.builtin.include_role:
         name: baseline
         tasks_from: sudo_users
+    # Reproduces site.yml's handler order: baseline include_role's
+    # devsec.hardening.os_hardening, whose same-named "Reload systemd" handler is
+    # loaded last, shadows the roles' own and runs after "Restart decdn-node".
+    # verify.yml asserts the node still runs on the unit this converge wrote.
+    - name: Load a same-named "Reload systemd" handler, as devsec.hardening does
+      ansible.builtin.include_role:
+        name: shadow_reload_systemd  # molecule/default/roles, on ANSIBLE_ROLES_PATH
   roles:
     - role: grafana_alloy
       tags: [observability, decdn_node, node]

ansible/molecule/default/files/decdn-node-stub

@@ -10,6 +10,9 @@ can be exercised end-to-end without a published release or a live chain:
                      the role's `/metrics` readiness probe succeeds, and block
                      forever under systemd (Type=simple). CLI args (`--config`,
                      `--keystore-password-file`) are accepted and ignored.
+                     Under a unit with WatchdogSec it heartbeats WATCHDOG=1 to
+                     NOTIFY_SOCKET every WATCHDOG_USEC/2, as the real daemon
+                     does, so systemd does not kill it at the deadline.
                      If the marker file NO_METRICS_MARKER exists, it instead
                      blocks forever WITHOUT binding metrics — standing in for a
                      healthy node whose metrics listener binds late (behind the
@@ -36,8 +39,10 @@ It intentionally implements no protocol behaviour — the scenario verifies host
 prep, config/unit rendering, hardening, and loopback binding, not node logic.
 """
 import os
+import socket
 import tomllib
 import sys
+import threading
 import time
 from http.server import BaseHTTPRequestHandler, HTTPServer
 
@@ -156,6 +161,29 @@ def config_validate(args):
     return 0
 
 
+def _start_watchdog_heartbeat():
+    """Send WATCHDOG=1 every WATCHDOG_USEC/2, if systemd armed a watchdog.
+
+    systemd sets WATCHDOG_USEC (and NOTIFY_SOCKET) only when the unit that started
+    this process carries WatchdogSec, so a unit without one gets no thread.
+    """
+    usec = os.environ.get("WATCHDOG_USEC")
+    addr = os.environ.get("NOTIFY_SOCKET")
+    if not usec or not addr:
+        return
+    if addr.startswith("@"):  # abstract namespace
+        addr = "\0" + addr[1:]
+    interval = int(usec) / 2 / 1_000_000
+
+    def beat():
+        with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+            while True:
+                sock.sendto(b"WATCHDOG=1", addr)
+                time.sleep(interval)
+
+    threading.Thread(target=beat, daemon=True).start()
+
+
 def main():
     args = sys.argv[1:]
     if "--version" in args:
@@ -166,6 +194,7 @@ def main():
     if args[:2] == ["config", "validate"]:
         return config_validate(args[2:])
     if args and args[0] == "run":
+        _start_watchdog_heartbeat()
         # Late-bind simulation: stay alive (Type=simple => unit stays `running`)
         # but never bind metrics, so the role's advisory probe times out.
         if os.path.isfile(NO_METRICS_MARKER):

ansible/molecule/default/molecule.yml

@@ -34,7 +34,8 @@ platforms:
 provisioner:
   name: ansible
   env:
-    ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles"
+    # molecule/default/roles holds converge.yml's test-only shadow_reload_systemd.
+    ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles:${MOLECULE_PROJECT_DIRECTORY}/molecule/default/roles"
     ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections"
     # The docker connection plugin supports pipelining, which drops a `docker exec`
     # round trip per remote task — 105s -> 82s measured on this scenario. Safe here:

ansible/molecule/default/prepare.yml

@@ -42,3 +42,22 @@
         dest: "{{ decdn_home }}/node.secret"
         content: "molecule-placeholder-node-secret\n"
         mode: "0644"
+
+    # A stale decdn-node unit, loaded and running, so converge CHANGES the unit
+    # rather than creating it. systemd loads a new unit from disk on first use, but
+    # restarts a loaded one from its cached copy until a daemon-reload, which is
+    # the case the restart handler has to get right. verify.yml asserts the
+    # running process came from the role's unit, not this one.
+    - name: Stage a stale decdn-node unit
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/decdn-node.service
+        content: |
+          [Service]
+          ExecStart=/bin/sleep infinity
+        mode: "0644"
+
+    - name: Load and start the stale unit
+      ansible.builtin.systemd:
+        name: decdn-node
+        state: started
+        daemon_reload: true

ansible/molecule/default/roles/shadow_reload_systemd/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+# Stands in for devsec.hardening.os_hardening's handler of the same name. baseline
+# loads that role with include_role, which appends its handlers to the play at run
+# time, so on a real site.yml run it is the LAST "Reload systemd" loaded: it wins the
+# name lookup for every role's `notify: Reload systemd`, and runs after the static
+# roles' handlers — after "Restart decdn-node". converge.yml loads this role the same
+# way to reproduce that ordering in the container.
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

ansible/molecule/default/verify.yml

@@ -322,6 +322,48 @@
       vars:
         unit: "{{ decdn_unit.content | b64decode }}"
 
+    # The unit on disk is not what runs: systemd restarts a loaded unit from its
+    # cached copy until a daemon-reload. prepare.yml left a stale unit (sleep, no
+    # WatchdogSec) loaded and converge.yml shadows the roles' "Reload systemd" the
+    # way devsec.hardening does, so this passes only if the restart itself reloads.
+    # WATCHDOG_USEC is set by systemd in the started process's environment, so it
+    # proves the watchdog was armed for THIS process, not merely configured.
+    - name: Read the running decdn-node's command line and environment
+      ansible.builtin.shell:
+        cmd: |
+          set -euo pipefail
+          pid=$(systemctl show -p MainPID --value decdn-node)
+          # MainPID is 0 when the unit is not running; say so rather than fail
+          # on a confusing read of /proc/0.
+          if [ -z "$pid" ] || [ "$pid" = 0 ]; then
+            echo "decdn-node has no main process (MainPID='$pid');" \
+              "check: journalctl -u decdn-node -e" >&2
+            exit 1
+          fi
+          tr '\0' '\n' < "/proc/$pid/cmdline"
+          echo ---
+          # WATCHDOG_* only: the environment also carries DECDN_RPC_URL.
+          tr '\0' '\n' < "/proc/$pid/environ" | grep '^WATCHDOG_' || true
+        executable: /bin/bash
+      register: decdn_running
+      changed_when: false
+
+    - name: Assert decdn-node runs from the role's unit with the watchdog armed
+      ansible.builtin.assert:
+        that:
+          - "'/usr/local/bin/decdn-node' in running_cmd"
+          - "'WATCHDOG_USEC=60000000' in running_env"
+          - "'WatchdogSec=60' in unit and 'NotifyAccess=main' in unit"
+          - "'AF_NETLINK AF_UNIX' in unit"
+        fail_msg: >-
+          decdn-node is running from a stale unit or without its watchdog: a
+          restart handler ran before any daemon-reload. Command line:
+          {{ running_cmd }}; environment: {{ running_env }}
+      vars:
+        running_cmd: "{{ decdn_running.stdout.split('---')[0] }}"
+        running_env: "{{ decdn_running.stdout.split('---')[1] | default('') }}"
+        unit: "{{ decdn_unit.content | b64decode }}"
+
     - name: Collect listening TCP sockets
       ansible.builtin.command:
         cmd: ss -ltn

ansible/molecule/os-matrix/molecule.yml

@@ -49,7 +49,8 @@ platforms:
 provisioner:
   name: ansible
   env:
-    ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles"
+    # molecule/default/roles holds converge.yml's test-only shadow_reload_systemd.
+    ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles:${MOLECULE_PROJECT_DIRECTORY}/molecule/default/roles"
     ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections"
     # See default/molecule.yml for why pipelining is on.
     ANSIBLE_PIPELINING: "true"

ansible/roles/decdn_node/handlers/main.yml

@@ -3,10 +3,15 @@
   ansible.builtin.systemd:
     daemon_reload: true
 
+# Reloads systemd itself rather than relying on "Reload systemd" running first.
+# A same-named handler loaded later shadows this role's — baseline include_role's
+# devsec.hardening.os_hardening defines one — and runs after this restart, which
+# then starts decdn-node from systemd's cached copy of the unit it just rewrote.
 - name: Restart decdn-node
   ansible.builtin.systemd:
     name: decdn-node
     state: restarted
+    daemon_reload: true
 
 # Applies the five hot-reloadable config sections without dropping connections.
 # Nothing in this role notifies it — the node.toml handler restarts, because a

ansible/roles/grafana_alloy/handlers/main.yml

@@ -3,7 +3,12 @@
   ansible.builtin.systemd:
     daemon_reload: true
 
+# Reloads systemd itself rather than relying on "Reload systemd" running first.
+# A same-named handler loaded later shadows this role's — baseline include_role's
+# devsec.hardening.os_hardening defines one — and runs after this restart, which
+# then starts alloy from systemd's cached copy of the unit it just rewrote.
 - name: Restart alloy
   ansible.builtin.systemd:
     name: alloy
     state: restarted
+    daemon_reload: true