diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index ed81c55..c2bf647 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -78,6 +78,14 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- A changed `decdn-node` or `alloy` unit now takes effect on the converge that + writes it. The restart handlers relied on a separate `Reload systemd` + handler running first, but `devsec.hardening.os_hardening` (loaded by + `baseline` through `include_role`) defines a handler of the same name. Being + loaded last, it shadows the roles' own and runs after the restarts, so + systemd restarted the service from its cached unit: a new `WatchdogSec` was + on disk while the running node had no watchdog until the next restart. The + restart handlers now `daemon_reload` themselves. - The OTLP gateway is authenticated with its own instance ID. The role reused the Prometheus one, so a Grafana Cloud org whose stack ID differs 401s every trace while metrics and logs keep flowing. New `grafana_alloy_otlp_username` / diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml index 568f194..24f1538 100644 --- a/ansible/molecule/default/converge.yml +++ b/ansible/molecule/default/converge.yml @@ -33,6 +33,10 @@ # Makes the stub compare the environment received by `config validate` # against the exact literal above (not merely assert that no command ran). DECDN_MOLECULE_ASSERT_RPC_LITERAL: "1" + # Arms the unit's watchdog (the stub heartbeats), so verify.yml can prove the + # running process was started from the unit this converge wrote: prepare.yml + # leaves a stale unit loaded that has no WatchdogSec. + decdn_node_watchdog_sec: 60 decdn_chain_id: 421614 decdn_region: "US" decdn_payment_pool_address: "0x1111111111111111111111111111111111111111" @@ -124,6 +128,13 @@ ansible.builtin.include_role: name: baseline tasks_from: sudo_users + # Reproduces site.yml's handler order: baseline include_role's + # devsec.hardening.os_hardening, whose same-named "Reload systemd" handler is + # loaded last, shadows the roles' own and runs after "Restart decdn-node". + # verify.yml asserts the node still runs on the unit this converge wrote. + - name: Load a same-named "Reload systemd" handler, as devsec.hardening does + ansible.builtin.include_role: + name: shadow_reload_systemd # molecule/default/roles, on ANSIBLE_ROLES_PATH roles: - role: grafana_alloy tags: [observability, decdn_node, node] diff --git a/ansible/molecule/default/files/decdn-node-stub b/ansible/molecule/default/files/decdn-node-stub index a8b7344..a944f97 100755 --- a/ansible/molecule/default/files/decdn-node-stub +++ b/ansible/molecule/default/files/decdn-node-stub @@ -10,6 +10,9 @@ can be exercised end-to-end without a published release or a live chain: the role's `/metrics` readiness probe succeeds, and block forever under systemd (Type=simple). CLI args (`--config`, `--keystore-password-file`) are accepted and ignored. + Under a unit with WatchdogSec it heartbeats WATCHDOG=1 to + NOTIFY_SOCKET every WATCHDOG_USEC/2, as the real daemon + does, so systemd does not kill it at the deadline. If the marker file NO_METRICS_MARKER exists, it instead blocks forever WITHOUT binding metrics — standing in for a healthy node whose metrics listener binds late (behind the @@ -36,8 +39,10 @@ It intentionally implements no protocol behaviour — the scenario verifies host prep, config/unit rendering, hardening, and loopback binding, not node logic. """ import os +import socket import tomllib import sys +import threading import time from http.server import BaseHTTPRequestHandler, HTTPServer @@ -156,6 +161,29 @@ def config_validate(args): return 0 +def _start_watchdog_heartbeat(): + """Send WATCHDOG=1 every WATCHDOG_USEC/2, if systemd armed a watchdog. + + systemd sets WATCHDOG_USEC (and NOTIFY_SOCKET) only when the unit that started + this process carries WatchdogSec, so a unit without one gets no thread. + """ + usec = os.environ.get("WATCHDOG_USEC") + addr = os.environ.get("NOTIFY_SOCKET") + if not usec or not addr: + return + if addr.startswith("@"): # abstract namespace + addr = "\0" + addr[1:] + interval = int(usec) / 2 / 1_000_000 + + def beat(): + with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock: + while True: + sock.sendto(b"WATCHDOG=1", addr) + time.sleep(interval) + + threading.Thread(target=beat, daemon=True).start() + + def main(): args = sys.argv[1:] if "--version" in args: @@ -166,6 +194,7 @@ def main(): if args[:2] == ["config", "validate"]: return config_validate(args[2:]) if args and args[0] == "run": + _start_watchdog_heartbeat() # Late-bind simulation: stay alive (Type=simple => unit stays `running`) # but never bind metrics, so the role's advisory probe times out. if os.path.isfile(NO_METRICS_MARKER): diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml index 1fb90c9..5b8b13a 100644 --- a/ansible/molecule/default/molecule.yml +++ b/ansible/molecule/default/molecule.yml @@ -34,7 +34,8 @@ platforms: provisioner: name: ansible env: - ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + # molecule/default/roles holds converge.yml's test-only shadow_reload_systemd. + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles:${MOLECULE_PROJECT_DIRECTORY}/molecule/default/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" # The docker connection plugin supports pipelining, which drops a `docker exec` # round trip per remote task — 105s -> 82s measured on this scenario. Safe here: diff --git a/ansible/molecule/default/prepare.yml b/ansible/molecule/default/prepare.yml index 0f467c6..d2e197d 100644 --- a/ansible/molecule/default/prepare.yml +++ b/ansible/molecule/default/prepare.yml @@ -42,3 +42,22 @@ dest: "{{ decdn_home }}/node.secret" content: "molecule-placeholder-node-secret\n" mode: "0644" + + # A stale decdn-node unit, loaded and running, so converge CHANGES the unit + # rather than creating it. systemd loads a new unit from disk on first use, but + # restarts a loaded one from its cached copy until a daemon-reload, which is + # the case the restart handler has to get right. verify.yml asserts the + # running process came from the role's unit, not this one. + - name: Stage a stale decdn-node unit + ansible.builtin.copy: + dest: /etc/systemd/system/decdn-node.service + content: | + [Service] + ExecStart=/bin/sleep infinity + mode: "0644" + + - name: Load and start the stale unit + ansible.builtin.systemd: + name: decdn-node + state: started + daemon_reload: true diff --git a/ansible/molecule/default/roles/shadow_reload_systemd/handlers/main.yml b/ansible/molecule/default/roles/shadow_reload_systemd/handlers/main.yml new file mode 100644 index 0000000..128c705 --- /dev/null +++ b/ansible/molecule/default/roles/shadow_reload_systemd/handlers/main.yml @@ -0,0 +1,10 @@ +--- +# Stands in for devsec.hardening.os_hardening's handler of the same name. baseline +# loads that role with include_role, which appends its handlers to the play at run +# time, so on a real site.yml run it is the LAST "Reload systemd" loaded: it wins the +# name lookup for every role's `notify: Reload systemd`, and runs after the static +# roles' handlers — after "Restart decdn-node". converge.yml loads this role the same +# way to reproduce that ordering in the container. +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 815052a..988f237 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -322,6 +322,48 @@ vars: unit: "{{ decdn_unit.content | b64decode }}" + # The unit on disk is not what runs: systemd restarts a loaded unit from its + # cached copy until a daemon-reload. prepare.yml left a stale unit (sleep, no + # WatchdogSec) loaded and converge.yml shadows the roles' "Reload systemd" the + # way devsec.hardening does, so this passes only if the restart itself reloads. + # WATCHDOG_USEC is set by systemd in the started process's environment, so it + # proves the watchdog was armed for THIS process, not merely configured. + - name: Read the running decdn-node's command line and environment + ansible.builtin.shell: + cmd: | + set -euo pipefail + pid=$(systemctl show -p MainPID --value decdn-node) + # MainPID is 0 when the unit is not running; say so rather than fail + # on a confusing read of /proc/0. + if [ -z "$pid" ] || [ "$pid" = 0 ]; then + echo "decdn-node has no main process (MainPID='$pid');" \ + "check: journalctl -u decdn-node -e" >&2 + exit 1 + fi + tr '\0' '\n' < "/proc/$pid/cmdline" + echo --- + # WATCHDOG_* only: the environment also carries DECDN_RPC_URL. + tr '\0' '\n' < "/proc/$pid/environ" | grep '^WATCHDOG_' || true + executable: /bin/bash + register: decdn_running + changed_when: false + + - name: Assert decdn-node runs from the role's unit with the watchdog armed + ansible.builtin.assert: + that: + - "'/usr/local/bin/decdn-node' in running_cmd" + - "'WATCHDOG_USEC=60000000' in running_env" + - "'WatchdogSec=60' in unit and 'NotifyAccess=main' in unit" + - "'AF_NETLINK AF_UNIX' in unit" + fail_msg: >- + decdn-node is running from a stale unit or without its watchdog: a + restart handler ran before any daemon-reload. Command line: + {{ running_cmd }}; environment: {{ running_env }} + vars: + running_cmd: "{{ decdn_running.stdout.split('---')[0] }}" + running_env: "{{ decdn_running.stdout.split('---')[1] | default('') }}" + unit: "{{ decdn_unit.content | b64decode }}" + - name: Collect listening TCP sockets ansible.builtin.command: cmd: ss -ltn diff --git a/ansible/molecule/os-matrix/molecule.yml b/ansible/molecule/os-matrix/molecule.yml index 7a08fc9..e920951 100644 --- a/ansible/molecule/os-matrix/molecule.yml +++ b/ansible/molecule/os-matrix/molecule.yml @@ -49,7 +49,8 @@ platforms: provisioner: name: ansible env: - ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + # molecule/default/roles holds converge.yml's test-only shadow_reload_systemd. + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles:${MOLECULE_PROJECT_DIRECTORY}/molecule/default/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" # See default/molecule.yml for why pipelining is on. ANSIBLE_PIPELINING: "true" diff --git a/ansible/roles/decdn_node/handlers/main.yml b/ansible/roles/decdn_node/handlers/main.yml index fd30903..64fae43 100644 --- a/ansible/roles/decdn_node/handlers/main.yml +++ b/ansible/roles/decdn_node/handlers/main.yml @@ -3,10 +3,15 @@ ansible.builtin.systemd: daemon_reload: true +# Reloads systemd itself rather than relying on "Reload systemd" running first. +# A same-named handler loaded later shadows this role's — baseline include_role's +# devsec.hardening.os_hardening defines one — and runs after this restart, which +# then starts decdn-node from systemd's cached copy of the unit it just rewrote. - name: Restart decdn-node ansible.builtin.systemd: name: decdn-node state: restarted + daemon_reload: true # Applies the five hot-reloadable config sections without dropping connections. # Nothing in this role notifies it — the node.toml handler restarts, because a diff --git a/ansible/roles/grafana_alloy/handlers/main.yml b/ansible/roles/grafana_alloy/handlers/main.yml index 73c5354..091ae41 100644 --- a/ansible/roles/grafana_alloy/handlers/main.yml +++ b/ansible/roles/grafana_alloy/handlers/main.yml @@ -3,7 +3,12 @@ ansible.builtin.systemd: daemon_reload: true +# Reloads systemd itself rather than relying on "Reload systemd" running first. +# A same-named handler loaded later shadows this role's — baseline include_role's +# devsec.hardening.os_hardening defines one — and runs after this restart, which +# then starts alloy from systemd's cached copy of the unit it just rewrote. - name: Restart alloy ansible.builtin.systemd: name: alloy state: restarted + daemon_reload: true