From 8efa26512ae90789611fea725454a54fe1c8a6f6 Mon Sep 17 00:00:00 2001 From: Alper Gundogdu Date: Thu, 24 Sep 2026 11:36:09 +0100 Subject: [PATCH 1/2] feat(decdn_node): enable the systemd watchdog on the node unit decdn-node heartbeats WATCHDOG=1 from its tokio runtime when the unit sets WatchdogSec (decdn/decdn#2145). A node whose runtime workers are all held by work that never yields stops the heartbeat, and systemd restarts it instead of leaving it `active` while it serves nothing. - WatchdogSec={{ decdn_node_watchdog_sec }} (default 60) plus NotifyAccess=main, so systemd accepts the main PID's heartbeat. - AF_UNIX joins RestrictAddressFamilies, since the heartbeat goes over NOTIFY_SOCKET. - decdn_node_watchdog_sec: 0 omits all three, for a binary that predates the heartbeat and would otherwise be killed every period. Co-Authored-By: Claude Opus 5.5 --- ansible/roles/decdn_node/defaults/main.yml | 8 ++++++++ .../decdn_node/templates/decdn-node.service.j2 | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ansible/roles/decdn_node/defaults/main.yml b/ansible/roles/decdn_node/defaults/main.yml index f90f854..d13ef66 100644 --- a/ansible/roles/decdn_node/defaults/main.yml +++ b/ansible/roles/decdn_node/defaults/main.yml @@ -406,6 +406,14 @@ decdn_cli_bin: /usr/local/bin/decdn # templates/decdn-node.service.j2. decdn_node_stop_timeout_sec: 300 +# systemd WatchdogSec for the node unit (seconds). The node heartbeats from its +# tokio runtime every half period, so a runtime whose workers never yield is +# killed and restarted instead of staying `active` while it serves nothing. It +# needs a decdn-node build that sends the heartbeat: an older binary sends none +# and is killed every period, so set 0 to omit the watchdog for such a binary. +# See templates/decdn-node.service.j2. +decdn_node_watchdog_sec: 60 + # --- Lifecycle: backup + decommission (tasks_from: backup / decommission) ------- # Only read by those entry points (playbooks/backup.yml, playbooks/decommission.yml); # a normal deploy ignores them. See docs/lifecycle.md. diff --git a/ansible/roles/decdn_node/templates/decdn-node.service.j2 b/ansible/roles/decdn_node/templates/decdn-node.service.j2 index 15b6148..c5cfe03 100644 --- a/ansible/roles/decdn_node/templates/decdn-node.service.j2 +++ b/ansible/roles/decdn_node/templates/decdn-node.service.j2 @@ -47,6 +47,18 @@ TimeoutStopSec={{ decdn_node_stop_timeout_sec }} Restart=always RestartSec=5 +{% if decdn_node_watchdog_sec | int > 0 %} +# Runtime-liveness watchdog. The node sends WATCHDOG=1 from a task on its tokio +# runtime every WatchdogSec/2, so the heartbeat stops when every runtime worker +# is held by work that never yields. The unit then stays `active` while the node +# serves nothing, logs nothing and answers no /metrics scrape. systemd kills it +# with SIGABRT at the deadline and Restart=always brings it back. The heartbeat +# arrives over NOTIFY_SOCKET, so the unit allows the main PID to notify and +# allows AF_UNIX below. +WatchdogSec={{ decdn_node_watchdog_sec }} +NotifyAccess=main +{% endif %} + # --- Hardening ------------------------------------------------------------- # StateDirectory owns {{ decdn_home }} (the only writable path under # ProtectSystem=strict): node.secret, keystore.json, cache, redb state. @@ -61,8 +73,10 @@ ProtectKernelTunables=true ProtectKernelModules=true ProtectControlGroups=true ProtectClock=true -# AF_INET/AF_INET6 for QUIC; AF_NETLINK for interface/address discovery (STUN). -RestrictAddressFamilies=AF_INET AF_INET6 AF_NETLINK +# AF_INET/AF_INET6 for QUIC; AF_NETLINK for interface/address discovery (STUN); +# AF_UNIX for the watchdog heartbeat on NOTIFY_SOCKET. +RestrictAddressFamilies=AF_INET AF_INET6 AF_NETLINK{% if decdn_node_watchdog_sec | int > 0 %} AF_UNIX{% endif %} + RestrictNamespaces=true RestrictSUIDSGID=true LockPersonality=true From be0447eed71e3bfec23271f7d94188feac0753cd Mon Sep 17 00:00:00 2001 From: Alper Gundogdu Date: Thu, 24 Sep 2026 11:43:03 +0100 Subject: [PATCH 2/2] fix(decdn_node): make the systemd watchdog opt-in The role cannot tell whether the binary it installs heartbeats, and a binary that does not is killed every period, so decdn_node_watchdog_sec defaults to 0. Co-Authored-By: Claude Opus 5.5 --- ansible/roles/decdn_node/defaults/main.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ansible/roles/decdn_node/defaults/main.yml b/ansible/roles/decdn_node/defaults/main.yml index d13ef66..1c480a4 100644 --- a/ansible/roles/decdn_node/defaults/main.yml +++ b/ansible/roles/decdn_node/defaults/main.yml @@ -406,13 +406,14 @@ decdn_cli_bin: /usr/local/bin/decdn # templates/decdn-node.service.j2. decdn_node_stop_timeout_sec: 300 -# systemd WatchdogSec for the node unit (seconds). The node heartbeats from its -# tokio runtime every half period, so a runtime whose workers never yield is -# killed and restarted instead of staying `active` while it serves nothing. It -# needs a decdn-node build that sends the heartbeat: an older binary sends none -# and is killed every period, so set 0 to omit the watchdog for such a binary. -# See templates/decdn-node.service.j2. -decdn_node_watchdog_sec: 60 +# systemd WatchdogSec for the node unit (seconds); 0 omits the watchdog. The node +# heartbeats from its tokio runtime every half period, so a runtime whose workers +# never yield is killed and restarted instead of staying `active` while it serves +# nothing. It needs a decdn-node build that sends the heartbeat (decdn/decdn#2145): +# an older binary sends none and is killed every period. The role cannot tell which +# build it installs, so the watchdog is opt-in: set e.g. 60 once the deployed binary +# heartbeats. See templates/decdn-node.service.j2. +decdn_node_watchdog_sec: 0 # --- Lifecycle: backup + decommission (tasks_from: backup / decommission) ------- # Only read by those entry points (playbooks/backup.yml, playbooks/decommission.yml);