feat(decdn_node): validate manual binaries are ELF + drain-safe stop timeout

ansible/molecule/validation/converge.yml

@@ -63,6 +63,62 @@
             decdn_rejected: "{{ decdn_rejected + ['cross-field'] }}"
           when: ansible_failed_task.name is match('^Validate optional')
 
+    # --- Case: binary-format (dir-mode ships a non-ELF binary) --------------------
+    # decdn_release_target_dir mode validates on the CONTROL MACHINE that the resolved
+    # binaries are ELF for decdn_node_target before shipping them. Point it at a dir
+    # holding the (non-ELF Python) stub named as the two binaries and confirm the
+    # format assert rejects it — the "built a macOS Mach-O and shipped it" guard.
+    # Staged + validated on the controller (delegate_to: localhost), so it aborts
+    # before any host mutation, like the other assert-only cases.
+    - name: "Case binary-format — dir-mode resolves a non-ELF binary"
+      vars:
+        _decdn_badbin_dir: >-
+          {{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/.molecule-badbin/target/release
+      block:
+        - name: Create the staging directory on the control machine
+          ansible.builtin.file:
+            path: "{{ _decdn_badbin_dir }}"
+            state: directory
+            mode: "0755"
+          delegate_to: localhost
+          become: false
+        - name: Stage a non-ELF binary pair on the control machine
+          ansible.builtin.copy:
+            src: "{{ stub_bin }}"
+            dest: "{{ _decdn_badbin_dir }}/{{ item }}"
+            mode: "0755"
+          loop: [decdn-node, decdn]
+          delegate_to: localhost
+          become: false
+        - name: Run decdn_node pointed at the non-ELF target dir
+          ansible.builtin.include_role:
+            name: decdn_node
+          vars:
+            decdn_node_manual_bin_src: ""
+            decdn_cli_manual_bin_src: ""
+            decdn_release_target_dir: "{{ _decdn_badbin_dir }}"
+      rescue:
+        - name: Record binary-format rejection (only if the ELF assert failed)
+          ansible.builtin.set_fact:
+            decdn_rejected: "{{ decdn_rejected + ['binary-format'] }}"
+          when: ansible_failed_task.name is match('^Require both binaries to be ELF')
+      always:
+        - name: Remove the staged non-ELF binaries
+          ansible.builtin.file:
+            path: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/.molecule-badbin"
+            state: absent
+          delegate_to: localhost
+          become: false
+        # The role's derive-paths set_fact left decdn_node_manual_bin_src pointing at
+        # the (now-removed) badbin dir as a HOST FACT, which outranks the play var.
+        # Reset the baseline so later cases — notably config-gate, which reaches the
+        # copy step and does NOT re-pass the path — resolve the stub again.
+        - name: Restore the baseline binary sources for the remaining cases
+          ansible.builtin.set_fact:
+            decdn_node_manual_bin_src: "{{ stub_bin }}"
+            decdn_cli_manual_bin_src: "{{ stub_bin }}"
+            decdn_release_target_dir: ""
+
     # --- Case: range (numeric but below the daemon's minimum) ---------------------
     - name: "Case range — event_poll_interval_ms below 250"
       block:
@@ -354,6 +410,6 @@
           the daemon and crash-loop it at config load).
       vars:
         _decdn_expected:
-          ["cross-field", "range", "bounded-range", "shape", "bool", "enum", "dict",
-           "float", "list", "string", "env-gate", "env-extra", "env-content",
-           "config-gate"]
+          ["cross-field", "binary-format", "range", "bounded-range", "shape", "bool",
+           "enum", "dict", "float", "list", "string", "env-gate", "env-extra",
+           "env-content", "config-gate"]

ansible/roles/decdn_node/README.md

@@ -55,15 +55,32 @@ Per the deCDN node-onboarding ADR (019), a node only serves paid traffic after
      from the target host; override `decdn_node_release_base` for a mirror.
      **No upstream release exists yet**, so this mode currently has nothing to
      fetch — the role's assert says so rather than surfacing a bare 404.
-   - **`manual`** (current default) — the role copies the two binaries **verbatim** from the paths
-     you give it (`decdn_node_manual_bin_src` + `decdn_cli_manual_bin_src`) on the
-     Ansible control machine; it does *not* consult `decdn_node_target`, so you are
-     responsible for building for the host's architecture (the default target is
-     `x86_64-unknown-linux-gnu`). Build both `decdn-node` and `decdn` from the
-     upstream `decdn` repo, then set the two paths. `decdn_node_version` is **not**
-     required in this mode — but if it is set (e.g. left over from a `release`
-     deploy) the `--version` backstop still enforces it, so clear it when switching
-     to `manual` unless you intend that binary to report that exact version.
+   - **`manual`** (current default) — the role copies the two binaries from the
+     Ansible control machine. Point it at them **either** way:
+     - **`decdn_release_target_dir`** (recommended) — the Cargo `target/release`
+       dir. The role derives both binary paths from it, **falls back** to the
+       cross / `--target` output dir (`target/{{ decdn_node_target }}/release`)
+       when the plain dir lacks them, and — crucially — **validates on the control
+       machine that both binaries are ELF for `decdn_node_target`** before shipping
+       them. A host-native `cargo build --release` on a non-Linux control machine
+       (macOS) silently produces a Mach-O the Linux node cannot exec; this catches
+       that with an actionable error instead of an opaque first-start failure.
+       Cross-compile for the node: `cross build --release --target
+       {{ decdn_node_target }}` (or `cargo build --release --target …`).
+     - **`decdn_node_manual_bin_src` + `decdn_cli_manual_bin_src`** — explicit paths,
+       copied **verbatim**. This is the escape hatch: the role trusts them and does
+       *not* consult `decdn_node_target` (so a test-harness stub that is legitimately
+       not an ELF for the node's arch still works). You own building for the host's
+       architecture (default target `x86_64-unknown-linux-gnu`). Set **both together
+       or neither** (a partial pair is rejected); when both are set they **take
+       precedence** over `decdn_release_target_dir`, so a host can override a
+       fleet-wide target dir without having to blank it.
+
+     Build both `decdn-node` and `decdn` from the upstream `decdn` repo.
+     `decdn_node_version` is **not** required in this mode — but if it is set (e.g.
+     left over from a `release` deploy) the `--version` backstop still enforces it,
+     so clear it when switching to `manual` unless you intend that binary to report
+     that exact version.
 
 2. **Eth wallet (operator-provisioned).** Generate the node identity + eth
    keystore on the host, as the `decdn` user, directly into the data dir. `key-gen`
@@ -101,8 +118,9 @@ Per the deCDN node-onboarding ADR (019), a node only serves paid traffic after
 
 ## Required variables (set in `host_vars/<node>/`)
 
-`decdn_node_version` (`release` mode only) **or** `decdn_node_manual_bin_src` +
-`decdn_cli_manual_bin_src` (`manual` mode), an RPC endpoint (sensitive — may embed an
+`decdn_node_version` (`release` mode only) **or**, in `manual` mode, either
+`decdn_release_target_dir` or `decdn_node_manual_bin_src` + `decdn_cli_manual_bin_src`
+(see [Prerequisites](#prerequisites) above), an RPC endpoint (sensitive — may embed an
 API key; provision it on the host or set `decdn_rpc_url` in the git-ignored
 `secret.yml` — see [Secrets](#secrets)), `decdn_region` (ISO 3166-1 alpha-2), and
 **four** contract addresses (all `0x`+40-hex, none the zero address):

ansible/roles/decdn_node/defaults/main.yml

@@ -17,6 +17,15 @@
 decdn_node_install_method: manual   # "release" | "manual"
 decdn_node_manual_bin_src: ""        # manual: control-machine path to the decdn-node daemon binary
 decdn_cli_manual_bin_src: ""         # manual: control-machine path to the decdn CLI binary
+# Alternative to the two explicit *_manual_bin_src paths: point at the control-
+# machine `target/release` dir and the role derives BOTH binary paths from it. It
+# also (a) falls back to the cross / `--target` output dir
+# (target/{{ decdn_node_target }}/release) when the plain dir lacks the binaries,
+# and (b) validates on the control machine that the binaries are ELF for
+# decdn_node_target BEFORE shipping them — because a host-native `cargo build
+# --release` on a non-Linux control machine (macOS) silently produces a Mach-O the
+# Linux node cannot exec. Leave "" to use the explicit paths above instead.
+decdn_release_target_dir: ""
 
 # --- Release to install (REQUIRED in "release" mode — a v<version> GitHub Release must exist) ---
 decdn_node_version: ""            # e.g. "0.1.1" (asserted non-empty in "release" mode)
@@ -352,3 +361,11 @@ decdn_node_generate_keystore: false
 decdn_cache_dir: /var/lib/decdn/cache
 decdn_bin: /usr/local/bin/decdn-node
 decdn_cli_bin: /usr/local/bin/decdn
+
+# systemd TimeoutStopSec for the node unit (seconds). SIGTERM triggers a full
+# graceful drain (await in-flight streams, flush settlement + receipt/audit tail);
+# the process exits the instant that finishes, so this is only the CAP before a
+# SIGKILL. It is generous on purpose — a SIGKILL mid-drain truncates paid, in-flight
+# deliveries (a money-losing cut), so prefer waiting over cutting. See
+# templates/decdn-node.service.j2.
+decdn_node_stop_timeout_sec: 300

ansible/roles/decdn_node/tasks/main.yml

@@ -40,16 +40,161 @@
       it off means the tarballs are taken on trust.
   when: decdn_node_install_method == "release"
 
-- name: Require manual binary sources when install method is manual
-  ansible.builtin.assert:
-    that:
-      - decdn_node_manual_bin_src | length > 0
-      - decdn_cli_manual_bin_src | length > 0
-    fail_msg: >-
-      decdn_node_install_method "manual" requires decdn_node_manual_bin_src and
-      decdn_cli_manual_bin_src — control-machine paths to the locally-built
-      decdn-node daemon and decdn CLI binaries. Set them in host_vars.
+# Resolve + validate the manual binary sources on the CONTROL MACHINE before they
+# are copied to the node. Two ways to point at them, in precedence order:
+#   1. both explicit *_manual_bin_src paths — the escape hatch, copied verbatim and
+#      NOT format-checked (a test-harness stub is legitimately not an ELF for the
+#      node's arch). Both must be set together; they override a fleet-wide
+#      decdn_release_target_dir, so a per-host override works without blanking it.
+#   2. decdn_release_target_dir — a target/release dir the role searches (plus the
+#      cross/`--target` output dir target/<triple>/release as a fallback) for a dir
+#      holding BOTH binaries, then checks they are ELF for decdn_node_target. A
+#      host-native `cargo build --release` on a non-Linux control machine (macOS)
+#      yields a Mach-O the Linux node cannot exec; this catches it at deploy time
+#      rather than as an opaque first-start crash.
+- name: Resolve and validate the manual binary sources
   when: decdn_node_install_method == "manual"
+  block:
+    - name: Classify the manual binary source (explicit paths vs target dir)
+      ansible.builtin.set_fact:
+        _decdn_both_explicit: "{{ decdn_node_manual_bin_src | length > 0 and decdn_cli_manual_bin_src | length > 0 }}"
+        _decdn_any_explicit: "{{ decdn_node_manual_bin_src | length > 0 or decdn_cli_manual_bin_src | length > 0 }}"
+
+    # Dir-mode (derive + fallback + ELF validation) applies only when a target dir is
+    # set AND the explicit pair has not overridden it.
+    - name: Decide whether target-dir resolution applies
+      ansible.builtin.set_fact:
+        _decdn_use_dir: "{{ decdn_release_target_dir | length > 0 and not (_decdn_both_explicit | bool) }}"
+
+    - name: Require a manual binary source (target dir or BOTH explicit paths)
+      ansible.builtin.assert:
+        that:
+          - decdn_release_target_dir | length > 0 or _decdn_both_explicit | bool
+        fail_msg: >-
+          decdn_node_install_method "manual" needs EITHER decdn_release_target_dir
+          (the control-machine target/release dir — the role derives + validates both
+          binaries from it) OR both decdn_node_manual_bin_src and
+          decdn_cli_manual_bin_src (explicit control-machine paths). Set one in host_vars.
+
+    - name: Reject a partial explicit binary override
+      ansible.builtin.assert:
+        that:
+          - not (_decdn_any_explicit | bool and not _decdn_both_explicit | bool)
+        fail_msg: >-
+          Set BOTH decdn_node_manual_bin_src and decdn_cli_manual_bin_src, or neither —
+          exactly one was given, an incomplete override that would silently leave the
+          other binary to decdn_release_target_dir.
+
+    - name: Build the candidate source directories
+      ansible.builtin.set_fact:
+        _decdn_bin_candidate_dirs: >-
+          {{
+            [decdn_release_target_dir,
+             (decdn_release_target_dir | dirname) ~ '/' ~ decdn_node_target ~ '/release']
+            if _decdn_use_dir | bool else []
+          }}
+
+    - name: Stat the daemon binary in each candidate directory
+      ansible.builtin.stat:
+        path: "{{ item }}/decdn-node"
+      register: _decdn_node_bin_stat
+      loop: "{{ _decdn_bin_candidate_dirs }}"
+      loop_control:
+        label: "{{ item }}"
+      delegate_to: localhost
+      become: false
+      when: _decdn_use_dir | bool
+
+    - name: Stat the CLI binary in each candidate directory
+      ansible.builtin.stat:
+        path: "{{ item }}/decdn"
+      register: _decdn_cli_bin_stat
+      loop: "{{ _decdn_bin_candidate_dirs }}"
+      loop_control:
+        label: "{{ item }}"
+      delegate_to: localhost
+      become: false
+      when: _decdn_use_dir | bool
+
+    # Pick the first candidate holding BOTH binaries (order-preserving), so a plain
+    # target/release with only decdn-node correctly yields to a cross dir that has
+    # the pair, and a dir with the daemon but no CLI is not chosen.
+    - name: Select the first candidate directory that holds both binaries
+      ansible.builtin.set_fact:
+        _decdn_resolved_dir: >-
+          {{ (_decdn_bin_candidate_dirs
+              | select('in', _decdn_node_bin_stat.results | selectattr('stat.exists') | map(attribute='item') | list)
+              | select('in', _decdn_cli_bin_stat.results | selectattr('stat.exists') | map(attribute='item') | list)
+              | first) | default('', true) }}
+      when: _decdn_use_dir | bool
+
+    - name: Fail if no candidate directory held both binaries
+      ansible.builtin.assert:
+        that:
+          - _decdn_resolved_dir | length > 0
+        fail_msg: >-
+          No decdn-node + decdn pair found under decdn_release_target_dir
+          ({{ decdn_release_target_dir }}) or its cross output dir
+          ({{ (decdn_release_target_dir | dirname) ~ '/' ~ decdn_node_target ~ '/release' }}).
+          Build both — `cargo build --release` (native Linux control machine) or,
+          cross-compiling for the node, `cross build --release --target {{ decdn_node_target }}`.
+      when: _decdn_use_dir | bool
+
+    # Derive the effective source paths from the resolved dir in dir-mode; otherwise
+    # keep the operator's explicit paths (which the copy step reads unchanged).
+    - name: Derive the effective binary source paths
+      ansible.builtin.set_fact:
+        decdn_node_manual_bin_src: >-
+          {{ (_decdn_resolved_dir ~ '/decdn-node')
+             if _decdn_use_dir | bool else decdn_node_manual_bin_src }}
+        decdn_cli_manual_bin_src: >-
+          {{ (_decdn_resolved_dir ~ '/decdn')
+             if _decdn_use_dir | bool else decdn_cli_manual_bin_src }}
+
+    # Format validation applies ONLY to dir-mode — the footgun is "I ran `cargo build
+    # --release` into target/release and pointed the dir here" on a non-Linux box.
+    # Explicit paths are trusted verbatim (see the precedence note above). argv + `--`
+    # keeps the check robust for paths with spaces or a leading dash.
+    - name: Identify the on-disk format of both resolved binaries
+      ansible.builtin.command:
+        argv:
+          - file
+          - -b
+          - --
+          - "{{ item }}"
+      register: _decdn_bin_file
+      changed_when: false
+      loop:
+        - "{{ decdn_node_manual_bin_src }}"
+        - "{{ decdn_cli_manual_bin_src }}"
+      loop_control:
+        label: "{{ item }}"
+      delegate_to: localhost
+      become: false
+      when: _decdn_use_dir | bool
+
+    - name: Require both binaries to be ELF for the node's target architecture
+      ansible.builtin.assert:
+        that:
+          - "'ELF' in item.stdout"
+          - _decdn_expect_arch in item.stdout
+        fail_msg: >-
+          {{ item.item }} is "{{ item.stdout }}" — not an ELF {{ _decdn_expect_arch }}
+          binary for {{ decdn_node_target }}. A host-native build on a non-Linux control
+          machine (e.g. `cargo build --release` on macOS) produces a Mach-O the node
+          cannot exec. Cross-compile for the node: `cross build --release --target
+          {{ decdn_node_target }}` (outputs to target/{{ decdn_node_target }}/release),
+          or `cargo build --release --target {{ decdn_node_target }}`.
+        quiet: true
+      loop: "{{ _decdn_bin_file.results | default([]) }}"
+      loop_control:
+        label: "{{ item.item }}"
+      vars:
+        _decdn_arch_file_tokens:
+          x86_64-unknown-linux-gnu: x86-64
+          aarch64-unknown-linux-gnu: aarch64
+        _decdn_expect_arch: "{{ _decdn_arch_file_tokens[decdn_node_target] | default('x86-64') }}"
+      when: _decdn_use_dir | bool
 
 - name: Validate decdn_node_generate_keystore is a real boolean
   ansible.builtin.assert:

ansible/roles/decdn_node/templates/decdn-node.service.j2

@@ -34,9 +34,15 @@ ExecStart={{ decdn_bin }} run \
 # diff touched, so reload is an operator's deliberate call.
 ExecReload=/bin/kill -HUP $MAINPID
 
-# Graceful stop so the node can drain/flush. SIGTERM is its shutdown signal.
+# Graceful stop so the node can drain/flush. SIGTERM is its shutdown signal, and
+# it enters the SAME teardown as `decdn node drain`: it stops accepting work, awaits
+# in-flight streams to zero, then flushes settlement + the receipt/audit tail before
+# exiting. The process exits the instant that drain completes, so systemd proceeds
+# immediately — TimeoutStopSec is only the CAP. Keep it generous: a SIGKILL at the
+# cap would truncate an in-flight drain, dropping paid deliveries and possibly the
+# audit tail (a money-losing cut). Hence the 300s default, not systemd's 90s norm.
 KillSignal=SIGTERM
-TimeoutStopSec=30
+TimeoutStopSec={{ decdn_node_stop_timeout_sec }}
 
 Restart=always
 RestartSec=5