From c23d062957a0459668cc75d0b9e79d486af74483 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 7 Oct 2026 21:40:51 +0300 Subject: [PATCH 1/6] feat(sponsord): refuse restarts while a pool top-up is held, add sponsord_log_level A restart forgets sponsord's hold on an unconfirmed top-up (upstream #41), and the pool can then be refilled twice. The role now reads /metrics before it flushes a pending restart and fails while the hold is on, leaving the restart for the next run; sponsord_restart_ignore_topup_hold overrides it. Also renders sponsord_log_level as RUST_LOG. Co-Authored-By: Claude Opus 5.5 --- .../molecule/sponsord-install/molecule.yml | 1 + ansible/molecule/sponsord/files/sponsord-stub | 16 ++- ansible/molecule/sponsord/molecule.yml | 1 + ansible/molecule/sponsord/side_effect.yml | 109 ++++++++++++++++++ ansible/molecule/sponsord/verify.yml | 1 + .../molecule/validation-sponsord/converge.yml | 29 ++++- ansible/roles/sponsord/defaults/main.yml | 40 +++++++ ansible/roles/sponsord/tasks/main.yml | 37 +++++- ansible/roles/sponsord/tasks/topup-hold.yml | 71 ++++++++++++ .../roles/sponsord/templates/sponsord.env.j2 | 1 + 10 files changed, 298 insertions(+), 8 deletions(-) create mode 100644 ansible/roles/sponsord/tasks/topup-hold.yml diff --git a/ansible/molecule/sponsord-install/molecule.yml b/ansible/molecule/sponsord-install/molecule.yml index 1da2d21..3367d08 100644 --- a/ansible/molecule/sponsord-install/molecule.yml +++ b/ansible/molecule/sponsord-install/molecule.yml @@ -66,6 +66,7 @@ provisioner: sponsord_rpc_url: "https://rpc.example.invalid/?key=$$HOME$$(touch /tmp/sponsord-rpc-expanded)&q=o'brien \"x\"" sponsord_max_ttl_secs: 3600 sponsord_pool_low_water_micro_usdc: 0 + sponsord_log_level: "sponsord=debug,info" sponsord_readiness_retries: 10 sponsord_readiness_delay: 1 decdn_grafana_cloud_enabled: true diff --git a/ansible/molecule/sponsord/files/sponsord-stub b/ansible/molecule/sponsord/files/sponsord-stub index f34538f..6ba7f54 100755 --- a/ansible/molecule/sponsord/files/sponsord-stub +++ b/ansible/molecule/sponsord/files/sponsord-stub @@ -37,6 +37,10 @@ from http.server import BaseHTTPRequestHandler, HTTPServer VERSION = "sponsord 0.0.0-molecule-stub" UNHEALTHY_MARKER = "/etc/sponsord/stub-unhealthy" +# While this file exists, /metrics reports a held pool top-up since the unix time +# it holds (the real daemon's sponsord_pool_topup_unconfirmed_since_unix), so the +# role's restart guard can be exercised without a chain. +HOLD_MARKER = "/etc/sponsord/stub-topup-hold" # Must match sponsord_rpc_url in the scenario's inventory, which carries shell # metacharacters and quotes on purpose. EXPECTED_RPC_URL = "https://rpc.example.invalid/?key=$HOME$(touch /tmp/sponsord-rpc-expanded)&q=o'brien \"x\"" @@ -104,7 +108,17 @@ class Handler(BaseHTTPRequestHandler): if self.path == "/healthz": body, ctype = b'{"ok":true}', "application/json" elif self.path == "/metrics": - body, ctype = b"sponsord_pool_topups_total 0\n", "text/plain; version=0.0.4" + try: + with open(HOLD_MARKER, "rb") as f: + held = int(f.read().strip() or b"0") + except OSError: + held = 0 + body = ( + b"sponsord_pool_topups_total 0\n" + b"# TYPE sponsord_pool_topup_unconfirmed_since_unix gauge\n" + + f"sponsord_pool_topup_unconfirmed_since_unix {held}\n".encode() + ) + ctype = "text/plain; version=0.0.4" elif self.path == "/v1/info": if self.headers.get("Authorization", "").encode() != b"Bearer " + API_TOKEN: self.send_error(401) diff --git a/ansible/molecule/sponsord/molecule.yml b/ansible/molecule/sponsord/molecule.yml index 1d5bb72..58d4dfd 100644 --- a/ansible/molecule/sponsord/molecule.yml +++ b/ansible/molecule/sponsord/molecule.yml @@ -74,6 +74,7 @@ provisioner: sponsord_rpc_url: "https://rpc.example.invalid/?key=$$HOME$$(touch /tmp/sponsord-rpc-expanded)&q=o'brien \"x\"" sponsord_max_ttl_secs: 3600 sponsord_pool_low_water_micro_usdc: 0 + sponsord_log_level: "sponsord=debug,info" sponsord_readiness_retries: 10 sponsord_readiness_delay: 1 decdn_grafana_cloud_enabled: true diff --git a/ansible/molecule/sponsord/side_effect.yml b/ansible/molecule/sponsord/side_effect.yml index 8824087..d8f3ac6 100644 --- a/ansible/molecule/sponsord/side_effect.yml +++ b/ansible/molecule/sponsord/side_effect.yml @@ -10,6 +10,10 @@ # sponsord_rpc_url must not adopt the role's own file; a handed-over file is # recorded `host` and locked to 0600; a host edit restarts the daemon; an # inventory URL must not overwrite the host's file until confirmed. +# 4. The top-up hold guard: while /metrics reports a held pool top-up, a converge +# that would restart the daemon fails before the restart, keeps the old PID +# and the old restart-inputs record, and the next converge after the hold +# clears applies the change; sponsord_restart_ignore_topup_hold overrides it. # Release mode and treasury wallet generation run in ../sponsord-install, beside # this scenario. The first play must run on a fresh converge (it asserts no restart # yet), and it records the API token verify.yml compares against. @@ -282,3 +286,108 @@ register: _sp_edit_gone changed_when: false failed_when: _sp_edit_gone.rc != 1 + +- name: Side effect — the top-up hold guard + hosts: all + become: true + vars: + _sp_inputs: /etc/sponsord/.inputs.sha256 + tasks: + - name: Read the unit state before the hold + ansible.builtin.systemd_service: + name: sponsord + register: _sp_hold_before + + - name: Hash the restart-inputs record before the hold + ansible.builtin.stat: + path: "{{ _sp_inputs }}" + checksum_algorithm: sha256 + register: _sp_hold_rec_before + + - name: Make the stub report a held top-up + ansible.builtin.copy: + dest: /etc/sponsord/stub-topup-hold + content: "1700000000\n" + mode: "0644" + + # A knob change re-renders sponsord.env, so a restart is pending. + - name: Converge a change while the top-up is held + block: + - name: Run the role with a changed TTL + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_max_ttl_secs: 7200 + + - name: Fail — the guard let a restart through during a hold + ansible.builtin.fail: + msg: The role finished although /metrics reported a held top-up. + rescue: + - name: Require the failure to come from the hold guard + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Refuse to restart or stop sponsord while a pool top-up is held' + fail_msg: >- + Expected the role to fail at the hold guard, but it failed at + "{{ ansible_failed_task.name }}". + + # Read before anything flushes the handler the failed run left queued. + - name: Read the unit state after the refused converge + ansible.builtin.systemd_service: + name: sponsord + register: _sp_hold_refused + + - name: Hash the restart-inputs record after the refused converge + ansible.builtin.stat: + path: "{{ _sp_inputs }}" + checksum_algorithm: sha256 + register: _sp_hold_rec_refused + + - name: Require no restart and no new record during the hold + ansible.builtin.assert: + that: + - _sp_hold_refused.status.MainPID == _sp_hold_before.status.MainPID + - _sp_hold_rec_refused.stat.checksum == _sp_hold_rec_before.stat.checksum + fail_msg: >- + The refused converge still restarted sponsord (PID + {{ _sp_hold_before.status.MainPID }} -> {{ _sp_hold_refused.status.MainPID }}) + or rewrote {{ _sp_inputs }}, so the next run would not retry the restart. + + - name: Converge the same change with the override + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_max_ttl_secs: 7200 + sponsord_restart_ignore_topup_hold: true + + - name: Read the unit state after the override + ansible.builtin.systemd_service: + name: sponsord + register: _sp_hold_override + + - name: Require the override to have restarted sponsord + ansible.builtin.assert: + that: + - _sp_hold_override.status.MainPID != _sp_hold_refused.status.MainPID + fail_msg: sponsord_restart_ignore_topup_hold=true did not restart sponsord. + + - name: Clear the hold + ansible.builtin.file: + path: /etc/sponsord/stub-topup-hold + state: absent + + # Back on the inventory's TTL, which verify.yml checks. + - name: Converge once the hold has cleared + ansible.builtin.include_role: + name: sponsord + + - name: Read the unit state after the hold cleared + ansible.builtin.systemd_service: + name: sponsord + register: _sp_hold_cleared + + - name: Require the plain converge to restart once the hold cleared + ansible.builtin.assert: + that: + - _sp_hold_cleared.status.MainPID != _sp_hold_override.status.MainPID + fail_msg: The converge after the hold cleared did not restart sponsord. diff --git a/ansible/molecule/sponsord/verify.yml b/ansible/molecule/sponsord/verify.yml index 73108e9..1a5690f 100644 --- a/ansible/molecule/sponsord/verify.yml +++ b/ansible/molecule/sponsord/verify.yml @@ -84,6 +84,7 @@ # Set knobs are emitted (0 included), unset ones are not. - "'SPONSORD_MAX_TTL_SECS=3600' in _sp_env_text" - "'SPONSORD_POOL_LOW_WATER_MICRO_USDC=0' in _sp_env_text" + - "'RUST_LOG=sponsord=debug,info' in _sp_env_text" - "'SPONSORD_MAX_SPENDING_CAP' not in _sp_env_text" - "'SPONSORD_POOL_REFILL' not in _sp_env_text" - "'RPC_URL' not in _sp_env_text" diff --git a/ansible/molecule/validation-sponsord/converge.yml b/ansible/molecule/validation-sponsord/converge.yml index 2066d1a..5e64700 100644 --- a/ansible/molecule/validation-sponsord/converge.yml +++ b/ansible/molecule/validation-sponsord/converge.yml @@ -216,6 +216,32 @@ decdn_rejected: "{{ decdn_rejected + ['sponsord-bool'] }}" when: ansible_failed_task.name == "Validate the role's boolean knobs" + - name: "Case sponsord-bool — a quoted hold override" + block: + - name: Run sponsord with a quoted hold override + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_restart_ignore_topup_hold: "true" + rescue: + - name: Record sponsord-bool rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-bool'] }}" + when: ansible_failed_task.name == "Validate the role's boolean knobs" + + - name: "Case sponsord-log-level — a log level with a space" + block: + - name: Run sponsord with a log level holding a space + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_log_level: "info sponsord=debug" + rescue: + - name: Record sponsord-log-level rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-log-level'] }}" + when: ansible_failed_task.name == "Validate sponsord_log_level" + - name: "Case sponsord-rpc-newline — an RPC URL that injects a second variable" block: - name: Run sponsord with an RPC URL that injects a second variable @@ -1146,7 +1172,8 @@ ["sponsord-release-version", "sponsord-manual-src", "sponsord-network-unknown", "sponsord-network-chain", "sponsord-pool-id", "sponsord-pool-id-empty", "sponsord-wallet-bool", "sponsord-wallet-cli", "sponsord-zero-address", "sponsord-bind", - "sponsord-limit", "sponsord-limit", "sponsord-bool", "sponsord-rpc-newline", + "sponsord-limit", "sponsord-limit", "sponsord-bool", "sponsord-bool", + "sponsord-log-level", "sponsord-rpc-newline", "sponsord-rpc-gate", "sponsord-treasury", "sponsord-treasury", "sponsord-token", "sponsord-dir-missing", "sponsord-dir-format", "sponsord-dir-cross", "sponsord-bind", "sponsord-overwrite-untracked", "sponsord-token", diff --git a/ansible/roles/sponsord/defaults/main.yml b/ansible/roles/sponsord/defaults/main.yml index 6a75119..009a703 100644 --- a/ansible/roles/sponsord/defaults/main.yml +++ b/ansible/roles/sponsord/defaults/main.yml @@ -103,6 +103,12 @@ sponsord_pool_low_water_micro_usdc: "" # [20000000 = $20] top up below this sponsord_pool_refill_micro_usdc: "" # [100000000 = $100] each top-up adds this sponsord_pool_watch_interval_secs: "" # [3600] how often the keeper checks the pool +# --- Logging ---------------------------------------------------------------------- +# RUST_LOG for the daemon: a level (info) or tracing directives (sponsord=debug, +# hyper=warn). "" omits it, and the daemon then logs errors only; info adds the +# listener and each top-up. No spaces or quotes. +sponsord_log_level: "" + # --- Secrets on the host -------------------------------------------------------- # Root-owned under sponsord_etc: the secret files 0600, sponsord.env 0644 (it holds # no secret). systemd reads the API token, treasury keystore and password as root @@ -159,6 +165,17 @@ sponsord_bin: /usr/local/bin/sponsord # is the cap before SIGKILL. sponsord_stop_timeout_sec: 120 +# --- Top-up hold ------------------------------------------------------------------ +# When the keeper broadcasts a pool top-up and cannot read its receipt, it holds every +# further top-up until that transaction mines or provably never can +# (sponsord_pool_topup_unconfirmed_since_unix > 0 on /metrics). A restart forgets the +# hold, and if the transaction is still pending the pool can be refilled twice +# (upstream docs/operator.md, "Monitor"). So the role reads /metrics before any +# restart or stop it is about to cause, and fails instead while a hold is on; the +# next converge retries once it clears. true restarts anyway: only for a hold you +# have confirmed can never mine. +sponsord_restart_ignore_topup_hold: false + # --- Readiness -------------------------------------------------------------------- # The daemon binds only after it has decrypted the keystore and confirmed on-chain # that the treasury owns sponsord_pool_id, so /healthz answering 200 is the role's @@ -166,3 +183,26 @@ sponsord_stop_timeout_sec: 120 # probe, a timeout here FAILS the deploy. Window = retries x delay (~60s). sponsord_readiness_retries: 30 sponsord_readiness_delay: 2 + +# --- Lifecycle: backup + decommission (tasks_from: backup / decommission) ------- +# Only read by those entry points (playbooks/backup.yml, playbooks/decommission.yml); +# a normal deploy ignores them. See docs/lifecycle.md. +# +# The daemon keeps no state, so a backup is the credentials under sponsord_etc: the +# treasury keystore and password (the pool's owner, holding its funds), the API +# token, and the onramp's Turnstile secret when present. It is taken hot and +# encrypted on the host to these PUBLIC keys (age1… recipients and/or ssh-ed25519 / +# ssh-rsa lines), so no plaintext key leaves the host. REQUIRED for a backup; by +# default the node's list, so one inventory list covers a co-located host. +sponsord_backup_age_recipients: "{{ decdn_backup_age_recipients | default([]) }}" +sponsord_backup_include_env: false # also archive secret.env (the RPC URL, maybe an API key) +sponsord_backup_dir: /var/backups/sponsord # on the host, root 0700 +sponsord_backup_fetch: true # copy the ENCRYPTED archive (kilobytes) to the control machine +sponsord_backup_local_dir: "{{ playbook_dir }}/../backups" # control machine; git-ignored in this repo +# +# Shared with decdn_node (and identical there, `make test-scripts` checks): one typed +# confirmation covers every service on the host. A non-interactive run sets +# decdn_decommission_confirm to the comma-joined, sorted host list (deliberately NOT +# defaulted here). +decdn_decommission_max_hosts: 1 +decdn_decommission_prompt_seconds: 300 # an unanswered confirmation prompt fails closed after this diff --git a/ansible/roles/sponsord/tasks/main.yml b/ansible/roles/sponsord/tasks/main.yml index 4bad357..53abd4c 100644 --- a/ansible/roles/sponsord/tasks/main.yml +++ b/ansible/roles/sponsord/tasks/main.yml @@ -313,9 +313,22 @@ that: - sponsord_generate_api_token is boolean - sponsord_secret_env_overwrite_host_file is boolean + - sponsord_restart_ignore_topup_hold is boolean fail_msg: >- - sponsord_generate_api_token and sponsord_secret_env_overwrite_host_file must be - booleans (true/false), not quoted strings. + sponsord_generate_api_token, sponsord_secret_env_overwrite_host_file and + sponsord_restart_ignore_topup_hold must be booleans (true/false), not quoted + strings. + +# RUST_LOG is a level or comma-separated tracing directives. It lands unquoted in an +# EnvironmentFile, so whitespace, quotes and control characters are refused. +- name: Validate sponsord_log_level + ansible.builtin.assert: + that: + - sponsord_log_level | string is match('^[A-Za-z0-9_=,.:-]*\Z') + fail_msg: >- + sponsord_log_level must be "" (errors only), a level such as info, or tracing + directives such as sponsord=debug,hyper=warn (letters, digits and _=,.:- + only), got "{{ sponsord_log_level }}". # --- RPC URL: inventory or host-provisioned secret.env ------------------------------ # The same two authoring paths, provenance record and guards as decdn_node's @@ -774,6 +787,15 @@ | map(attribute='checksum', default='') | join(' ') }} sponsord_inputs_recorded_sums: "{{ sponsord_inputs_recorded.content | default('') | b64decode | trim }}" +# Every notify source above (env files, unit, binary, secret.env) is also a hashed +# input, so this one test covers all of them; the hold guard below reads it too. +- name: Decide whether a sponsord restart is pending + ansible.builtin.set_fact: + _sponsord_restart_pending: >- + {{ (sponsord_inputs_recorded_sums | length > 0 and sponsord_inputs_sums != sponsord_inputs_recorded_sums) + or (sponsord_inputs_recorded_sums | length == 0 + and sponsord_is_active.status.ActiveState | default('') == 'active') }} + - name: Note restart inputs that changed since the daemon last started healthy ansible.builtin.debug: msg: >- @@ -784,10 +806,13 @@ — restarting sponsord. changed_when: true notify: Restart sponsord - when: >- - (sponsord_inputs_recorded_sums | length > 0 and sponsord_inputs_sums != sponsord_inputs_recorded_sums) - or (sponsord_inputs_recorded_sums | length == 0 - and sponsord_is_active.status.ActiveState | default('') == 'active') + when: _sponsord_restart_pending | bool + +# Fails, before the handler runs and before the inputs record is written, while the +# daemon holds an unconfirmed pool top-up: the restart is retried on the next run. +- name: Refuse a restart that would forget a held pool top-up + ansible.builtin.include_tasks: topup-hold.yml + when: _sponsord_restart_pending | bool - name: Apply pending sponsord changes ansible.builtin.meta: flush_handlers diff --git a/ansible/roles/sponsord/tasks/topup-hold.yml b/ansible/roles/sponsord/tasks/topup-hold.yml new file mode 100644 index 0000000..431d0d9 --- /dev/null +++ b/ansible/roles/sponsord/tasks/topup-hold.yml @@ -0,0 +1,71 @@ +--- +# Refuse to restart or stop a sponsord that holds an unconfirmed pool top-up. +# +# The keeper broadcast a topUp it could not read a receipt for, and sends no +# further top-up until that transaction mines or provably never can. The hold +# lives only in the process: a restart forgets it, and if the transaction is still +# pending somewhere the pool can be refilled twice (upstream docs/operator.md, +# "Monitor"). Included before the role flushes a pending restart (main.yml), and +# by the decommission entry points before they stop the daemon. +# +# A daemon that is not running holds nothing. A running one whose /metrics does +# not answer is refused: the hold state is unknown. A binary that predates the +# hold (no such gauge) passes, as there is nothing it could forget. +- name: Read whether sponsord is running before touching it + ansible.builtin.systemd_service: + name: sponsord + register: _sponsord_hold_unit + +- name: Check sponsord for a held pool top-up + when: + - not ansible_check_mode + - _sponsord_hold_unit.status.ActiveState | default('') == 'active' + block: + # KICS flags plain HTTP here; the target is the loopback listener the role asserts. + - name: Read sponsord /metrics + ansible.builtin.uri: + url: "http://{{ sponsord_bind_address }}:{{ sponsord_port }}/metrics" + return_content: true + timeout: 5 + register: _sponsord_hold_metrics + changed_when: false + failed_when: false + + - name: Parse the top-up hold gauge + ansible.builtin.set_fact: + _sponsord_hold_since: >- + {{ (_sponsord_hold_metrics.content | default('') + | regex_search('(?m)^sponsord_pool_topup_unconfirmed_since_unix\s+([0-9.eE+]+)\s*$', '\1') + | default([], true) | first | default('0')) | float | int }} + + - name: Refuse to proceed while the hold state cannot be read + ansible.builtin.assert: + that: + - _sponsord_hold_metrics.status | default(-1) == 200 + fail_msg: >- + sponsord is running but http://{{ sponsord_bind_address }}:{{ sponsord_port }}/metrics + did not answer 200 (status {{ _sponsord_hold_metrics.status | default('none') }}), + so whether it holds an unconfirmed pool top-up is unknown, and a restart or + stop could forget one. Find out why /metrics does not answer, or set + sponsord_restart_ignore_topup_hold=true once you have confirmed that no + top-up is pending (the treasury has no pending transaction). + when: not (sponsord_restart_ignore_topup_hold | bool) + + - name: Refuse to restart or stop sponsord while a pool top-up is held + ansible.builtin.fail: + msg: |- + sponsord holds an unconfirmed pool top-up (since {{ _since }}). + A restart or stop now forgets the hold, and if that transaction is still + pending the pool can be refilled twice. Nothing was restarted or stopped, + and a deploy's changed files wait on disk for the next run. + The hold clears by itself once the transaction mines. If it was dropped, + send 0-value transactions from the treasury to itself until the hold + clears; the daemon's error log names the transaction and how many it takes: + sudo journalctl -u sponsord -e | grep -i top + Then re-run. Only if you have confirmed the transaction can never mine, + re-run with sponsord_restart_ignore_topup_hold=true. + vars: + _since: "{{ '%Y-%m-%dT%H:%M:%SZ' | strftime(_sponsord_hold_since | int, utc=true) }}" + when: + - _sponsord_hold_since | int > 0 + - not (sponsord_restart_ignore_topup_hold | bool) diff --git a/ansible/roles/sponsord/templates/sponsord.env.j2 b/ansible/roles/sponsord/templates/sponsord.env.j2 index 3f8a848..e65ffa1 100644 --- a/ansible/roles/sponsord/templates/sponsord.env.j2 +++ b/ansible/roles/sponsord/templates/sponsord.env.j2 @@ -11,6 +11,7 @@ SPONSORD_POOL_ID={{ sponsord_pool_id }} ('SPONSORD_POOL_LOW_WATER_MICRO_USDC', sponsord_pool_low_water_micro_usdc), ('SPONSORD_POOL_REFILL_MICRO_USDC', sponsord_pool_refill_micro_usdc), ('SPONSORD_POOL_WATCH_INTERVAL_SECS', sponsord_pool_watch_interval_secs), + ('RUST_LOG', sponsord_log_level), ] if value | string | length > 0 %} {{ name }}={{ value }} {% endfor %} From 74df958ddd181c80f42e0a553cde6b494fff7c59 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 7 Oct 2026 21:40:53 +0300 Subject: [PATCH 2/6] feat(ansible): back up and decommission sponsord hosts backup.yml archives sponsord's credentials (treasury keystore and password, API token, Turnstile secret), encrypted on the host. decommission.yml checks every sponsord host for a held top-up, then removes the node, the onramp (and the role's Caddy) and sponsord, with one typed confirmation per host, and tears down Alloy on either kind of host. Covered by molecule/sponsord-onramp-lifecycle. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 20 +- ansible/Makefile | 2 +- ansible/README.md | 17 +- ansible/galaxy/CHANGELOG.md | 26 +++ .../sponsord-onramp-lifecycle/molecule.yml | 93 +++++++++ .../sponsord-onramp-lifecycle/side_effect.yml | 127 ++++++++++++ .../sponsord-onramp-lifecycle/verify.yml | 100 ++++++++++ ansible/playbooks/backup.yml | 23 ++- ansible/playbooks/decommission.yml | 53 ++++- .../roles/decdn_node/tasks/decommission.yml | 11 +- ansible/roles/sponsord/README.md | 43 +++- ansible/roles/sponsord/tasks/backup.yml | 188 ++++++++++++++++++ ansible/roles/sponsord/tasks/decommission.yml | 114 +++++++++++ ansible/roles/sponsord_onramp/README.md | 13 +- .../roles/sponsord_onramp/defaults/main.yml | 9 + .../sponsord_onramp/tasks/decommission.yml | 119 +++++++++++ docs/lifecycle.md | 50 ++++- tests/scripts-test.sh | 15 +- 18 files changed, 984 insertions(+), 39 deletions(-) create mode 100644 ansible/molecule/sponsord-onramp-lifecycle/molecule.yml create mode 100644 ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml create mode 100644 ansible/molecule/sponsord-onramp-lifecycle/verify.yml create mode 100644 ansible/roles/sponsord/tasks/backup.yml create mode 100644 ansible/roles/sponsord/tasks/decommission.yml create mode 100644 ansible/roles/sponsord_onramp/tasks/decommission.yml diff --git a/AGENTS.md b/AGENTS.md index e8cb679..baec403 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,7 +58,7 @@ mirror (`vars/main/networks.yml`), never hand-copied into inventory or docs. ``` ansible/ # the deployment project (DevSec-hardened, lean roles) - playbooks/ # site.yml (decdn node + sponsord), sponsord.yml (+ onramp), backup.yml, decommission.yml + playbooks/ # site.yml (decdn node + sponsord), sponsord.yml (+ onramp), backup.yml, decommission.yml (node + sponsord) roles/ # baseline, decdn_node, grafana_alloy, sponsord, sponsord_onramp inventory/ galaxy/ molecule/ # see ansible/README.md cloud-init/ # user-data{,-sponsord}.yaml + on-host bootstrap.sh; pinned ansible-core/collections (see its README.md) @@ -88,7 +88,10 @@ holds the deCDN Grafana dashboards and Prometheus alert rules; upstream ships no `decdn_*` series a panel or rule names must be one `decdn-node` exports — check the node's `/metrics` and upstream's `adr/appendix-observability.md` registry (a `planned` row emits nothing). Nothing in CI checks the names, so a typo renders `(no data)` or a -rule that never fires. See `.claude/skills/grafana-dashboards/SKILL.md`. +rule that never fires. See `.claude/skills/grafana-dashboards/SKILL.md`. sponsord's pair +lives in its `sponsord/` subdirectory, which the chart does not render; its `sponsord_*` +names must be ones upstream `decdn/sponsord` `crates/sponsord/src/metrics.rs` exports, +and its rules have promtool unit tests (`charts/decdn-node/tests/sponsord-alerts_test.yml`). ## Current services @@ -125,7 +128,10 @@ rule that never fires. See `.claude/skills/grafana-dashboards/SKILL.md`. `playbooks/backup.yml` / `decommission.yml` (`make backup` / `make decommission`) are role entry points (`tasks_from: backup|decommission`): backups are encrypted on the host to operator public keys; decommission needs `LIMIT` + typed confirmation, keeps the - identity and never touches the chain. + identity and never touches the chain. Both also cover `sponsord_hosts` (and the onramp), + with one confirmation per host (`_decdn_decommission_confirmed`; the + `decdn_decommission_*` cap and timeout are copied into all three roles' defaults, + `make test-scripts` checks). Leaving `decdn_rpc_url` empty means the operator wrote `0600 /etc/decdn/decdn.env` on the host and the role only gates on it, so no secret transits the control machine. See `ansible/README.md`. (On-chain node stake/registration, ADR 019 Phase 2, is a manual operator step, driven by `decdn setup`.) @@ -186,6 +192,14 @@ rule that never fires. See `.claude/skills/grafana-dashboards/SKILL.md`. - **Config gate:** sponsord has no `config validate` and only env/flag config, so `/healthz` after start is the gate, and it is **fatal**. The daemon binds only after the keystore decrypts and the on-chain pool-owner check passes. + - **Top-up hold guard:** a restart or stop forgets a held, unconfirmed pool top-up + (`sponsord_pool_topup_unconfirmed_since_unix` > 0), and the pool can be refilled + twice. `tasks/topup-hold.yml` reads `/metrics` before the role flushes a pending + restart and before decommission stops the daemon, and fails while a hold is on + (or `/metrics` does not answer). Every restart must go through it: a new notify + source must also be a hashed restart input, which is what "pending" reads. + `sponsord_restart_ignore_topup_hold` overrides it. Compose cannot guard itself; + its README has the manual check. - **Alloy toggles:** `playbooks/group_vars/all.yml` derives `grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled` from group membership. They are host-scoped so a co-located host's two plays render one diff --git a/ansible/Makefile b/ansible/Makefile index d889c1b..277e1e5 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -164,7 +164,7 @@ decommission: SCENARIOS ?= all JOBS ?= SLOW_FIRST ?= cloud-init-sponsord cloud-init sponsord-onramp validation \ - validation-sponsord sponsord-onramp-caddy sponsord-install sponsord \ + validation-sponsord sponsord-onramp-caddy sponsord-onramp-lifecycle sponsord-install sponsord \ validation-alloy os-matrix grafana-cloud MOLECULE_LOCK_PREFIX ?= /tmp/decdn-devops-molecule molecule_env = SCENARIOS='$(SCENARIOS)' JOBS='$(JOBS)' SLOW_FIRST='$(SLOW_FIRST)' \ diff --git a/ansible/README.md b/ansible/README.md index ddce589..64b2a41 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -6,8 +6,8 @@ hardened host baseline. The roles also ship as the `decdn.node` Galaxy collectio | Playbook | Purpose | Make target | |----------|---------|-------------| | **`site.yml`** | Harden the host and deploy the public **deCDN node** (`decdn-node`). | `make check` / `make deploy` | -| `backup.yml` | Encrypted backup of a node's identity or full state. | `make backup` | -| `decommission.yml` | Stop a node and remove its service (keeps the identity; no on-chain steps). | `make decommission` | +| `backup.yml` | Encrypted backup of a node's identity or full state, and of sponsord's credentials. | `make backup` | +| `decommission.yml` | Stop a node, the onramp and sponsord and remove their services (keeps the keys; no on-chain steps). | `make decommission` | | `sponsord.yml` | Harden the host and deploy **sponsord**, the onboarding sponsor, on hosts in `sponsord_hosts`, with or without a node; then its public **sponsord-onramp** (behind Caddy) on hosts in `sponsord_onramp_hosts`. Also run by `site.yml`. | `make check-sponsord` / `make deploy-sponsord` | ``` @@ -301,8 +301,11 @@ make backup LIMIT= ANSIBLE_ARGS='-e decdn_backup_scope=full' # full state make decommission LIMIT= # typed confirmation; keeps the identity ``` -Backups need `decdn_backup_age_recipients` (public keys). Restore, host migration and -the on-chain exit: [`docs/lifecycle.md`](../docs/lifecycle.md). +Backups need `decdn_backup_age_recipients` (public keys); on `sponsord_hosts` they +archive the treasury keystore and password, the API token and the Turnstile secret. +On a sponsord host, decommission refuses while sponsord holds an unconfirmed pool +top-up. Restore, host migration and the on-chain exit: +[`docs/lifecycle.md`](../docs/lifecycle.md). --- @@ -333,7 +336,7 @@ git-ignored inventory instead: role-authored env file + provenance record), itself is Debian 12), `lifecycle` (a `decdn_network` profile with an override, both backup scopes decrypted and checked, a rejected and a real decommission), `sponsord` (`playbooks/sponsord.yml` on a host with no node: generated token, credential -rotation restart, the fatal `/healthz` gate, Alloy scraping sponsord and not the node; +rotation restart, the fatal `/healthz` gate, the top-up hold guard, Alloy scraping sponsord and not the node; `grafana-cloud` covers sponsord co-located with a node), `sponsord-install` (the same converge, then release mode against a signed loopback mirror and treasury wallet generation), `sponsord-onramp` (the onramp beside its daemon, behind the real distro @@ -341,7 +344,9 @@ Caddy with `tls internal`: the https chain, the client address Caddy passes on, Turnstile rotation restart, the fatal gates), `sponsord-onramp-caddy` (the same converge, then ACME and `none` modes and a refused foreign Caddyfile), `sponsord-onramp-source` (the same converge, then both sponsord roles built from a -fixture repo with `install_method: source`), `source-build` (the node built from source +fixture repo with `install_method: source`), `sponsord-onramp-lifecycle` (the same converge, then +sponsord's backup decrypted and checked, decommission refused for a held top-up and a +bad confirmation, then a real one), `source-build` (the node built from source on the host: the build sandbox, stamps, rollback, an untrusted earlier build home), and `cloud-init` and `cloud-init-sponsord` (the `cloud-init/` templates booted for real). A slow scenario is split along its side effects rather than allowed to set the wall diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index 7510059..600b9c0 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -184,6 +184,26 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and on Debian Ansible needs ACL support to hand the temp module file to that user. Without it both fail without an `rc`, which the role can only report after the fact ("becoming the unprivileged decdn user needs the `acl` package on this host"). +- `sponsord`: a top-up hold guard. When the keeper broadcasts a pool top-up and + cannot read its receipt, it holds further top-ups until that transaction mines + (upstream sponsord #41); a restart forgets the hold and the pool can be refilled + twice. Before the role restarts sponsord (any changed restart input) it reads + `/metrics`, and fails while `sponsord_pool_topup_unconfirmed_since_unix` is + non-zero or `/metrics` does not answer, leaving the restart for the next run. + `sponsord_restart_ignore_topup_hold` overrides it. +- `sponsord`: `sponsord_log_level`, rendered as `RUST_LOG` (a level or tracing + directives; `""` keeps the daemon's errors-only default). There is no onramp + equivalent: the onramp release ignores `RUST_LOG`. +- `sponsord`: `tasks_from: backup` (`sponsord_backup_*`): the treasury keystore and + password, the API token and the onramp's Turnstile secret (`secret.env` opt-in), + taken hot and encrypted on the host to `sponsord_backup_age_recipients` (default + `decdn_backup_age_recipients`), then fetched. +- `sponsord`, `sponsord_onramp`: `tasks_from: decommission`. The onramp's stops it, + removes its unit, and stops and disables the role's Caddy (marker-gated). + sponsord's refuses while a top-up is held, then stops sponsord and removes its + unit. Both keep `/etc/sponsord` and never touch the chain. They share + `decdn_decommission_max_hosts` / `_prompt_seconds` / `_confirm` with `decdn_node`, + and a host that confirmed in an earlier play is not asked again. ### Removed @@ -198,6 +218,12 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Changed +- `playbooks/backup.yml` and `playbooks/decommission.yml` also cover + `sponsord_hosts` (and `sponsord_onramp_hosts`): a backup of sponsord's + credentials; a decommission that first checks every sponsord host for a held + top-up, then removes the node, the onramp and sponsord, and tears down Alloy on + either kind of host. + - **Breaking:** `decdn_node_install_method`, `sponsord_install_method` and `sponsord_onramp_install_method` now default to `release` (was `manual`). An inventory that relied on the old default must set `manual` explicitly. Upstream diff --git a/ansible/molecule/sponsord-onramp-lifecycle/molecule.yml b/ansible/molecule/sponsord-onramp-lifecycle/molecule.yml new file mode 100644 index 0000000..8afcfb1 --- /dev/null +++ b/ansible/molecule/sponsord-onramp-lifecycle/molecule.yml @@ -0,0 +1,93 @@ +--- +# sponsord's lifecycle entry points, split from ../sponsord-onramp so the two run +# side by side: the same host shape, inventory and converge (that scenario's +# prepare and converge, by path), then side_effect.yml here: +# playbooks/backup.yml the treasury keystore + password, the API token and +# the Turnstile secret, encrypted on the host to a +# throwaway age key and fetched; +# decommission refusals a held pool top-up (the stub's hold marker), a +# missing confirmation, a wrong one: nothing touched; +# playbooks/decommission.yml for real: the onramp, its Caddy and sponsord stopped +# and removed, Alloy torn down, every credential kept. +# verify.yml here decrypts the archive and checks what decommission removed and kept. +# No idempotence step: every backup writes a new archive and decommission is one-shot. +# +# The inventory below must stay identical to ../sponsord-onramp/molecule.yml's +# (`make test-scripts` checks); the stubs are that scenario's files. +driver: + name: docker +platforms: + - name: onramp-lifecycle-molecule + # Same digest-pinned images as ../sponsord (repo convention). + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - sponsord_hosts + - sponsord_onramp_hosts +provisioner: + name: ansible + options: + skip-tags: baseline + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # Same pipelining note as ../default/molecule.yml. + ANSIBLE_PIPELINING: "true" + playbooks: + prepare: ../sponsord-onramp/prepare.yml + converge: ../sponsord-onramp/converge.yml + inventory: + group_vars: + sponsord_hosts: + sponsord_install_method: manual + sponsord_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/sponsord/files/sponsord-stub" + sponsord_version: "0.0.0-molecule-stub" + sponsord_network: arbitrum-sepolia + sponsord_pool_id: "0x00000000000000000000000000000000000000000000000000000000000000aa" + sponsord_rpc_url: "https://rpc.example.invalid/v2/molecule" + sponsord_readiness_retries: 10 + sponsord_readiness_delay: 1 + decdn_grafana_cloud_enabled: true + grafana_alloy_install_method: manual + grafana_alloy_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/grafana-cloud/files/alloy-stub" + grafana_alloy_region: US + grafana_alloy_loki_url: https://logs-prod-xx.molecule.invalid/loki/api/v1/push + sponsord_onramp_hosts: + sponsord_onramp_install_method: manual + sponsord_onramp_manual_bin_src: >- + ${MOLECULE_PROJECT_DIRECTORY}/molecule/sponsord-onramp/files/sponsord-onramp-stub + sponsord_onramp_version: "0.0.0-molecule-stub" + # sponsord_onramp_network follows sponsord_network (arbitrum-sepolia). + sponsord_onramp_rpc_url: https://sepolia-rollup.arbitrum.io/rpc + sponsord_onramp_domain: onramp.molecule.test + sponsord_onramp_caddy_tls: internal + # Renders (and so validates) the Caddyfile's global email option. + sponsord_onramp_acme_email: ops@molecule.test + sponsord_onramp_turnstile_sitekey: 1x00000000000000000000AA + # Must equal EXPECTED_BRAND in ../sponsord-onramp/files/sponsord-onramp-stub: + # quotes and a dollar sign prove the env file's quoting ($$ is molecule's escape). + sponsord_onramp_brand_name: 'deCDN "molecule" $$HOME' + sponsord_onramp_decdn_release: v0.1.0 + sponsord_onramp_decdn_sums_sha256: "1111111111111111111111111111111111111111111111111111111111111111" + sponsord_onramp_cli_release: decdn-sponsored-v0.1.0 + sponsord_onramp_cli_sums_sha256: "2222222222222222222222222222222222222222222222222222222222222222" + sponsord_onramp_fund_rate_per_min: 0 + sponsord_onramp_readiness_retries: 10 + sponsord_onramp_readiness_delay: 1 +verifier: + name: ansible +scenario: + test_sequence: + # Leading destroy: see ../default/molecule.yml. + - destroy + - create + - prepare + - converge + - side_effect + - verify + - destroy diff --git a/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml b/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml new file mode 100644 index 0000000..36f4f72 --- /dev/null +++ b/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml @@ -0,0 +1,127 @@ +--- +# Backup, then decommission: refused while sponsord holds a pool top-up, without a +# confirmation and with a wrong one, then for real through the playbook. The +# header of molecule.yml lists what each step proves; verify.yml checks the result. +- name: Prepare the backup recipient + hosts: all + become: true + tasks: + - name: Install age + ansible.builtin.apt: + name: age + state: present + update_cache: true + + - name: Generate a throwaway age identity + ansible.builtin.command: + cmd: age-keygen -o /root/molecule-age.key + creates: /root/molecule-age.key + + - name: Read the backup recipient's public key + ansible.builtin.command: + cmd: age-keygen -y /root/molecule-age.key + register: _molecule_age_pub + changed_when: false + + # Host facts, not inventory: the inventory must stay identical to + # ../sponsord-onramp's. sponsord_backup_age_recipients defaults to this list. + - name: Encrypt the backups to it, and fetch them into the ephemeral dir + ansible.builtin.set_fact: + decdn_backup_age_recipients: ["{{ _molecule_age_pub.stdout | trim }}"] + sponsord_backup_local_dir: "{{ lookup('ansible.builtin.env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/backups" + +- name: Backup + ansible.builtin.import_playbook: ../../playbooks/backup.yml + +- name: Decommission refuses a held top-up, a missing and a wrong confirmation + hosts: all + become: true + tasks: + - name: Make the stub report a held top-up + ansible.builtin.copy: + dest: /etc/sponsord/stub-topup-hold + content: "1700000000\n" + mode: "0644" + + # Confirmed, so only the hold can stop it. + - name: Decommission while a top-up is held + block: + - name: Run the decommission playbook's hold check + ansible.builtin.include_role: + name: sponsord + tasks_from: decommission + vars: + decdn_decommission_confirm: "{{ ansible_play_hosts_all | sort | join(',') }}" + - name: Fail if decommission stopped sponsord during a hold + ansible.builtin.fail: + msg: decommission proceeded while sponsord held a pool top-up. + rescue: + - name: Require the hold guard to be what failed + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Refuse to restart or stop sponsord while a pool top-up is held' + fail_msg: "Failed at '{{ ansible_failed_task.name }}', not the hold guard." + + - name: Clear the hold + ansible.builtin.file: + path: /etc/sponsord/stub-topup-hold + state: absent + + # Molecule has no TTY, and the 1 s timeout covers an interactive run: either way + # the prompt yields empty input, which must not count as confirmation. + - name: Decommission the onramp with no confirmation given + block: + - name: Run the onramp decommission without decdn_decommission_confirm + ansible.builtin.include_role: + name: sponsord_onramp + tasks_from: decommission + vars: + decdn_decommission_prompt_seconds: 1 + - name: Fail if the onramp decommission proceeded unconfirmed + ansible.builtin.fail: + msg: the onramp decommission proceeded without a typed confirmation. + rescue: + - name: Require the confirmation assert to be what failed (unconfirmed) + ansible.builtin.assert: + that: + - ansible_failed_task.name is match('^Require the typed confirmation') + fail_msg: "Failed at '{{ ansible_failed_task.name }}', not the confirmation check." + + - name: Decommission sponsord with the wrong confirmation + block: + - name: Run the sponsord decommission with a mistyped host list + ansible.builtin.include_role: + name: sponsord + tasks_from: decommission + vars: + decdn_decommission_confirm: not-this-host + - name: Fail if the sponsord decommission accepted a wrong confirmation + ansible.builtin.fail: + msg: the sponsord decommission proceeded with a confirmation that did not match. + rescue: + - name: Require the confirmation assert to be what failed (mistyped) + ansible.builtin.assert: + that: + - ansible_failed_task.name is match('^Require the typed confirmation') + fail_msg: "Failed at '{{ ansible_failed_task.name }}', not the confirmation check." + + - name: Read the services after the rejected runs + ansible.builtin.systemd_service: + name: "{{ item }}" + register: _molecule_units + loop: [sponsord, sponsord-onramp, caddy] + + - name: Require every service to survive the rejected runs + ansible.builtin.assert: + that: + - _molecule_units.results | map(attribute='status.ActiveState') | unique == ['active'] + fail_msg: >- + A rejected decommission stopped something: + {{ _molecule_units.results | map(attribute='status.ActiveState') | list }}. + + - name: Confirm the real decommission non-interactively + ansible.builtin.set_fact: + decdn_decommission_confirm: "{{ ansible_play_hosts_all | sort | join(',') }}" + +- name: Decommission + ansible.builtin.import_playbook: ../../playbooks/decommission.yml diff --git a/ansible/molecule/sponsord-onramp-lifecycle/verify.yml b/ansible/molecule/sponsord-onramp-lifecycle/verify.yml new file mode 100644 index 0000000..e7f2a71 --- /dev/null +++ b/ansible/molecule/sponsord-onramp-lifecycle/verify.yml @@ -0,0 +1,100 @@ +--- +- name: Verify the sponsord lifecycle + hosts: all + become: true + vars: + _local_dir: "{{ lookup('ansible.builtin.env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/backups/{{ inventory_hostname }}" + tasks: + # --- Backup --------------------------------------------------------------------- + - name: Find the archives on the host + ansible.builtin.find: + paths: /var/backups/sponsord + patterns: ["*.tar.age", "*.partial"] + register: _archives + + - name: Require one root-only archive and no partials + ansible.builtin.assert: + that: + - _archives.files | length == 1 + - _archives.files[0].path is search('-sponsord-[0-9]{8}T[0-9]{6}Z\.tar\.age$') + - _archives.files[0].mode == '0600' + fail_msg: "Unexpected archives: {{ _archives.files | map(attribute='path') | list }}" + + - name: List the archive's members (decrypting with the throwaway key) + ansible.builtin.shell: + cmd: set -o pipefail; age -d -i /root/molecule-age.key {{ _archives.files[0].path | quote }} | tar -tzf - + executable: /bin/bash + register: _members + changed_when: false + + # The Turnstile secret is the onramp's, beside the daemon's files; secret.env + # (the RPC URL) only with sponsord_backup_include_env. + - name: Require the credentials and nothing else + ansible.builtin.assert: + that: + - _members.stdout_lines | sort == ['etc/sponsord/api-token', 'etc/sponsord/treasury-keystore.json', + 'etc/sponsord/treasury-password', 'etc/sponsord/turnstile-secret'] + fail_msg: "The archive holds {{ _members.stdout_lines }}" + + - name: Checksum the archive on the host + ansible.builtin.stat: + path: "{{ _archives.files[0].path }}" + checksum_algorithm: sha256 + register: _remote + + - name: Checksum the copy on the control machine + ansible.builtin.stat: + path: "{{ _local_dir }}/{{ _archives.files[0].path | basename }}" + checksum_algorithm: sha256 + register: _local + delegate_to: localhost + become: false + + - name: Require the archive fetched intact + ansible.builtin.assert: + that: + - _local.stat.exists + - _local.stat.checksum == _remote.stat.checksum + fail_msg: The fetched copy is missing or differs from the archive on the host. + + # --- Decommission ----------------------------------------------------------------- + - name: Stat the units, the Alloy unit and the credentials + ansible.builtin.stat: + path: "{{ item }}" + loop: + - /etc/systemd/system/sponsord.service + - /etc/systemd/system/sponsord-onramp.service + - /etc/systemd/system/alloy.service + - /etc/sponsord/treasury-keystore.json + - /etc/sponsord/treasury-password + - /etc/sponsord/api-token + - /etc/sponsord/turnstile-secret + - /etc/caddy/Caddyfile + register: _after + + - name: Require the units gone and every credential kept + ansible.builtin.assert: + that: + - _after.results[0:3] | map(attribute='stat.exists') | unique == [false] + - _after.results[3:] | map(attribute='stat.exists') | unique == [true] + fail_msg: >- + Decommission must remove the sponsord, onramp and Alloy units and keep the + treasury files, the API token, the Turnstile secret and the Caddyfile: + {{ _after.results | map(attribute='item') | zip(_after.results | map(attribute='stat.exists')) | list }} + + - name: Read the services' state + ansible.builtin.systemd_service: + name: "{{ item }}" + register: _units + loop: [sponsord, sponsord-onramp, caddy] + + - name: Require sponsord, the onramp and the role's Caddy stopped, Caddy disabled + ansible.builtin.assert: + that: + - _units.results | map(attribute='status.ActiveState') | select('equalto', 'active') | list | length == 0 + - _units.results[2].status.UnitFileState == 'disabled' + fail_msg: >- + Still active or enabled after decommission: + {{ _units.results | map(attribute='item') + | zip(_units.results | map(attribute='status.ActiveState', default='none'), + _units.results | map(attribute='status.UnitFileState', default='none')) | list }} diff --git a/ansible/playbooks/backup.yml b/ansible/playbooks/backup.yml index 52fc85b..1835c04 100644 --- a/ansible/playbooks/backup.yml +++ b/ansible/playbooks/backup.yml @@ -1,9 +1,12 @@ --- -# Encrypted backup of each node's identity (default) or full state. +# Encrypted backups: each node's identity (default) or full state, and each +# sponsord host's credentials (the treasury keystore and password, the API token, +# the onramp's Turnstile secret). # make backup LIMIT= ANSIBLE_ARGS='-e decdn_backup_scope=full' -# decdn_backup_age_recipients (public keys) is required; set it in group_vars. -# See roles/decdn_node/tasks/backup.yml for what is archived and docs/lifecycle.md -# for restore and host migration. +# decdn_backup_age_recipients (public keys) is required; set it in group_vars +# (sponsord_backup_age_recipients defaults to it). See roles/decdn_node/tasks/backup.yml +# and roles/sponsord/tasks/backup.yml for what is archived and docs/lifecycle.md for +# restore and host migration. - name: Back up deCDN nodes hosts: decdn_nodes become: true @@ -16,3 +19,15 @@ ansible.builtin.include_role: name: decdn_node tasks_from: backup + +# Hot: sponsord keeps no state and is never stopped for its backup. +- name: Back up sponsord hosts + hosts: sponsord_hosts + become: true + serial: 1 + gather_facts: false + tasks: + - name: Back up this host's sponsord credentials + ansible.builtin.include_role: + name: sponsord + tasks_from: backup diff --git a/ansible/playbooks/decommission.yml b/ansible/playbooks/decommission.yml index ca1285b..e5d6366 100644 --- a/ansible/playbooks/decommission.yml +++ b/ansible/playbooks/decommission.yml @@ -1,9 +1,25 @@ --- -# Take a node out of service: stop decdn-node, remove its unit, tear down the -# opt-in Grafana Alloy agent. Keeps the identity (needed to withdraw the bond) and -# does NOT touch the chain; the last task prints the deregister/unbond steps. +# Take a host out of service: stop and remove every service this repo deployed on +# it (decdn-node, sponsord-onramp and its Caddy, sponsord), then tear down the +# opt-in Grafana Alloy agent. Keeps the identities and keys (the node's keystore +# withdraws the bond; sponsord's treasury keystore owns the pool) and does NOT touch +# the chain; each role's last task prints what is left to do on-chain. # make decommission LIMIT= # LIMIT is required; asks for typed confirmation # Take a backup first (make backup). See docs/lifecycle.md. + +# Before anything stops: sponsord must not be stopped while it holds an unconfirmed +# pool top-up (a stop forgets the hold, and the pool could be refilled twice). The +# sponsord play below checks again right before its stop. +- name: Check sponsord hosts for a held pool top-up + hosts: sponsord_hosts + become: true + gather_facts: false + tasks: + - name: Refuse while sponsord holds a pool top-up + ansible.builtin.include_role: + name: sponsord + tasks_from: topup-hold + - name: Decommission deCDN nodes hosts: decdn_nodes become: true @@ -13,9 +29,34 @@ name: decdn_node tasks_from: decommission - # grafana_alloy's disabled path removes only an installation this role wrote - # (it checks the managed-by marker in the unit), so this is a no-op on a host - # that never enabled Grafana Cloud or runs a foreign Alloy. +# The onramp needs the daemon, so it goes first. +- name: Decommission sponsord onramps + hosts: sponsord_onramp_hosts + become: true + gather_facts: false + tasks: + - name: Stop and remove the onramp service + ansible.builtin.include_role: + name: sponsord_onramp + tasks_from: decommission + +- name: Decommission sponsord + hosts: sponsord_hosts + become: true + gather_facts: false + tasks: + - name: Stop and remove the sponsord service + ansible.builtin.include_role: + name: sponsord + tasks_from: decommission + +# grafana_alloy's disabled path removes only an installation this role wrote (it +# checks the managed-by marker in the unit), so this is a no-op on a host that never +# enabled Grafana Cloud or runs a foreign Alloy. +- name: Tear down the Grafana Alloy agent + hosts: decdn_nodes:sponsord_hosts + become: true + tasks: - name: Tear down the Grafana Alloy agent this repo installed ansible.builtin.include_role: name: grafana_alloy diff --git a/ansible/roles/decdn_node/tasks/decommission.yml b/ansible/roles/decdn_node/tasks/decommission.yml index 45249bb..cf080d7 100644 --- a/ansible/roles/decdn_node/tasks/decommission.yml +++ b/ansible/roles/decdn_node/tasks/decommission.yml @@ -52,7 +52,9 @@ seconds: "{{ decdn_decommission_prompt_seconds | int }}" register: _decdn_decommission_typed run_once: true - when: decdn_decommission_confirm is not defined + when: + - decdn_decommission_confirm is not defined + - not (_decdn_decommission_confirmed | default(false) | bool) - name: Require the typed confirmation to match the targeted hosts ansible.builtin.assert: @@ -62,6 +64,13 @@ fail_msg: >- Confirmation did not match "{{ ansible_play_hosts_all | sort | join(',') }}". Nothing was changed. + when: not (_decdn_decommission_confirmed | default(false) | bool) + +# The sponsord and sponsord_onramp decommissions read this, so a co-located host is +# asked once per run. +- name: Remember the confirmation for the rest of this run + ansible.builtin.set_fact: + _decdn_decommission_confirmed: true - name: Check for the decdn-node unit ansible.builtin.stat: diff --git a/ansible/roles/sponsord/README.md b/ansible/roles/sponsord/README.md index 4677573..7cff58e 100644 --- a/ansible/roles/sponsord/README.md +++ b/ansible/roles/sponsord/README.md @@ -133,7 +133,12 @@ See [`defaults/main.yml`](defaults/main.yml) for the full list with comments. | `sponsord_max_spending_cap_micro_usdc`, `_max_ttl_secs`, `_pool_low_water_micro_usdc`, `_pool_refill_micro_usdc`, `_pool_watch_interval_secs` | `""` | `""` uses the daemon's default. These are economic choices, not repo facts. | | `sponsord_generate_api_token` | `true` | `false`: provision `/etc/sponsord/api-token` yourself (≥ 32 bytes). | | `sponsord_secret_env_overwrite_host_file` | `false` | Confirm that `sponsord_rpc_url` may replace a `secret.env` the role did not write. | +| `sponsord_log_level` | `""` | `RUST_LOG`: a level (`info`) or tracing directives. `""` logs errors only. | | `sponsord_stop_timeout_sec` | `120` | Cap on the graceful stop. The daemon waits for a top-up it already sent. | +| `sponsord_restart_ignore_topup_hold` | `false` | `true` restarts despite a held top-up (see Day 2). Only for a hold that can never mine. | +| `sponsord_backup_age_recipients` | `decdn_backup_age_recipients` | `tasks_from: backup`: public keys the archive is encrypted to. | +| `sponsord_backup_include_env` / `_dir` / `_fetch` / `_local_dir` | `false` / `/var/backups/sponsord` / `true` / `ansible/backups` | `tasks_from: backup`. | +| `decdn_decommission_max_hosts` / `_prompt_seconds` | `1` / `300` | `tasks_from: decommission`, shared with the other roles. | | `sponsord_readiness_retries` / `_delay` | `30` / `2` | `/healthz` window (about 60 s). | ## Observability @@ -146,8 +151,25 @@ that is not in `decdn_nodes` (`playbooks/group_vars/all.yml`). From your own playbook, set `grafana_alloy_sponsord_enabled` / `grafana_alloy_node_enabled` yourself. +A sponsord dashboard and alert rules on those series (keeper failures, a stale pool +read, a held top-up, an empty pool, sponsord's own request errors) are in +[`charts/decdn-node/files/monitoring/sponsord/`](../../../charts/decdn-node/files/monitoring/README.md#sponsord). +The treasury wallet's own USDC and gas balance is not among them: sponsord does not +export it, so watch the address yourself. + ## Day 2 +- **A held top-up blocks restarts.** When the keeper broadcasts a pool top-up and + cannot read its receipt, it holds every further top-up until that transaction + mines or provably never can (`sponsord_pool_topup_unconfirmed_since_unix` > 0). A + restart forgets the hold, and the pool can be refilled twice. So before the role + restarts sponsord (any changed input: a knob, a credential, the binary) it reads + `/metrics`, and fails at "Refuse to restart or stop sponsord while a pool top-up is + held" instead. Nothing is restarted and the restart record is not written, so the + next run retries. Wait for the hold to clear (upstream `docs/operator.md`, + "Monitor": if the transaction was dropped, send 0-value transactions from the + treasury to itself), then re-run. A running daemon whose `/metrics` does not answer + is refused too, since its hold state is unknown. - **Rotate the API token:** write the new token to `/etc/sponsord/api-token`, then re-run the role, which restarts sponsord. The onramp reads the same file and restarts with the daemon (`PartOf=`). @@ -155,12 +177,16 @@ yourself. The new wallet must own `sponsord_pool_id`, or sponsord refuses to start and the deploy fails at `/healthz`. - **Logs:** `journalctl -u sponsord`. Plain text, with colour off. In Loki its `level` label comes from the line itself, not journald's priority, which is info for every line. -- **Removal** (no decommission playbook yet; the daemon itself keeps no state): +- **Backup:** `make backup LIMIT=` (`tasks_from: backup`) archives the + treasury keystore and password, the API token and, on an onramp host, the + Turnstile secret, encrypted on the host to `sponsord_backup_age_recipients`. Hot: + sponsord keeps no state. See [`docs/lifecycle.md`](../../../docs/lifecycle.md#sponsord-the-onboarding-sponsor). +- **Removal:** `make decommission LIMIT=` (`tasks_from: decommission`, after + the onramp's) refuses while a top-up is held, asks for the typed host list, then + stops sponsord and removes its unit. It keeps `/etc/sponsord`, the binary and the + build user. To remove the rest by hand afterwards: ```bash - sudo systemctl disable --now sponsord - sudo rm /etc/systemd/system/sponsord.service - sudo systemctl daemon-reload sudo rm -f /usr/local/bin/sponsord sudo rm -rf /usr/local/lib/sponsord # the version or source stamp and decdn CLI ``` @@ -168,7 +194,7 @@ yourself. After a `source` install, the build user and its home stay behind (shared with the other roles on the host): `sudo userdel decdn-build && sudo rm -rf /var/lib/decdn-build` once nothing on the host builds from source. Then delete `/etc/sponsord` (secrets, env files and the role's two `.sha256` - records) once the treasury files are safe elsewhere. + records) once the treasury files are safe elsewhere and the pool is drained. ## Testing @@ -178,6 +204,9 @@ yourself. - a restart after a credential rotation; - that the `/healthz` gate fails the deploy when the daemon will not start; - each `secret.env` hand-off between inventory and host, with its guards; + - the top-up hold guard: a held top-up (the stub's marker) fails the converge + before the restart and keeps the restart record, the override restarts, and the + next run after the hold clears restarts; - the Alloy toggles. - `molecule/sponsord-install` runs beside it on the same converge. It checks: - release mode against a locally signed mirror: the version stamp, a re-run @@ -185,6 +214,10 @@ yourself. a binary that is not the pinned version; - treasury wallet generation through a `key-gen` stub: the files and modes, the stop on an empty pool id, an existing wallet kept, and the daemon starting on it. +- `molecule/sponsord-onramp-lifecycle` runs `playbooks/backup.yml` and + `playbooks/decommission.yml` on the `sponsord-onramp` converge: the archive's + members, decommission refused for a held top-up and a missing or wrong + confirmation, then the units removed and every credential kept. - `molecule/grafana-cloud` co-locates sponsord with a node. - `molecule/cloud-init-sponsord` deploys it from `cloud-init/user-data-sponsord.yaml` (no control machine), in release mode, beside the onramp. diff --git a/ansible/roles/sponsord/tasks/backup.yml b/ansible/roles/sponsord/tasks/backup.yml new file mode 100644 index 0000000..2cbeaa9 --- /dev/null +++ b/ansible/roles/sponsord/tasks/backup.yml @@ -0,0 +1,188 @@ +--- +# Encrypted backup of sponsord's credentials: `include_role: {name: sponsord, +# tasks_from: backup}` (playbooks/backup.yml, `make backup`). See docs/lifecycle.md +# for restore. +# +# The daemon keeps no state, so the backup is the credentials under sponsord_etc: +# the treasury keystore and its password (the wallet that owns the pool and pays +# its top-ups), the API token the onramp shares, and the onramp's Turnstile secret +# when one is on the host. Taken hot: the daemon never writes them. secret.env (the +# RPC URL) only with sponsord_backup_include_env. +# +# As decdn_node's backup: the archive is encrypted ON THE HOST to the operator's +# age recipients (public keys), so plaintext key material never leaves the host and +# only the encrypted file is fetched. + +- name: Refuse to take a backup in check mode + ansible.builtin.assert: + that: + - not ansible_check_mode + fail_msg: A backup writes an archive on the host, so it has no dry run. Run it without --check. + +- name: Validate the backup settings + ansible.builtin.assert: + that: + - sponsord_backup_age_recipients is sequence + - sponsord_backup_age_recipients is not string + - sponsord_backup_age_recipients is not mapping + - sponsord_backup_age_recipients | length > 0 + - >- + sponsord_backup_age_recipients + | reject('match', '^(age1[0-9a-z]{58}|ssh-ed25519 [A-Za-z0-9+/=]+|ssh-rsa [A-Za-z0-9+/=]+)( .*)?$') + | list | length == 0 + - sponsord_backup_include_env is boolean + - sponsord_backup_fetch is boolean + fail_msg: >- + sponsord_backup_age_recipients must be a non-empty LIST of public keys to + encrypt to: age recipients (age1…, from `age-keygen -y`) or SSH public keys + (ssh-ed25519 / ssh-rsa lines), never a private key. It defaults to + decdn_backup_age_recipients. sponsord_backup_include_env and + sponsord_backup_fetch must be booleans. + +# The archive is written into sponsord_backup_dir, which the role makes root 0700, +# so its shape is enforced before anything is touched (as decdn_node's +# validate-lifecycle-paths.yml). +- name: Constrain the backup paths (absolute, no traversal, no empty segments) + ansible.builtin.assert: + that: + - item.value is match('^/[A-Za-z0-9._/-]+$') + - "'.' not in item.value.split('/')" + - "'..' not in item.value.split('/')" + - "'' not in item.value.rstrip('/').split('/')[1:]" + quiet: true + fail_msg: >- + {{ item.name }} must be an absolute path whose segments use only + [A-Za-z0-9._-], with no '.', '..' or empty segments (got "{{ item.value }}"). + loop: + - {name: sponsord_etc, value: "{{ sponsord_etc }}"} + - {name: sponsord_backup_dir, value: "{{ sponsord_backup_dir }}"} + loop_control: + label: "{{ item.name }}" + +- name: Require the backup dir to stay clear of sponsord and system directories + vars: + _dir: "{{ sponsord_backup_dir.rstrip('/') ~ '/' }}" + ansible.builtin.assert: + that: + - _dir not in ['/etc/', '/var/', '/var/lib/', '/usr/', '/root/', '/home/', '/opt/', '/srv/', '/boot/', '/tmp/'] + - not (sponsord_etc.rstrip('/') ~ '/').startswith(_dir) + - not _dir.startswith(sponsord_etc.rstrip('/') ~ '/') + quiet: true + fail_msg: >- + sponsord_backup_dir ({{ sponsord_backup_dir }}) must be a dedicated directory + outside {{ sponsord_etc }}, and not a system directory: the role makes it + root-only (0700) and writes archives into it. The default is /var/backups/sponsord. + +# The onramp keeps its Turnstile secret beside the daemon's files; archived when present. +- name: Check for the onramp's Turnstile secret + ansible.builtin.stat: + path: "{{ sponsord_etc }}/turnstile-secret" + register: _sponsord_backup_turnstile + +- name: Build the list of paths to archive + ansible.builtin.set_fact: + _sponsord_backup_paths: >- + {{ [sponsord_treasury_keystore_file, sponsord_treasury_password_file, sponsord_api_token_file] + + ([sponsord_etc ~ '/turnstile-secret'] if _sponsord_backup_turnstile.stat.exists else []) + + ([sponsord_secret_env_file] if sponsord_backup_include_env | bool else []) }} + _sponsord_backup_file: >- + {{ sponsord_backup_dir }}/{{ inventory_hostname }}-sponsord-{{ + now(utc=true, fmt='%Y%m%dT%H%M%SZ') }}.tar.age + +- name: Check that every path to archive exists + ansible.builtin.stat: + path: "{{ item }}" + register: _sponsord_backup_stat + loop: "{{ _sponsord_backup_paths }}" + +- name: Require every path to archive + ansible.builtin.assert: + that: + - _sponsord_backup_stat.results | rejectattr('stat.exists') | list | length == 0 + fail_msg: >- + Missing on the host: {{ _sponsord_backup_stat.results | rejectattr('stat.exists') + | map(attribute='item') | join(', ') }}. There is no treasury to back up yet + (deploy sponsord first), or a path variable does not match this host. + +- name: Install age + ansible.builtin.apt: + name: age + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Create the backup directory + ansible.builtin.file: + path: "{{ sponsord_backup_dir }}" + state: directory + owner: root + group: root + mode: "0700" + +# Recipients go in a file (`age -R`) rather than argv: SSH public keys carry spaces +# and comments, and a file needs no shell quoting at all. +- name: Write the recipients file + ansible.builtin.copy: + dest: "{{ sponsord_backup_dir }}/.recipients" + content: "{{ sponsord_backup_age_recipients | join('\n') }}\n" + owner: root + group: root + mode: "0600" + +# tar reads the absolute paths relative to / (so a restore is `tar -C /`) and +# stores numeric owners. Written to a .partial name and renamed, so an interrupted +# run never leaves a truncated archive behind that looks complete. +- name: Take the backup + block: + - name: Write the encrypted archive + ansible.builtin.shell: + cmd: >- + set -euo pipefail; + umask 077; + tar -C / --numeric-owner + -czf - -- {{ _sponsord_backup_paths | map('regex_replace', '^/', '') | map('quote') | join(' ') }} + | age -R {{ (sponsord_backup_dir ~ '/.recipients') | quote }} + -o {{ (_sponsord_backup_file ~ '.partial') | quote }}; + mv -- {{ (_sponsord_backup_file ~ '.partial') | quote }} {{ _sponsord_backup_file | quote }} + executable: /bin/bash + changed_when: true + always: + - name: Remove a partial archive left by a failed run + ansible.builtin.file: + path: "{{ _sponsord_backup_file }}.partial" + state: absent + +- name: Checksum the archive + ansible.builtin.stat: + path: "{{ _sponsord_backup_file }}" + checksum_algorithm: sha256 + register: _sponsord_backup_result + +- name: Create the local backup directory for this host + ansible.builtin.file: + path: "{{ sponsord_backup_local_dir }}/{{ inventory_hostname }}" + state: directory + mode: "0700" + delegate_to: localhost + become: false + when: sponsord_backup_fetch | bool + +- name: Fetch the encrypted archive to the control machine + ansible.builtin.fetch: + src: "{{ _sponsord_backup_file }}" + dest: "{{ sponsord_backup_local_dir }}/{{ inventory_hostname }}/" + flat: true + validate_checksum: true + when: sponsord_backup_fetch | bool + +- name: Report the backup + ansible.builtin.debug: + msg: >- + sponsord backup written to {{ _sponsord_backup_file }} + (sha256 {{ _sponsord_backup_result.stat.checksum }}, {{ _sponsord_backup_result.stat.size }} bytes) + {{- (', fetched to ' ~ sponsord_backup_local_dir ~ '/' ~ inventory_hostname ~ '/') + if sponsord_backup_fetch | bool + else ('. Not fetched; copy it off the host with: ssh ' ~ inventory_hostname + ~ ' sudo cat ' ~ _sponsord_backup_file ~ ' > ' ~ (_sponsord_backup_file | basename)) }}. + It holds the treasury keystore: whoever decrypts it controls the pool's funds. + Prove it restores before you rely on it: `age -d -i | tar -tz`. diff --git a/ansible/roles/sponsord/tasks/decommission.yml b/ansible/roles/sponsord/tasks/decommission.yml new file mode 100644 index 0000000..2454bea --- /dev/null +++ b/ansible/roles/sponsord/tasks/decommission.yml @@ -0,0 +1,114 @@ +--- +# Take sponsord out of service: `include_role: {name: sponsord, tasks_from: +# decommission}` (playbooks/decommission.yml, `make decommission LIMIT=`). +# +# What it does: refuse while the daemon holds an unconfirmed pool top-up (a stop +# forgets the hold, and the pool could be refilled twice), then stop the daemon and +# disable and remove its systemd unit. What it deliberately KEEPS: sponsord_etc (the +# treasury keystore owns the pool and holds its funds; the API token), the binary +# and the shared decdn-build user and toolchain. The onramp needs the daemon, so +# playbooks/decommission.yml runs sponsord_onramp's decommission first. +# +# It does NOT touch the chain: the pool and its USDC stay with the treasury wallet, +# and withdrawing them is an operator step with the decdn CLI; the final task says so. + +- name: Refuse to decommission in check mode + ansible.builtin.assert: + that: + - not ansible_check_mode + fail_msg: Decommissioning has no dry run; run it without --check once you mean it. + +- name: Refuse to decommission more hosts than allowed at once + ansible.builtin.assert: + that: + - decdn_decommission_max_hosts | int > 0 + - ansible_play_hosts_all | length <= decdn_decommission_max_hosts | int + - decdn_decommission_prompt_seconds | int > 0 + fail_msg: >- + This run targets {{ ansible_play_hosts_all | length }} hosts + ({{ ansible_play_hosts_all | sort | join(', ') }}), more than + decdn_decommission_max_hosts ({{ decdn_decommission_max_hosts }}). Scope it + with LIMIT, or raise decdn_decommission_max_hosts deliberately. + decdn_decommission_prompt_seconds must be a positive number of seconds. + +# Checked before the prompt, so nobody types a confirmation for a run that cannot +# proceed; the stop below follows within seconds. +- name: Refuse to stop a sponsord that holds a pool top-up + ansible.builtin.include_tasks: topup-hold.yml + +# Typed confirmation (AGENTS.md hard rule 4), as decdn_node's. A host that already +# confirmed in an earlier play of this run (a co-located node, or the onramp) is not +# asked again. +- name: Ask for typed confirmation + ansible.builtin.pause: + prompt: >- + About to STOP sponsord and REMOVE its systemd unit on: {{ ansible_play_hosts_all | sort | join(',') }}. + The pool and the treasury's funds are NOT touched by this playbook. + Type the host list above exactly to continue + (times out after {{ decdn_decommission_prompt_seconds }} s) + seconds: "{{ decdn_decommission_prompt_seconds | int }}" + register: _sponsord_decommission_typed + run_once: true + when: + - decdn_decommission_confirm is not defined + - not (_decdn_decommission_confirmed | default(false) | bool) + +- name: Require the typed confirmation to match the targeted hosts + ansible.builtin.assert: + that: + - (decdn_decommission_confirm | default(_sponsord_decommission_typed.user_input | default('')) | trim) + == (ansible_play_hosts_all | sort | join(',')) + fail_msg: >- + Confirmation did not match "{{ ansible_play_hosts_all | sort | join(',') }}". + Nothing was changed. + when: not (_decdn_decommission_confirmed | default(false) | bool) + +- name: Remember the confirmation for the rest of this run + ansible.builtin.set_fact: + _decdn_decommission_confirmed: true + +- name: Check for the sponsord unit + ansible.builtin.stat: + path: /etc/systemd/system/sponsord.service + register: _sponsord_decommission_unit + +# SIGTERM: the daemon stops serving, then waits for a top-up it already sent to +# settle, for up to sponsord_stop_timeout_sec. +- name: Stop and disable sponsord + ansible.builtin.systemd_service: + name: sponsord + state: stopped + enabled: false + when: _sponsord_decommission_unit.stat.exists + +# The record goes too: a later redeploy then starts from a fresh daemon rather +# than comparing against inputs of one that no longer exists. +- name: Remove the sponsord unit and its restart-inputs record + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/systemd/system/sponsord.service + - "{{ sponsord_inputs_checksum_file }}" + +# Unconditional: cheap, and right even after an earlier run removed the file but +# failed before reloading. +- name: Reload systemd after removing the unit + ansible.builtin.systemd_service: + daemon_reload: true + +- name: Report what is left to do + ansible.builtin.debug: + msg: + - "sponsord is stopped and its unit removed on {{ inventory_hostname }}." + - >- + Kept: {{ sponsord_etc }} (the treasury keystore and password, the API token, + secret.env), {{ sponsord_bin }} and, after a source install, the shared + {{ decdn_build_home }}. Take a backup first (make backup) and keep the keystore + until the pool is drained: it is the only key that controls the pool's funds. + - >- + On-chain: the pool and its USDC stay with the treasury wallet. Withdraw them + with the decdn CLI from the treasury wallet; the role does not. + - >- + Take the host out of sponsord_hosts (and sponsord_onramp_hosts) so the next + deploy does not bring sponsord back. diff --git a/ansible/roles/sponsord_onramp/README.md b/ansible/roles/sponsord_onramp/README.md index fa41719..7582e47 100644 --- a/ansible/roles/sponsord_onramp/README.md +++ b/ansible/roles/sponsord_onramp/README.md @@ -138,6 +138,7 @@ See [`defaults/main.yml`](defaults/main.yml) for the full list with comments. | `sponsord_onramp_fund_rate_per_min`, `_poll_rate_per_min` | `""` | Per-address rate limits (upstream 10 and 120; `0` turns one off). | | `sponsord_onramp_bind_address` / `_port` | `127.0.0.1` / `8080` | IPv4 loopback only (asserted). | | `sponsord_onramp_readiness_retries` / `_delay` | `30` / `2` | `/healthz` window (about 60 s). | +| `decdn_decommission_max_hosts` / `_prompt_seconds` | `1` / `300` | `tasks_from: decommission`, shared with the other roles. | ## Bringing your own proxy @@ -169,13 +170,15 @@ collector; its access logs are not shipped. `sponsord_onramp_min_cli_version` to make existing users re-run the installer. - **Certificate trouble:** `sudo journalctl -u caddy -e`. Check that the domain resolves to the host and that tcp/80 and tcp/443 reach it. -- **Logs:** `journalctl -u sponsord-onramp`. -- **Removal:** +- **Logs:** `journalctl -u sponsord-onramp`. The onramp always logs at info: its + release is built without tracing's env-filter, so `RUST_LOG` does nothing. +- **Removal:** `make decommission LIMIT=` (`tasks_from: decommission`, run + before sponsord's) asks for the typed host list, stops the onramp and removes its + unit, and stops and disables the role's Caddy (only when `/etc/caddy/Caddyfile` + carries the role's marker). It keeps the files under `/etc/sponsord`, the binary + and the caddy package. To remove those by hand afterwards: ```bash - sudo systemctl disable --now sponsord-onramp caddy - sudo rm /etc/systemd/system/sponsord-onramp.service - sudo systemctl daemon-reload sudo rm -f /usr/local/bin/sponsord-onramp /etc/sponsord/sponsord-onramp.env \ /etc/sponsord/onramp-gate.html /etc/sponsord/.onramp-inputs.sha256 \ /etc/sponsord/turnstile-secret diff --git a/ansible/roles/sponsord_onramp/defaults/main.yml b/ansible/roles/sponsord_onramp/defaults/main.yml index 103d94e..fd568e8 100644 --- a/ansible/roles/sponsord_onramp/defaults/main.yml +++ b/ansible/roles/sponsord_onramp/defaults/main.yml @@ -174,3 +174,12 @@ sponsord_onramp_bin: /usr/local/bin/sponsord-onramp # 200 is the config gate. A timeout FAILS the deploy. Window ~60s. sponsord_onramp_readiness_retries: 30 sponsord_onramp_readiness_delay: 2 + +# --- Lifecycle: decommission (tasks_from: decommission) ------------------------- +# Only read by that entry point (playbooks/decommission.yml). Shared with decdn_node +# and sponsord (and identical there, `make test-scripts` checks): one typed +# confirmation covers every service on the host. A non-interactive run sets +# decdn_decommission_confirm to the comma-joined, sorted host list (deliberately NOT +# defaulted here). +decdn_decommission_max_hosts: 1 +decdn_decommission_prompt_seconds: 300 # an unanswered confirmation prompt fails closed after this diff --git a/ansible/roles/sponsord_onramp/tasks/decommission.yml b/ansible/roles/sponsord_onramp/tasks/decommission.yml new file mode 100644 index 0000000..bbeb022 --- /dev/null +++ b/ansible/roles/sponsord_onramp/tasks/decommission.yml @@ -0,0 +1,119 @@ +--- +# Take the onramp out of service: `include_role: {name: sponsord_onramp, +# tasks_from: decommission}` (playbooks/decommission.yml, `make decommission +# LIMIT=`). It runs before sponsord's own decommission: the onramp needs the +# daemon. +# +# What it does: stop the onramp and disable and remove its systemd unit, and stop +# and disable the role's Caddy (only when /etc/caddy/Caddyfile carries the role's +# marker; a foreign Caddy is never touched). What it KEEPS: the Turnstile secret +# and the other files under sponsord_onramp_etc, the binary and the caddy package. +# The tcp/80 + tcp/443 firewall holes stay until the host leaves +# sponsord_onramp_hosts and baseline runs again. + +- name: Refuse to decommission in check mode + ansible.builtin.assert: + that: + - not ansible_check_mode + fail_msg: Decommissioning has no dry run; run it without --check once you mean it. + +- name: Refuse to decommission more hosts than allowed at once + ansible.builtin.assert: + that: + - decdn_decommission_max_hosts | int > 0 + - ansible_play_hosts_all | length <= decdn_decommission_max_hosts | int + - decdn_decommission_prompt_seconds | int > 0 + fail_msg: >- + This run targets {{ ansible_play_hosts_all | length }} hosts + ({{ ansible_play_hosts_all | sort | join(', ') }}), more than + decdn_decommission_max_hosts ({{ decdn_decommission_max_hosts }}). Scope it + with LIMIT, or raise decdn_decommission_max_hosts deliberately. + decdn_decommission_prompt_seconds must be a positive number of seconds. + +# Typed confirmation (AGENTS.md hard rule 4), as decdn_node's and sponsord's; a host +# that already confirmed in an earlier play of this run is not asked again. +- name: Ask for typed confirmation + ansible.builtin.pause: + prompt: >- + About to STOP sponsord-onramp (and the Caddy in front of it) and REMOVE its + systemd unit on: {{ ansible_play_hosts_all | sort | join(',') }}. + Type the host list above exactly to continue + (times out after {{ decdn_decommission_prompt_seconds }} s) + seconds: "{{ decdn_decommission_prompt_seconds | int }}" + register: _sponsord_onramp_decommission_typed + run_once: true + when: + - decdn_decommission_confirm is not defined + - not (_decdn_decommission_confirmed | default(false) | bool) + +- name: Require the typed confirmation to match the targeted hosts + ansible.builtin.assert: + that: + - (decdn_decommission_confirm | default(_sponsord_onramp_decommission_typed.user_input | default('')) | trim) + == (ansible_play_hosts_all | sort | join(',')) + fail_msg: >- + Confirmation did not match "{{ ansible_play_hosts_all | sort | join(',') }}". + Nothing was changed. + when: not (_decdn_decommission_confirmed | default(false) | bool) + +- name: Remember the confirmation for the rest of this run + ansible.builtin.set_fact: + _decdn_decommission_confirmed: true + +- name: Check for the sponsord-onramp unit + ansible.builtin.stat: + path: /etc/systemd/system/sponsord-onramp.service + register: _sponsord_onramp_decommission_unit + +- name: Stop and disable sponsord-onramp + ansible.builtin.systemd_service: + name: sponsord-onramp + state: stopped + enabled: false + when: _sponsord_onramp_decommission_unit.stat.exists + +- name: Remove the sponsord-onramp unit and its restart-inputs record + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/systemd/system/sponsord-onramp.service + - "{{ sponsord_onramp_inputs_checksum_file }}" + +# Unconditional: cheap, and right even after an earlier run removed the file but +# failed before reloading. +- name: Reload systemd after removing the unit + ansible.builtin.systemd_service: + daemon_reload: true + +# Only a Caddy serving this role's Caddyfile: with the backend gone it would answer +# every request with a 502 on the public ports. +- name: Check the Caddyfile for this role's marker + ansible.builtin.command: + argv: [grep, -qF, "MANAGED BY the sponsord_onramp role", "{{ _sponsord_onramp_caddyfile }}"] + register: _sponsord_onramp_decommission_caddy + changed_when: false + # 1 = no marker, 2 = no file (or unreadable): neither is the role's Caddy. + failed_when: false + +- name: Stop and disable the role's Caddy + ansible.builtin.systemd_service: + name: caddy + state: stopped + enabled: false + when: _sponsord_onramp_decommission_caddy.rc == 0 + +- name: Report what is left to do + ansible.builtin.debug: + msg: + - >- + sponsord-onramp is stopped and its unit removed on {{ inventory_hostname }} + {{- "; the role's Caddy is stopped and disabled" if _sponsord_onramp_decommission_caddy.rc == 0 + else "; no Caddy of this role's was found" }}. + - >- + Kept: {{ sponsord_onramp_turnstile_secret_file }} and the other files under + {{ sponsord_onramp_etc }}, {{ sponsord_onramp_bin }}, the caddy package and + {{ _sponsord_onramp_caddyfile }}. + - >- + tcp/80 and tcp/443 stay open until you take the host out of + sponsord_onramp_hosts and re-run baseline. diff --git a/docs/lifecycle.md b/docs/lifecycle.md index da8f99d..caa2642 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -1,7 +1,8 @@ # Node lifecycle: backup, restore, migration, decommission -Day-2 procedures for a running deCDN node. The Ansible targets run from `ansible/`; -the manual commands work for any deploy path. +Day-2 procedures for a running deCDN node, and for sponsord in +[its own section](#sponsord-the-onboarding-sponsor). The Ansible targets run from +`ansible/`; the manual commands work for any deploy path. ## What makes a node *that* node @@ -211,12 +212,57 @@ sudo systemd-run --pty --wait --collect -p User=decdn \ --keystore-password-file /etc/decdn/keystore.password ``` +## sponsord (the onboarding sponsor) + +sponsord keeps no state. What makes it *that* sponsor are its credentials under +`/etc/sponsord/`: + +| File | What it is | Lose it and… | +|------|-----------|--------------| +| `treasury-keystore.json` | the treasury wallet, encrypted: it owns the pool, signs every capability and pays the top-ups | the pool and the USDC in it are out of reach | +| `treasury-password` | its password | same as losing the keystore | +| `api-token` | the bearer token the onramp presents | regenerate it (the onramp reads the new one) | +| `turnstile-secret` | the onramp's Turnstile secret | re-copy it from the Cloudflare dashboard | + +**Backup.** `make backup` also covers hosts in `sponsord_hosts`: it archives those +files (and `secret.env` with `-e sponsord_backup_include_env=true`) hot, encrypted on +the host to `sponsord_backup_age_recipients` (by default `decdn_backup_age_recipients`, +so one list covers a co-located host), into `/var/backups/sponsord/`, and fetches the +archive to `ansible/backups//`. Whoever decrypts it controls the pool's funds. + +**The top-up hold.** When sponsord's keeper broadcasts a pool top-up and cannot read +its receipt, it holds every further top-up until that transaction mines or provably +never can (`sponsord_pool_topup_unconfirmed_since_unix` > 0 on `/metrics`). The hold +lives only in the process: a restart or a stop forgets it, and the pool can then be +refilled twice. So the role reads `/metrics` before any restart a deploy would cause, +and `make decommission` before it stops sponsord, and both **fail** while a hold is +on. Wait for it to clear (send 0-value transactions from the treasury to itself if +the transaction was dropped; the daemon's error log line says how many), then re-run. +`-e sponsord_restart_ignore_topup_hold=true` overrides the deploy's check, only for a +hold you have confirmed can never mine. + +**Migration.** Never run two sponsord daemons on one treasury: both would top up the +pool. Decommission the old host first (`make decommission LIMIT=old-host`), stream +the archive into the new host as for a node (step 3 above, without the `chown`: the +files stay `root` `0600`), then deploy it. + +**Decommission.** On a host in `sponsord_onramp_hosts`, `make decommission` stops and +removes the onramp's unit and stops and disables the role's Caddy (only when +`/etc/caddy/Caddyfile` carries the role's marker). On a host in `sponsord_hosts`, it +then stops and removes sponsord's unit. A co-located host is asked to confirm once. +It keeps `/etc/sponsord` (the treasury owns the pool), the binaries and the caddy +package, and does not touch the chain: the pool and its USDC stay with the treasury +wallet until you withdraw them with the `decdn` CLI. tcp/80 and tcp/443 stay open +until the host leaves `sponsord_onramp_hosts` and baseline runs again. + ## Compose and Kubernetes - **Compose** ([`compose/`](../compose/README.md)) uses the same host paths (`/var/lib/decdn`, `/etc/decdn`), so the manual backup command and the restore steps above apply unchanged; stop the node with `sudo docker compose -f compose/compose.yaml stop decdn-node`. + sponsord's Compose layout uses `/etc/sponsord` too; its README shows the manual + backup and the top-up-hold check before a restart. - **Helm**: the identity lives in the operator-provisioned `existingSecret`, which you created off-cluster and should already hold elsewhere. The daemon's state is on the PVC; snapshot it with your storage's `VolumeSnapshot` support after scaling the diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index 82efb8e..3ae109f 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -167,7 +167,7 @@ want="$(cd "$repo/ansible/molecule" && for d in */molecule.yml; do echo "${d%/mo # Split scenarios share one converge, so their inventories must not drift apart. if command -v yq >/dev/null; then for pair in sponsord:sponsord-install sponsord-onramp:sponsord-onramp-caddy \ - sponsord-onramp:sponsord-onramp-source; do + sponsord-onramp:sponsord-onramp-source sponsord-onramp:sponsord-onramp-lifecycle; do a="$repo/ansible/molecule/${pair%%:*}/molecule.yml" b="$repo/ansible/molecule/${pair##*:}/molecule.yml" [[ "$(yq -o=json '.provisioner.inventory' "$a")" == "$(yq -o=json '.provisioner.inventory' "$b")" ]] \ || fail "molecule ${pair%%:*} and ${pair##*:} inventories differ (they share one converge)" @@ -271,6 +271,8 @@ variant "onramp short grace" "sponsord-onramp: stop_grace_period" variant "onramp by tag" "sponsord-onramp: image is pinned" "${onr} s#^(\s*)image: .*#\1image: ghcr.io/decdn/sponsord-onramp:latest#" variant "onramp holds keystore" "sponsord-onramp: mounts are exactly" "${onr} s#^(\s*)volumes:\$#\1volumes:\n\1 - /etc/sponsord/treasury-keystore.json:/run/secrets/k:ro#" variant "writable turnstile mount" "sponsord-onramp: mounts are exactly" "${onr} {/target: \/run\/secrets\/turnstile-secret$/{n;s/read_only: true/read_only: false/}}" +variant "writable gate-page mount" "sponsord-onramp: mounts are exactly" "${onr} {/target: \/etc\/sponsord\/onramp-gate$/{n;s/read_only: true/read_only: false/}}" +variant "gate page from a secret" "sponsord-onramp: mounts are exactly" "${onr} s#source: /etc/sponsord/onramp-gate\$#source: /etc/sponsord/treasury-keystore.json#" variant "inline Turnstile secret" "sponsord-onramp: no inline ONRAMP_TURNSTILE_SECRET" "${onr} s/^(\s*)ONRAMP_BIND: (.*)$/&\n\1ONRAMP_TURNSTILE_SECRET: x/" variant "onramp on sponsord hosts" "sponsord-onramp: profiles are [onramp]" "${onr} s/^(\s*)profiles: \[onramp\]$/\1profiles: [onramp, sponsord]/" variant "resolvable domain default" "sponsord-onramp: unset domain" "${onr} s#ONRAMP_PUBLIC_URL: .*#ONRAMP_PUBLIC_URL: https://\\\${SPONSORD_ONRAMP_DOMAIN:-unset-SPONSORD_ONRAMP_DOMAIN.invalid}#" @@ -387,22 +389,23 @@ else skipped+=("lint-cloud-init negatives (needs cloud-init on PATH; CI installs it)") fi -# --- the source install method's shared pieces stay identical across roles -------- +# --- the shared pieces stay identical across roles --------------------------------- # decdn_build_user/home and the rustup pins are defined in all three roles' defaults -# (one build user and toolchain per host), and the root-side installer is copied +# (one build user and toolchain per host), as are the decommission cap and prompt +# timeout (one typed confirmation per host), and the root-side installer is copied # into decdn_node and sponsord (sponsord_onramp uses sponsord's). shared_keys() { # - grep -E '^(decdn_build_user|decdn_build_home|decdn_rustup_version|decdn_rustup_sha256):|^ (x86_64|aarch64)-unknown-linux-gnu:' "$1" + grep -E '^(decdn_build_user|decdn_build_home|decdn_rustup_version|decdn_rustup_sha256|decdn_decommission_max_hosts|decdn_decommission_prompt_seconds):|^ (x86_64|aarch64)-unknown-linux-gnu:' "$1" } want=$(shared_keys "$repo/ansible/roles/decdn_node/defaults/main.yml") for r in sponsord sponsord_onramp; do [ "$(shared_keys "$repo/ansible/roles/$r/defaults/main.yml")" = "$want" ] \ - || fail "$r/defaults/main.yml: decdn_build_* / decdn_rustup_* differ from decdn_node's" + || fail "$r/defaults/main.yml: decdn_build_* / decdn_rustup_* / decdn_decommission_* differ from decdn_node's" done cmp -s "$repo/ansible/roles/decdn_node/files/install-build-output.py" \ "$repo/ansible/roles/sponsord/files/install-build-output.py" \ || fail "decdn_node and sponsord files/install-build-output.py differ" -pass "source-build defaults and installer identical across roles" +pass "source-build and decommission defaults, and the installer, identical across roles" # --- baseline firewall holes per host shape, and playbook guards --------------------- if command -v ansible >/dev/null; then From 9dae6fcd81af1c47ee07a5543d3e2a9928540c30 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 7 Oct 2026 21:40:55 +0300 Subject: [PATCH 3/6] feat(monitoring): sponsord dashboard and alert rules files/monitoring/sponsord/ holds a dashboard (uid decdn-sponsord) and rules for keeper failures, a stale pool read, a held top-up, an empty pool and request errors, with promtool unit tests in lint-helm. The chart does not render them: sponsord has no Kubernetes path. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/grafana-dashboards/SKILL.md | 9 + Makefile | 1 + charts/decdn-node/CHANGELOG.md | 5 + charts/decdn-node/files/monitoring/README.md | 25 + .../sponsord/dashboard-sponsord.json | 876 ++++++++++++++++++ .../monitoring/sponsord/prometheus-alerts.yml | 153 +++ charts/decdn-node/tests/render-test.sh | 47 +- .../decdn-node/tests/sponsord-alerts_test.yml | 105 +++ 8 files changed, 1220 insertions(+), 1 deletion(-) create mode 100644 charts/decdn-node/files/monitoring/sponsord/dashboard-sponsord.json create mode 100644 charts/decdn-node/files/monitoring/sponsord/prometheus-alerts.yml create mode 100644 charts/decdn-node/tests/sponsord-alerts_test.yml diff --git a/.claude/skills/grafana-dashboards/SKILL.md b/.claude/skills/grafana-dashboards/SKILL.md index 828f39a..79e0ce2 100644 --- a/.claude/skills/grafana-dashboards/SKILL.md +++ b/.claude/skills/grafana-dashboards/SKILL.md @@ -40,6 +40,15 @@ their only home. `decdn-delivery`. Every rule carries a `component` label. Only rules with a matching section in upstream's `docs/runbook.md` (20 of 51) also carry a `runbook_url`. +- `sponsord/` — the onboarding sponsor's own pair: `dashboard-sponsord.json` (uid + `decdn-sponsord`) and `prometheus-alerts.yml` (group `sponsord`, `job="sponsord"`). + The chart does not render this subdirectory (sponsord has no Kubernetes path). Its + `sponsord_*` names come from upstream `decdn/sponsord` `crates/sponsord/src/metrics.rs`; + run the name check below with `sponsord_` and that file. Its logs are plain text, + not JSON: select `{unit="sponsord.service"}` and filter on the `level` label the + `grafana_alloy` role parses. The rules have promtool unit tests in + `charts/decdn-node/tests/sponsord-alerts_test.yml`: keep them in step. + Add a row to an existing dashboard before starting a fifth one. The Helm chart renders these files as-is (`templates/prometheusrule.yaml`, `templates/dashboards-configmap.yaml`), so a new `*.json` becomes a new ConfigMap and `charts/decdn-node/tests/render-test.sh` diff --git a/Makefile b/Makefile index 9ba688c..4566083 100644 --- a/Makefile +++ b/Makefile @@ -148,6 +148,7 @@ test-scripts: ## test the Makefile and molecule-driver guards, release ga lint-helm: ## helm lint + render tests + kubeconform + promtool + shared schema-key check (needs helm, yq, python3>=3.11, docker) KUBECONFORM="docker run --rm -i $(KUBECONFORM_IMAGE)" \ PROMTOOL="docker run --rm -i --entrypoint promtool $(PROMTOOL_IMAGE)" \ + PROMTOOL_IMAGE="$(PROMTOOL_IMAGE)" \ $(CHART)/tests/render-test.sh # The molecule grafana-cloud scenario runs the role against a stub that exits 0 diff --git a/charts/decdn-node/CHANGELOG.md b/charts/decdn-node/CHANGELOG.md index 9a8f52f..97f0da7 100644 --- a/charts/decdn-node/CHANGELOG.md +++ b/charts/decdn-node/CHANGELOG.md @@ -17,6 +17,11 @@ both (see [RELEASING.md](../../RELEASING.md)). (`metrics.prometheusRule`), and the deCDN Grafana dashboards as sidecar-labelled ConfigMaps (`metrics.grafanaDashboards`). Both live in `files/monitoring/`, which this repo maintains; `decdn/decdn` no longer ships them. +- `files/monitoring/sponsord/`: a Grafana dashboard (`uid: decdn-sponsord`) and + alert rules for the onboarding sponsor (down, keeper failing, stale pool read, a + held top-up, an empty pool, request errors), with promtool unit tests. sponsord + has no Kubernetes path, so the chart does not render them; they ship for import + on the Ansible and Compose paths. ### Fixed diff --git a/charts/decdn-node/files/monitoring/README.md b/charts/decdn-node/files/monitoring/README.md index efc9f08..2a0d29d 100644 --- a/charts/decdn-node/files/monitoring/README.md +++ b/charts/decdn-node/files/monitoring/README.md @@ -12,6 +12,7 @@ node's metric surface is documented in upstream `decdn/decdn`'s | `dashboard-chain.json` | Chain, payments and slash safety (`uid: decdn-chain`): watcher liveness, chain RPC, registries, payments. | | `dashboard-node.json` | Single-node drilldown (`uid: decdn-node`): host, process, iroh transport, DHT and probe, logs, traces. | | `prometheus-alerts.yml` | Rule groups `decdn-slash-safety`, `decdn-liveness`, `decdn-delivery`. A rule with a matching runbook section carries a `runbook_url` into upstream's `docs/runbook.md`. | +| `sponsord/` | The onboarding sponsor's dashboard and alert rules ([below](#sponsord)). Not rendered by the chart. | **Editing.** Every `decdn_*` series a panel or rule names must be one `decdn-node` exports; nothing in CI checks this. See @@ -40,3 +41,27 @@ metrics and logs carry `job="integrations/node_exporter"`. So: - **Alerts:** load `prometheus-alerts.yml` as a rule group, e.g. with `mimirtool rules load prometheus-alerts.yml` against your Grafana Cloud Prometheus endpoint, or through *Alerting → Alert rules → Import*. + +## sponsord + +`sponsord/` holds the same pair for the deCDN onboarding sponsor (`decdn/sponsord`), +which runs on the Ansible and Compose paths only. The chart never renders it: its +`PrometheusRule` reads only `prometheus-alerts.yml`, and its dashboard ConfigMaps glob +`*.json` in this directory, not below it. + +| File | What it is | +|------|------------| +| `sponsord/dashboard-sponsord.json` | `uid: decdn-sponsord`: pool balance, the top-up hold, keeper reads and top-ups, capabilities issued, errors by code, the daemon's and the onramp's logs. | +| `sponsord/prometheus-alerts.yml` | Rule group `sponsord`: down, keeper failing, stale pool read, a held top-up (and one held over 2 h), an empty pool, sponsord's own request errors. `runbook_url` points at upstream's `docs/operator.md`, "Monitor". | + +They select `job="sponsord"`, which the `grafana_alloy` role stamps on sponsord's +`/metrics` (`grafana_alloy_sponsord_job`); log panels select `unit="sponsord.service"` +and `unit="sponsord-onramp.service"` with the `level` label the role parses. Import +them as above, e.g. `mimirtool rules load sponsord/prometheus-alerts.yml`. Every +`sponsord_*` series they name must be one upstream's `crates/sponsord/src/metrics.rs` +exports; `make lint-helm` runs `promtool check rules` on the rules and the unit tests +in `../../tests/sponsord-alerts_test.yml`, and checks the dashboard's uid. + +The treasury wallet's own USDC and gas balance, which pays every top-up, is not +covered: sponsord does not export it. Watch that address with a balance exporter of +your own; `SponsordKeeperFailing` is the symptom when it runs dry. diff --git a/charts/decdn-node/files/monitoring/sponsord/dashboard-sponsord.json b/charts/decdn-node/files/monitoring/sponsord/dashboard-sponsord.json new file mode 100644 index 0000000..d274c36 --- /dev/null +++ b/charts/decdn-node/files/monitoring/sponsord/dashboard-sponsord.json @@ -0,0 +1,876 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": false, + "iconColor": "orange", + "name": "Top-ups", + "expr": "increase(sponsord_pool_topups_total{job=\"sponsord\", instance=~\"$instance\"}[$__rate_interval]) > 0", + "titleFormat": "pool topped up", + "textFormat": "{{instance}}", + "step": "60s", + "useValueForTime": false + } + ] + }, + "description": "The deCDN onboarding sponsor (decdn/sponsord): pool balance, keeper, top-up hold, issuance and errors, logs. Metrics job=\"sponsord\" as the grafana_alloy role ships them.", + "editable": true, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Pool and keeper", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "sponsord", + "description": "Whether Prometheus can scrape sponsord (`up`). Down: no capability is issued and the pool is not topped up.", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "up{job=\"sponsord\", instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pool balance", + "description": "The pool's remaining balance at the keeper's last read (sponsord_pool_remaining_micro_usdc, shown in USDC). The keeper tops it up below SPONSORD_POOL_LOW_WATER_MICRO_USDC.", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sponsord_pool_remaining_micro_usdc{job=\"sponsord\", instance=~\"$instance\"} / 1e6", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1e-06 + } + ] + }, + "unit": "currencyUSD", + "mappings": [], + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 4, + "y": 1 + }, + "id": 3 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Top-up hold", + "description": "sponsord_pool_topup_unconfirmed_since_unix: non-zero while a top-up came back unconfirmed and further top-ups are held. DO NOT restart sponsord while HELD: a restart forgets the hold and the pool can be refilled twice (the deploy tooling refuses). See upstream docs/operator.md, Monitor.", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sponsord_pool_topup_unconfirmed_since_unix{job=\"sponsord\", instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "none" + } + } + }, + { + "type": "range", + "options": { + "from": 1, + "to": null, + "result": { + "text": "HELD — do not restart" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 9, + "y": 1 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Since last balance read", + "description": "time() - sponsord_pool_last_check_unix. The keeper reads the pool at start and every SPONSORD_POOL_WATCH_INTERVAL_SECS (1h by default); SponsordPoolCheckStale fires past 3h.", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "time() - sponsord_pool_last_check_unix{job=\"sponsord\", instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 7200 + }, + { + "color": "red", + "value": 10800 + } + ] + }, + "unit": "s", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 14, + "y": 1 + }, + "id": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Since last top-up", + "description": "time() - sponsord_pool_last_topup_unix, for a sponsord that has topped up at least once since it started.", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "time() - (sponsord_pool_last_topup_unix{job=\"sponsord\", instance=~\"$instance\"} > 0)", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "s", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 6 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pool balance", + "description": "sponsord_pool_remaining_micro_usdc in USDC, as of each keeper read.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sponsord_pool_remaining_micro_usdc{job=\"sponsord\", instance=~\"$instance\"} / 1e6", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "currencyUSD", + "min": 0, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 7 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Top-ups and keeper failures per hour", + "description": "Successful pool top-ups (sponsord_pool_topups_total) and failed chain reads or top-ups (sponsord_pool_keeper_failures_total). Failures usually mean the treasury is out of USDC or gas, or the RPC endpoint refuses it.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(sponsord_pool_topups_total{job=\"sponsord\", instance=~\"$instance\"}[1h])", + "refId": "A", + "legendFormat": "top-ups {{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "increase(sponsord_pool_keeper_failures_total{job=\"sponsord\", instance=~\"$instance\"}[1h])", + "refId": "B", + "legendFormat": "keeper failures {{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 8 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 9, + "panels": [], + "title": "Issuance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Capabilities issued per minute", + "description": "sponsord_capabilities_issued_total by registered: false is a fresh signer, true a re-signed capability for a signer already registered on-chain.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (instance, registered) (rate(sponsord_capabilities_issued_total{job=\"sponsord\", instance=~\"$instance\"}[$__rate_interval])) * 60", + "refId": "A", + "legendFormat": "{{instance}} registered={{registered}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Errors per minute by code", + "description": "sponsord_request_errors_total by code. chain_unavailable and internal are sponsord's own faults (SponsordRequestFailures); unauthorized, bad_request, exceeds_max, zero and signer_expired are the caller's.", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (instance, code) (rate(sponsord_request_errors_total{job=\"sponsord\", instance=~\"$instance\"}[$__rate_interval])) * 60", + "refId": "A", + "legendFormat": "{{instance}} {{code}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 11 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 12, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "sponsord log", + "description": "The daemon's journal (unit sponsord.service), level parsed by the grafana_alloy role. It logs errors only unless sponsord_log_level (RUST_LOG) is set.", + "type": "logs", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "queryType": "range", + "expr": "{unit=\"sponsord.service\", instance=~\"$instance\", level=~\"$level\"}" + } + ], + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": true, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 13 + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Onramp log", + "description": "sponsord-onramp's journal (unit sponsord-onramp.service), when the onramp runs on this host.", + "type": "logs", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refId": "A", + "queryType": "range", + "expr": "{unit=\"sponsord-onramp.service\", instance=~\"$instance\", level=~\"$level\"}" + } + ], + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": true, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 14 + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "decdn", + "sponsord" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-prom", + "value": "grafanacloud-prom" + }, + "hide": 0, + "includeAll": false, + "label": "Metrics", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-logs", + "value": "grafanacloud-logs" + }, + "hide": 0, + "includeAll": false, + "label": "Logs", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(up{job=\"sponsord\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up{job=\"sponsord\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Log level", + "multi": true, + "name": "level", + "options": [], + "query": "debug,info,warning,error", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "deCDN — sponsord", + "uid": "decdn-sponsord", + "version": 1 +} diff --git a/charts/decdn-node/files/monitoring/sponsord/prometheus-alerts.yml b/charts/decdn-node/files/monitoring/sponsord/prometheus-alerts.yml new file mode 100644 index 0000000..6fa067d --- /dev/null +++ b/charts/decdn-node/files/monitoring/sponsord/prometheus-alerts.yml @@ -0,0 +1,153 @@ +# sponsord reference Prometheus alerting rules +# +# For the deCDN onboarding sponsor (decdn/sponsord), whose /metrics the +# grafana_alloy role ships as job="sponsord". The series and what upstream asks +# operators to alert on are in upstream docs/operator.md, "Monitor"; the names +# are crates/sponsord/src/metrics.rs. Thresholds are operator-onboarding +# defaults, not normative. +# +# NOT covered: the treasury wallet's own USDC and gas balance, which pays every +# top-up. sponsord exports no such series; watch the address with a balance +# exporter of your own. SponsordKeeperFailing is the symptom when it runs dry. +# +# sponsord runs on the Ansible and Compose paths only. The decdn-node chart does +# NOT render this file (its PrometheusRule reads ../prometheus-alerts.yml). +# +# Validate locally: +# promtool check rules sponsord/prometheus-alerts.yml +# promtool test rules ../../tests/sponsord-alerts_test.yml +# +# Load into Grafana Cloud with mimirtool, or into Prometheus via rule_files (see +# ../README.md). + +groups: + - name: sponsord + interval: 30s + rules: + - alert: SponsordDown + expr: up{job="sponsord"} == 0 + for: 2m + labels: + component: sponsor + severity: critical + annotations: + summary: "sponsord on {{ $labels.instance }} unreachable for 2m" + description: | + Prometheus has been unable to scrape sponsord for two minutes. + No capability is issued and the pool is not topped up while it + is down, and the onramp in front of it answers every download + with an error. Read why on the host: + journalctl -u sponsord -e. Adjust the job="sponsord" selector + if your scrape job is named differently. + + # increase() over two keeper periods at the default + # SPONSORD_POOL_WATCH_INTERVAL_SECS (3600): a keeper that fails every + # tick keeps this firing, one that recovers lets it resolve. + - alert: SponsordKeeperFailing + expr: increase(sponsord_pool_keeper_failures_total[2h]) > 0 + for: 0m + labels: + component: sponsor + severity: warning + annotations: + runbook_url: "https://github.com/decdn/sponsord/blob/main/docs/operator.md#5-monitor" + summary: "sponsord pool keeper failing on {{ $labels.instance }}" + description: | + The keeper failed a chain read or a top-up in the last two + hours (sponsord_pool_keeper_failures_total rose). The usual + cause is a treasury wallet out of USDC or gas, or an RPC + endpoint that refuses it. The pool drains while top-ups fail. + journalctl -u sponsord names the failing call. + + # The keeper reads the pool at start, then every + # SPONSORD_POOL_WATCH_INTERVAL_SECS (3600 by default). Three missed + # periods; raise it if you lengthened the interval. A gauge of 0 (never + # read since start) also fires, after the same wait. + - alert: SponsordPoolCheckStale + expr: time() - sponsord_pool_last_check_unix > 3 * 3600 + for: 15m + labels: + component: sponsor + severity: warning + annotations: + runbook_url: "https://github.com/decdn/sponsord/blob/main/docs/operator.md#5-monitor" + summary: "sponsord has not read its pool balance for 3h on {{ $labels.instance }}" + description: | + sponsord_pool_last_check_unix has not advanced for three + keeper periods, so the pool balance on the dashboard is stale + and no top-up is being decided. Check the RPC endpoint and the + keeper's errors: journalctl -u sponsord -e. + + - alert: SponsordTopupHeld + expr: sponsord_pool_topup_unconfirmed_since_unix > 0 + for: 15m + labels: + component: sponsor + severity: warning + annotations: + runbook_url: "https://github.com/decdn/sponsord/blob/main/docs/operator.md#5-monitor" + summary: "sponsord is holding pool top-ups on {{ $labels.instance }}" + description: | + The keeper broadcast a top-up and could not read its receipt, + so it sends no further top-up until that transaction mines or + provably never can. DO NOT restart sponsord: a restart forgets + the hold, and the pool can be refilled twice. The deploy + tooling refuses restarts while this holds. It clears by itself + once the transaction mines. If it was dropped, send 0-value + transactions from the treasury to itself until it clears; the + error log line names the transaction and how many it takes. + + - alert: SponsordTopupHeldLong + expr: >- + sponsord_pool_topup_unconfirmed_since_unix > 0 + and time() - sponsord_pool_topup_unconfirmed_since_unix > 2 * 3600 + for: 0m + labels: + component: sponsor + severity: critical + annotations: + runbook_url: "https://github.com/decdn/sponsord/blob/main/docs/operator.md#5-monitor" + summary: "sponsord top-ups held for over 2h on {{ $labels.instance }}" + description: | + A top-up has sat unconfirmed for more than two hours, so the + pool is not being refilled. The transaction was most likely + dropped: send 0-value transactions from the treasury to itself + until the hold clears (the error log line says how many). Do + not restart sponsord to clear it. + + # The keeper has read the pool (last check > 0) and found it empty: every + # capability issued now is unusable until a top-up lands. + - alert: SponsordPoolEmpty + expr: >- + sponsord_pool_remaining_micro_usdc == 0 + and sponsord_pool_last_check_unix > 0 + for: 10m + labels: + component: sponsor + severity: critical + annotations: + runbook_url: "https://github.com/decdn/sponsord/blob/main/docs/operator.md#5-monitor" + summary: "sponsord pool empty on {{ $labels.instance }}" + description: | + The pool's remaining balance was 0 at the keeper's last read. + Sponsored downloads cannot pay nodes until it is refilled. + Check SponsordKeeperFailing and SponsordTopupHeld, and the + treasury wallet's USDC and gas. + + # chain_unavailable (503): a signer's on-chain registration could not be + # read; internal (500): a server-side failure. Both are sponsord's own + # faults, not a caller's. + - alert: SponsordRequestFailures + expr: increase(sponsord_request_errors_total{code=~"chain_unavailable|internal"}[15m]) > 0 + for: 0m + labels: + component: sponsor + severity: warning + annotations: + summary: "sponsord answering {{ $labels.code }} errors on {{ $labels.instance }}" + description: | + sponsord answered capability requests with {{ $labels.code }} + in the last 15 minutes. chain_unavailable means its RPC + endpoint could not read a signer's registration; internal is + a server-side failure. Users see failed downloads. + journalctl -u sponsord -e. diff --git a/charts/decdn-node/tests/render-test.sh b/charts/decdn-node/tests/render-test.sh index 7fa7bc2..459554c 100755 --- a/charts/decdn-node/tests/render-test.sh +++ b/charts/decdn-node/tests/render-test.sh @@ -11,6 +11,10 @@ # PROMTOOL command to run promtool, reading the rules on stdin (default: # `promtool`; the Makefile passes a digest-pinned container). Set # to "" to skip, loudly. +# PROMTOOL_IMAGE promtool's container image, for `promtool test rules`, which +# reads files rather than stdin: the test directory is mounted into +# it (the Makefile passes the same pinned image). Unset: the local +# `promtool`. Skipped with PROMTOOL. # DECDN_CLI path to a real `decdn` binary: also run `decdn config validate` # on every positive render (catches value/type errors the key check # can't). Skipped, loudly, when unset. @@ -142,10 +146,49 @@ if [ -n "$promtool" ]; then $promtool check rules < "$chart/files/monitoring/prometheus-alerts.yml" > "$work/promtool.out" 2>&1 \ || { cat "$work/promtool.out" >&2; fail "promtool check rules (files/monitoring/prometheus-alerts.yml)"; } pass "promtool check rules: $(grep -o '[0-9]* rules found' "$work/promtool.out")" + $promtool check rules < "$chart/files/monitoring/sponsord/prometheus-alerts.yml" > "$work/promtool.out" 2>&1 \ + || { cat "$work/promtool.out" >&2; fail "promtool check rules (files/monitoring/sponsord/prometheus-alerts.yml)"; } + pass "promtool check rules (sponsord): $(grep -o '[0-9]* rules found' "$work/promtool.out")" + + # The sponsord rules do time arithmetic on unix-time gauges, which only a unit + # test catches. promtool compares annotations exactly and they are prose, so the + # test runs against a copy with them stripped (tests/sponsord-alerts_test.yml). + mkdir "$work/rules-test" + yq 'del(.groups[].rules[].annotations)' "$chart/files/monitoring/sponsord/prometheus-alerts.yml" \ + > "$work/rules-test/sponsord-alerts.yml" + cp "$here/sponsord-alerts_test.yml" "$work/rules-test/" + chmod -R a+rX "$work/rules-test" + if [ -n "${PROMTOOL_IMAGE:-}" ]; then + docker run --rm -v "$work/rules-test:/w:ro" -w /w --entrypoint promtool "$PROMTOOL_IMAGE" \ + test rules sponsord-alerts_test.yml > "$work/promtool.out" 2>&1 + else + (cd "$work/rules-test" && promtool test rules sponsord-alerts_test.yml) > "$work/promtool.out" 2>&1 + fi || { cat "$work/promtool.out" >&2; fail "promtool test rules (tests/sponsord-alerts_test.yml)"; } + pass "promtool test rules: sponsord alerts" else - skipped+=("promtool (PROMTOOL is empty): alert rule syntax is NOT checked") + skipped+=("promtool (PROMTOOL is empty): alert rule syntax and the sponsord rule tests are NOT checked") fi +# The sponsord dashboard sits in a subdirectory, so the chart's files/monitoring/*.json +# glob never renders it (the ConfigMap check below pins that). It must still parse +# and carry a uid no node dashboard uses: Grafana imports by uid. +python3 - "$chart/files/monitoring" <<'PY' || fail "sponsord dashboard" +import json, pathlib, sys +root = pathlib.Path(sys.argv[1]) +uids = [json.load(open(p)).get("uid") for p in sorted(root.glob("*.json"))] +dash = sorted((root / "sponsord").glob("*.json")) +if not dash: + sys.exit("no dashboard in files/monitoring/sponsord/") +for p in dash: + uid = json.load(open(p)).get("uid") + if not (isinstance(uid, str) and 0 < len(uid) <= 40): + sys.exit(f"{p.name}: uid {uid!r} is not a 1-40 character string") + uids.append(uid) +if len(uids) != len(set(uids)): + sys.exit(f"duplicate dashboard uids: {sorted(uids)}") +PY +pass "sponsord dashboard parses with its own uid" + # The monitoring files the ci-values render must carry in full. yq -o=json '.' "$chart/files/monitoring/prometheus-alerts.yml" > "$work/alerts.json" for name in ci-values ci-origins ci-resolve-only; do @@ -241,6 +284,8 @@ if name == "ci-values.yaml": if pr: rules = [r for g in pr["spec"]["groups"] for r in g["rules"]] check(len(rules) == len(shipped) > 0, f"PrometheusRule has {len(rules)} rules, files/monitoring has {len(shipped)}") + # sponsord has no Kubernetes path: its rules must never reach the node's. + check(not [r for r in rules if r["alert"].startswith("Sponsord")], "sponsord rules in the PrometheusRule") check(all(r["labels"].get("team") == "node-ops" for r in rules), "ruleLabels not merged into every rule") # ci-values sets ruleLabels.severity: a rule's own severity must win, and # only a rule without one may take the extra. diff --git a/charts/decdn-node/tests/sponsord-alerts_test.yml b/charts/decdn-node/tests/sponsord-alerts_test.yml new file mode 100644 index 0000000..bf38f1d --- /dev/null +++ b/charts/decdn-node/tests/sponsord-alerts_test.yml @@ -0,0 +1,105 @@ +# promtool unit tests for files/monitoring/sponsord/prometheus-alerts.yml, run by +# render-test.sh (`make lint-helm`). The rules do time arithmetic on unix-time +# gauges, which promtool check cannot catch. The test clock starts at 0. +# +# render-test.sh runs this against a copy of the rules with the annotations +# stripped (promtool compares annotations exactly, and they are prose), placed +# beside this file as sponsord-alerts.yml. +rule_files: + - sponsord-alerts.yml + +evaluation_interval: 30s + +tests: + # Down for 2m fires; a healthy target does not. + - interval: 30s + input_series: + - series: 'up{job="sponsord", instance="a"}' + values: '0x10' + - series: 'up{job="sponsord", instance="b"}' + values: '1x10' + alert_rule_test: + - eval_time: 1m + alertname: SponsordDown + exp_alerts: [] + - eval_time: 3m + alertname: SponsordDown + exp_alerts: + - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: critical} + + # A hold warns after 15m and escalates once it is 2h old; 0 is no hold. + # The gauge holds the unix time the hold began (here t=60s). + - interval: 1m + input_series: + - series: 'sponsord_pool_topup_unconfirmed_since_unix{job="sponsord", instance="a"}' + values: '0 60x200' + - series: 'sponsord_pool_topup_unconfirmed_since_unix{job="sponsord", instance="b"}' + values: '0x200' + alert_rule_test: + - eval_time: 10m + alertname: SponsordTopupHeld + exp_alerts: [] + - eval_time: 20m + alertname: SponsordTopupHeld + exp_alerts: + - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: warning} + - eval_time: 2h + alertname: SponsordTopupHeldLong + exp_alerts: [] + - eval_time: 2h5m + alertname: SponsordTopupHeldLong + exp_alerts: + - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: critical} + + # Last check at t=60s and never again: stale once 3h have passed, plus for: 15m. + # A keeper that checks every hour never fires. + - interval: 5m + input_series: + - series: 'sponsord_pool_last_check_unix{job="sponsord", instance="a"}' + values: '60x60' + - series: 'sponsord_pool_last_check_unix{job="sponsord", instance="b"}' + values: '0+300x60' + alert_rule_test: + - eval_time: 3h + alertname: SponsordPoolCheckStale + exp_alerts: [] + - eval_time: 3h30m + alertname: SponsordPoolCheckStale + exp_alerts: + - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: warning} + + # An empty pool fires only once the keeper has read it. + - interval: 1m + input_series: + - series: 'sponsord_pool_remaining_micro_usdc{job="sponsord", instance="a"}' + values: '0x20' + - series: 'sponsord_pool_last_check_unix{job="sponsord", instance="a"}' + values: '60x20' + - series: 'sponsord_pool_remaining_micro_usdc{job="sponsord", instance="b"}' + values: '0x20' + - series: 'sponsord_pool_last_check_unix{job="sponsord", instance="b"}' + values: '0x20' + alert_rule_test: + - eval_time: 15m + alertname: SponsordPoolEmpty + exp_alerts: + - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: critical} + + # A keeper failure fires at once; the request-error rule ignores caller faults. + - interval: 1m + input_series: + - series: 'sponsord_pool_keeper_failures_total{job="sponsord", instance="a"}' + values: '0 0 1 1 1' + - series: 'sponsord_request_errors_total{job="sponsord", instance="a", code="chain_unavailable"}' + values: '0 0 2 2 2' + - series: 'sponsord_request_errors_total{job="sponsord", instance="a", code="bad_request"}' + values: '0 0 9 9 9' + alert_rule_test: + - eval_time: 3m + alertname: SponsordKeeperFailing + exp_alerts: + - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: warning} + - eval_time: 3m + alertname: SponsordRequestFailures + exp_alerts: + - exp_labels: {job: sponsord, instance: a, code: chain_unavailable, component: sponsor, severity: warning} From 1be4aef0fa063f2fa5a353fe1ec941d62358e89b Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 7 Oct 2026 21:40:57 +0300 Subject: [PATCH 4/6] feat(compose): custom onramp gate page, top-up hold check before restarts The onramp mounts /etc/sponsord/onramp-gate/ read-only for ONRAMP_GATE_TEMPLATE (a fixed directory, so no setting can mount a credential into the page server). The README adds the manual top-up hold check before a restart, a backup command, and notes the onramp ignores RUST_LOG. Co-Authored-By: Claude Opus 5.5 --- compose/README.md | 40 ++++++++++++++++++++++++++++- compose/compose.yaml | 10 ++++++++ compose/sponsord-onramp.env.example | 7 ++++- compose/tests/invariants.jq | 1 + 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/compose/README.md b/compose/README.md index a0ef053..dc499ec 100644 --- a/compose/README.md +++ b/compose/README.md @@ -169,6 +169,7 @@ what each setting does; this section covers the Compose side. | `/etc/sponsord/treasury-keystore.json` | `sponsord`, `0600` | sponsord | read-only file in `/run/secrets/` | | `/etc/sponsord/treasury-password` | `sponsord`, `0600` | sponsord | read-only file in `/run/secrets/` | | `/etc/sponsord/turnstile-secret` | `sponsord`, `0600` | onramp | read-only file in `/run/secrets/` | +| `/etc/sponsord/onramp-gate/` | `root`, `0755` | onramp | read-only directory, same path: an optional custom gate page | | `/var/lib/caddy/` | `caddy`, `0700` | Caddy | `/data`: certificates, ACME account | The paths match the Ansible roles, with one difference: the credential files belong @@ -241,6 +242,18 @@ the proxy. `sudoedit` writes the file back with its existing owner and mode. + **Your own gate page** (optional; upstream `docs/operator.md`, "Your own gate + page", lists its placeholders): put the HTML in `/etc/sponsord/onramp-gate/`, the + one directory the onramp mounts for it (created empty when absent), and point + `ONRAMP_GATE_TEMPLATE` at it in `sponsord-onramp.env`: + + ```bash + sudo install -m 0644 -o root -g root gate.html /etc/sponsord/onramp-gate/gate.html + # in sponsord-onramp.env: ONRAMP_GATE_TEMPLATE=/etc/sponsord/onramp-gate/gate.html + ``` + + A path the onramp cannot read stops it at start, with the path in its log. + 6. **TLS in front of the onramp.** Either: - **Caddy** (`caddy` profile): point the domain's DNS at this host and open **tcp/80** and **tcp/443** in the host firewall and the cloud security group. @@ -278,6 +291,22 @@ the proxy. As for the node, run these from the repository root with `sudo`. +**Before anything that restarts or stops sponsord** (a restart, a stop, an `up -d` +that recreates it, a treasury rotation), check that it holds no unconfirmed pool +top-up: + +```bash +curl -s 127.0.0.1:8090/metrics | grep '^sponsord_pool_topup_unconfirmed_since_unix' +``` + +Anything but `0` is a held top-up: the keeper broadcast one and could not read its +receipt, so it sends no other until that transaction mines or provably never can. A +restart forgets the hold, and the pool can then be refilled twice. Wait for it to +clear; if the transaction was dropped, send 0-value transactions from the treasury to +itself until it does (the error log line names the transaction and how many). The +Ansible role refuses such a restart on its own; Compose cannot, so this check is +yours. The `SponsordTopupHeld` alert (below) flags it. + - **Logs:** `sudo docker compose -f compose/compose.yaml logs -f sponsord sponsord-onramp caddy`. - **Stop:** `sudo docker compose -f compose/compose.yaml stop caddy sponsord-onramp sponsord` (leave out `caddy` without that profile). On SIGTERM sponsord waits for a pool @@ -298,7 +327,16 @@ As for the node, run these from the repository root with `sudo`. wallet must own `SPONSORD_POOL_ID`, or sponsord exits at start. - **Health:** no container healthchecks, as for the node. Probe the two `/healthz` endpoints above from the host. sponsord also serves `/metrics` on the same port, and - logs errors only unless `RUST_LOG=info` is set in `sponsord.env`. + logs errors only unless `RUST_LOG=info` is set in `sponsord.env`. The onramp ignores + `RUST_LOG` and always logs at info. +- **Dashboard and alerts:** a sponsord dashboard and alert rules (pool balance, keeper + failures, a held top-up) are in + [`charts/decdn-node/files/monitoring/sponsord/`](../charts/decdn-node/files/monitoring/README.md#sponsord). +- **Backup:** the treasury keystore and password are the only copy of the key that + owns the pool and its funds. Copy them off the host encrypted, e.g. + `sudo tar -C / -czf - etc/sponsord/treasury-keystore.json etc/sponsord/treasury-password etc/sponsord/api-token | age -r > sponsord.tar.age` + (the Ansible path's `make backup` does the same, see + [`docs/lifecycle.md`](../docs/lifecycle.md)). ## Before a release: a local image diff --git a/compose/compose.yaml b/compose/compose.yaml index d2b82bd..7c3a424 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -162,6 +162,16 @@ services: target: /run/secrets/turnstile-secret read_only: true bind: {create_host_path: false} + # A custom gate page (upstream docs/operator.md, "Your own gate page"): put + # the HTML in this directory and set ONRAMP_GATE_TEMPLATE to its path in + # sponsord-onramp.env. A fixed directory rather than a configurable file, so + # no setting can mount a credential into a server that publishes the page. + # Created empty when absent: unused, the built-in page is served. + - type: bind + source: /etc/sponsord/onramp-gate + target: /etc/sponsord/onramp-gate + read_only: true + bind: {create_host_path: true} # It reads the daemon's limits only at start, so `docker compose restart # sponsord` restarts it too, like the unit's PartOf= (a stop does not # propagate). It exits while the daemon is unreachable; the restart policy diff --git a/compose/sponsord-onramp.env.example b/compose/sponsord-onramp.env.example index c36b899..942c942 100644 --- a/compose/sponsord-onramp.env.example +++ b/compose/sponsord-onramp.env.example @@ -42,8 +42,13 @@ ONRAMP_CLI_SUMS_SHA256= # ONRAMP_SPENDING_CAP_MICRO_USDC= # ONRAMP_TTL_SECS= # ONRAMP_BRAND_NAME=deCDN +# Your own gate page instead of the built-in one (placeholders: upstream +# docs/operator.md). It must live in /etc/sponsord/onramp-gate/, the only +# directory compose.yaml mounts for it. +# ONRAMP_GATE_TEMPLATE=/etc/sponsord/onramp-gate/gate.html # POST /v1/fund and GET /v1/capability per address per minute; 0 turns a limit off # ONRAMP_FUND_RATE_PER_MIN=10 # ONRAMP_POLL_RATE_PER_MIN=120 # ONRAMP_RELEASES_BASE=https://github.com/decdn -# RUST_LOG=info +# No RUST_LOG: the onramp release is built without tracing's env-filter, so it +# ignores it and always logs at info. diff --git a/compose/tests/invariants.jq b/compose/tests/invariants.jq index dd8959c..d7e817a 100644 --- a/compose/tests/invariants.jq +++ b/compose/tests/invariants.jq @@ -29,6 +29,7 @@ def allowed_mounts: { ["/etc/sponsord/treasury-password", "/run/secrets/treasury-password", true], ["/etc/sponsord/treasury-keystore.json", "/run/secrets/treasury-keystore.json", true]], "sponsord-onramp": [["/etc/sponsord/api-token", "/run/secrets/api-token", true], + ["/etc/sponsord/onramp-gate", "/etc/sponsord/onramp-gate", true], ["/etc/sponsord/turnstile-secret", "/run/secrets/turnstile-secret", true]], "caddy": [["Caddyfile", "/etc/caddy/Caddyfile", true], ["/var/lib/caddy", "/data", false]] }; From e0d1c817d1c2bf41a5227fea0d0d273e8036fc0f Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 7 Oct 2026 22:15:23 +0300 Subject: [PATCH 5/6] fix: close the review findings on sponsord ops coverage - sponsord: the restart-inputs comparison is the only restart trigger, and the top-up hold guard runs before it queues anything, so a write landing back on the recorded content can no longer restart past the guard (nor can --force-handlers). The guard probes the address the running process listens on, passes a daemon that has not bound yet, requires sponsord's keeper series, accepts a labelled gauge, retries briefly, and words its refusal for a restart or a stop. The documented override is the JSON form the boolean check accepts. - decommission: check the hold again right before sponsord's stop (the prompt can wait minutes); each prompt lists every service the run stops on its hosts. - backup: the Turnstile secret at the onramp role's path, required on onramp hosts. - sponsord_onramp_log_level: the onramp does read RUST_LOG (as Targets: a level or target=level pairs); correct the docs that said it ignored it. - alerts: SponsordRequestFailures fires on a code's first appearance (upstream exports it from its first error, already non-zero); a pool never read since start is SponsordPoolNeverChecked (3h), not "stale" after 15m. Tests model the exporter as it is, with internal-code and resolve cases. - tests: a revert during a hold, an unreadable /metrics, backup and log-level negatives, the decommission play order, a second decommission run, no confirmation left behind by a refusal. - docs: co-located caveats in lifecycle.md, the gate page's path rule, safer manual check and backup in the Compose README, stale scenario and galaxy text. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/grafana-dashboards/SKILL.md | 18 +++- AGENTS.md | 17 ++-- ansible/Makefile | 3 +- ansible/galaxy/CHANGELOG.md | 30 ++++-- ansible/galaxy/README.md | 4 +- ansible/galaxy/galaxy.yml | 4 +- .../sponsord-onramp-lifecycle/molecule.yml | 5 +- .../sponsord-onramp-lifecycle/side_effect.yml | 17 +++- .../sponsord-onramp-lifecycle/verify.yml | 11 ++- ansible/molecule/sponsord-onramp/molecule.yml | 5 +- .../molecule/sponsord-onramp/side_effect.yml | 4 +- ansible/molecule/sponsord/files/sponsord-stub | 6 ++ ansible/molecule/sponsord/side_effect.yml | 70 +++++++++++++- .../molecule/validation-sponsord/converge.yml | 93 ++++++++++++++++++- ansible/playbooks/decommission.yml | 14 ++- ansible/roles/decdn_node/README.md | 3 +- .../roles/decdn_node/tasks/decommission.yml | 16 +++- ansible/roles/sponsord/README.md | 23 +++-- ansible/roles/sponsord/defaults/main.yml | 10 +- ansible/roles/sponsord/tasks/backup.yml | 15 ++- ansible/roles/sponsord/tasks/decommission.yml | 27 +++++- ansible/roles/sponsord/tasks/install.yml | 2 - ansible/roles/sponsord/tasks/main.yml | 34 ++++--- ansible/roles/sponsord/tasks/source.yml | 1 - ansible/roles/sponsord/tasks/topup-hold.yml | 80 ++++++++++++---- ansible/roles/sponsord_onramp/README.md | 10 +- .../roles/sponsord_onramp/defaults/main.yml | 8 ++ .../sponsord_onramp/tasks/decommission.yml | 16 +++- ansible/roles/sponsord_onramp/tasks/main.yml | 11 +++ .../templates/sponsord-onramp.env.j2 | 1 + charts/decdn-node/files/monitoring/README.md | 2 +- .../sponsord/dashboard-sponsord.json | 10 +- .../monitoring/sponsord/prometheus-alerts.yml | 47 ++++++++-- .../decdn-node/tests/sponsord-alerts_test.yml | 31 ++++++- compose/README.md | 30 ++++-- compose/compose.yaml | 2 + compose/sponsord-onramp.env.example | 10 +- docs/lifecycle.md | 38 ++++++-- tests/scripts-test.sh | 15 +++ 39 files changed, 589 insertions(+), 154 deletions(-) diff --git a/.claude/skills/grafana-dashboards/SKILL.md b/.claude/skills/grafana-dashboards/SKILL.md index 79e0ce2..4cfa3bd 100644 --- a/.claude/skills/grafana-dashboards/SKILL.md +++ b/.claude/skills/grafana-dashboards/SKILL.md @@ -43,13 +43,13 @@ their only home. - `sponsord/` — the onboarding sponsor's own pair: `dashboard-sponsord.json` (uid `decdn-sponsord`) and `prometheus-alerts.yml` (group `sponsord`, `job="sponsord"`). The chart does not render this subdirectory (sponsord has no Kubernetes path). Its - `sponsord_*` names come from upstream `decdn/sponsord` `crates/sponsord/src/metrics.rs`; - run the name check below with `sponsord_` and that file. Its logs are plain text, + `sponsord_*` names come from upstream `decdn/sponsord` `crates/sponsord/src/metrics.rs` + (check them with the recipe after the node one below). Its logs are plain text, not JSON: select `{unit="sponsord.service"}` and filter on the `level` label the `grafana_alloy` role parses. The rules have promtool unit tests in `charts/decdn-node/tests/sponsord-alerts_test.yml`: keep them in step. -Add a row to an existing dashboard before starting a fifth one. The Helm chart renders +Add a row to an existing dashboard before starting a fifth node one. The Helm chart renders these files as-is (`templates/prometheusrule.yaml`, `templates/dashboards-configmap.yaml`), so a new `*.json` becomes a new ConfigMap and `charts/decdn-node/tests/render-test.sh` checks the rule and dashboard counts against the directory. Never pass them through @@ -78,6 +78,18 @@ curl -s http://127.0.0.1:9090/metrics | grep -v '^#' | grep -oE '^decdn_[a-z0-9_ comm -23 /tmp/used /tmp/exported # must print nothing ``` +For sponsord, from the same directory (`sponsord_log_level` in a panel description is an +Ansible variable, not a series): + +```bash +grep -hv '^\s*#' sponsord/*.json sponsord/*.yml | grep -oE 'sponsord_[a-z0-9_]+' | sort -u > /tmp/used-sd +curl -s http://127.0.0.1:8090/metrics | grep -v '^#' | grep -oE '^sponsord_[a-z0-9_]+' | sort -u > /tmp/exported-sd +comm -23 /tmp/used-sd /tmp/exported-sd # prints only sponsord_log_level +``` + +A code's `sponsord_request_errors_total{code}` series appears only after that code's first +error, so a fresh daemon may not export it yet; check against upstream `metrics.rs` then. + `decdn_iroh_*` series appear only once the node's endpoint is up, and a labelled family with no child (e.g. `decdn_staker_set_active_by_region` before any staker declares a region) emits nothing — read a stray hit with that in mind. `decdn_health` is a Prometheus diff --git a/AGENTS.md b/AGENTS.md index baec403..1124754 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,7 +129,8 @@ and its rules have promtool unit tests (`charts/decdn-node/tests/sponsord-alerts role entry points (`tasks_from: backup|decommission`): backups are encrypted on the host to operator public keys; decommission needs `LIMIT` + typed confirmation, keeps the identity and never touches the chain. Both also cover `sponsord_hosts` (and the onramp), - with one confirmation per host (`_decdn_decommission_confirmed`; the + with one confirmation per run, in the first play that reaches a host, whose prompt + lists every service it covers (`_decdn_decommission_confirmed`; the `decdn_decommission_*` cap and timeout are copied into all three roles' defaults, `make test-scripts` checks). Leaving `decdn_rpc_url` empty means the operator wrote `0600 /etc/decdn/decdn.env` @@ -194,12 +195,14 @@ and its rules have promtool unit tests (`charts/decdn-node/tests/sponsord-alerts after the keystore decrypts and the on-chain pool-owner check passes. - **Top-up hold guard:** a restart or stop forgets a held, unconfirmed pool top-up (`sponsord_pool_topup_unconfirmed_since_unix` > 0), and the pool can be refilled - twice. `tasks/topup-hold.yml` reads `/metrics` before the role flushes a pending - restart and before decommission stops the daemon, and fails while a hold is on - (or `/metrics` does not answer). Every restart must go through it: a new notify - source must also be a hashed restart input, which is what "pending" reads. - `sponsord_restart_ignore_topup_hold` overrides it. Compose cannot guard itself; - its README has the manual check. + twice. `tasks/topup-hold.yml` reads `/metrics` (at the address the running + process listens on) before the role queues a restart and before decommission + stops the daemon, and fails while a hold is on or `/metrics` does not answer. + **The restart-inputs comparison is the role's only restart trigger:** never add + `notify: Restart sponsord` to another task; make the file a hashed input + instead, or the restart skips the guard. `sponsord_restart_ignore_topup_hold` + overrides it (pass it as JSON). Compose cannot guard itself; its README has the + manual check. - **Alloy toggles:** `playbooks/group_vars/all.yml` derives `grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled` from group membership. They are host-scoped so a co-located host's two plays render one diff --git a/ansible/Makefile b/ansible/Makefile index 277e1e5..7e3e1b7 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -128,7 +128,8 @@ decommission: # install-method and wallet side effects run in sponsord-install), sponsord-onramp # and sponsord-onramp-caddy (the onramp beside its daemon, behind the real distro # Caddy; the proxy side effects run in sponsord-onramp-caddy), sponsord-onramp-source -# (that converge, then both sponsord roles built from source), source-build (the +# (that converge, then both sponsord roles built from source), sponsord-onramp-lifecycle +# (that converge, then sponsord's backup and decommission), source-build (the # node built from source on the host: sandbox, stamps, rollback), and cloud-init and # cloud-init-sponsord (the cloud-init/ templates, booted for real). The glob in # scripts/molecule.sh is the authority; this list is a guide. diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index 600b9c0..a5ef0a0 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -187,13 +187,16 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - `sponsord`: a top-up hold guard. When the keeper broadcasts a pool top-up and cannot read its receipt, it holds further top-ups until that transaction mines (upstream sponsord #41); a restart forgets the hold and the pool can be refilled - twice. Before the role restarts sponsord (any changed restart input) it reads - `/metrics`, and fails while `sponsord_pool_topup_unconfirmed_since_unix` is - non-zero or `/metrics` does not answer, leaving the restart for the next run. - `sponsord_restart_ignore_topup_hold` overrides it. -- `sponsord`: `sponsord_log_level`, rendered as `RUST_LOG` (a level or tracing - directives; `""` keeps the daemon's errors-only default). There is no onramp - equivalent: the onramp release ignores `RUST_LOG`. + twice. Before the role restarts sponsord it reads `/metrics` at the address the + running daemon listens on, and fails while + `sponsord_pool_topup_unconfirmed_since_unix` is non-zero or `/metrics` does not + answer, before anything is queued, leaving the restart for the next run. A daemon + that has not bound yet holds nothing and passes. `sponsord_restart_ignore_topup_hold` + (passed as JSON) overrides it. +- `sponsord`, `sponsord_onramp`: `sponsord_log_level` / `sponsord_onramp_log_level`, + rendered as `RUST_LOG` (`""` keeps the daemon's errors-only and the onramp's info + default). The onramp is built without tracing's env-filter, so it takes a level or + `target=level` pairs only; the role refuses anything else. - `sponsord`: `tasks_from: backup` (`sponsord_backup_*`): the treasury keystore and password, the API token and the onramp's Turnstile secret (`secret.env` opt-in), taken hot and encrypted on the host to `sponsord_backup_age_recipients` (default @@ -201,9 +204,11 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - `sponsord`, `sponsord_onramp`: `tasks_from: decommission`. The onramp's stops it, removes its unit, and stops and disables the role's Caddy (marker-gated). sponsord's refuses while a top-up is held, then stops sponsord and removes its - unit. Both keep `/etc/sponsord` and never touch the chain. They share - `decdn_decommission_max_hosts` / `_prompt_seconds` / `_confirm` with `decdn_node`, - and a host that confirmed in an earlier play is not asked again. + unit, checking for a held top-up again right before the stop. Both keep + `/etc/sponsord` and never touch the chain. They share + `decdn_decommission_max_hosts` / `_prompt_seconds` / `_confirm` with `decdn_node`; + the first play to reach a host asks once, and its prompt lists every service the + run stops there. ### Removed @@ -218,6 +223,11 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Changed +- `sponsord`: the restart-inputs comparison is now the only thing that restarts the + daemon. The tasks that write secret.env, sponsord.env, the unit or the binary no + longer notify the handler themselves (each is a hashed input already), so a write + that lands back on the recorded content restarts nothing, and no restart can skip + the hold guard. Under `--check`, no restart is reported. - `playbooks/backup.yml` and `playbooks/decommission.yml` also cover `sponsord_hosts` (and `sponsord_onramp_hosts`): a backup of sponsord's credentials; a decommission that first checks every sponsord host for a held diff --git a/ansible/galaxy/README.md b/ansible/galaxy/README.md index 71a9001..5144036 100644 --- a/ansible/galaxy/README.md +++ b/ansible/galaxy/README.md @@ -9,8 +9,8 @@ repository — five roles and nothing else: | `decdn.node.baseline` | Debian/Ubuntu host baseline — nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo user, then DevSec OS + SSH hardening (applied last). | | `decdn.node.decdn_node` | The `decdn-node` daemon under a hardened systemd unit — from a GPG-verified release tarball (the default), a build on the host from a git ref, or locally built binaries; public QUIC udp/4433, loopback metrics + admin RPC. `decdn_network` sets the chain from upstream's manifest; `tasks_from: backup` / `decommission` for day 2. | | `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export; scrapes the node, `sponsord`, or both (`grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled`), and labels the onramp's journal (`grafana_alloy_sponsord_onramp_enabled`). | -| `decdn.node.sponsord` | The `sponsord` onboarding sponsor (treasury signer + PaymentPool keeper), standalone or beside a node — local binary or GPG-verified `sponsord-v*` release, `DynamicUser` unit with the API token and treasury wallet as systemd credentials, loopback-only API, `/healthz` deploy gate. `sponsord_network` sets the chain and PaymentPool from upstream's manifest. | -| `decdn.node.sponsord_onramp` | `sponsord-onramp`, sponsord's public side (Turnstile gate, installers, CLI API), on the daemon's host — local binary or GPG-verified `sponsord-onramp-v*` release, `DynamicUser` unit with the daemon token and Turnstile secret as systemd credentials, loopback listener behind Caddy (distro package, ACME TLS, admin API off; or your own proxy), `/healthz` deploy gate. | +| `decdn.node.sponsord` | The `sponsord` onboarding sponsor (treasury signer + PaymentPool keeper), standalone or beside a node — local binary or GPG-verified `sponsord-v*` release, `DynamicUser` unit with the API token and treasury wallet as systemd credentials, loopback-only API, `/healthz` deploy gate. `sponsord_network` sets the chain and PaymentPool from upstream's manifest. Restarts refuse while a pool top-up is held; `tasks_from: backup` / `decommission` for day 2. | +| `decdn.node.sponsord_onramp` | `sponsord-onramp`, sponsord's public side (Turnstile gate, installers, CLI API), on the daemon's host — local binary or GPG-verified `sponsord-onramp-v*` release, `DynamicUser` unit with the daemon token and Turnstile secret as systemd credentials, loopback listener behind Caddy (distro package, ACME TLS, admin API off; or your own proxy), `/healthz` deploy gate; `tasks_from: decommission`. | ## Requirements diff --git a/ansible/galaxy/galaxy.yml b/ansible/galaxy/galaxy.yml index 3e63d97..2c7add2 100644 --- a/ansible/galaxy/galaxy.yml +++ b/ansible/galaxy/galaxy.yml @@ -16,8 +16,8 @@ authors: - deCDN Contributors description: >- Deploy and harden a public deCDN node: DevSec host baseline, the decdn_node - daemon (with backup and decommission), the sponsord onboarding sponsor and its - public onramp, and opt-in Grafana Cloud observability. + daemon and the sponsord onboarding sponsor with its public onramp (each with + backup and decommission), and opt-in Grafana Cloud observability. license: - MIT # Galaxy tags must match ^[a-z0-9]+$ (no hyphens/underscores). diff --git a/ansible/molecule/sponsord-onramp-lifecycle/molecule.yml b/ansible/molecule/sponsord-onramp-lifecycle/molecule.yml index 8afcfb1..ff1b1c1 100644 --- a/ansible/molecule/sponsord-onramp-lifecycle/molecule.yml +++ b/ansible/molecule/sponsord-onramp-lifecycle/molecule.yml @@ -8,12 +8,13 @@ # decommission refusals a held pool top-up (the stub's hold marker), a # missing confirmation, a wrong one: nothing touched; # playbooks/decommission.yml for real: the onramp, its Caddy and sponsord stopped -# and removed, Alloy torn down, every credential kept. +# and removed, Alloy torn down, every credential kept; +# then once more, which must be a safe no-op. # verify.yml here decrypts the archive and checks what decommission removed and kept. # No idempotence step: every backup writes a new archive and decommission is one-shot. # # The inventory below must stay identical to ../sponsord-onramp/molecule.yml's -# (`make test-scripts` checks); the stubs are that scenario's files. +# (`make test-scripts` checks); the stubs are those it names. driver: name: docker platforms: diff --git a/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml b/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml index 36f4f72..619547c 100644 --- a/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml +++ b/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml @@ -1,6 +1,6 @@ --- # Backup, then decommission: refused while sponsord holds a pool top-up, without a -# confirmation and with a wrong one, then for real through the playbook. The +# confirmation and with a wrong one, then for real through the playbook, twice. The # header of molecule.yml lists what each step proves; verify.yml checks the result. - name: Prepare the backup recipient hosts: all @@ -46,7 +46,7 @@ # Confirmed, so only the hold can stop it. - name: Decommission while a top-up is held block: - - name: Run the decommission playbook's hold check + - name: Run sponsord's decommission (confirmed) during a hold ansible.builtin.include_role: name: sponsord tasks_from: decommission @@ -119,9 +119,22 @@ A rejected decommission stopped something: {{ _molecule_units.results | map(attribute='status.ActiveState') | list }}. + # A refusal must not leave the run-wide confirmation behind, or a later play + # of a real run would skip its prompt. + - name: Require no rejected run to have recorded a confirmation + ansible.builtin.assert: + that: + - _decdn_decommission_confirmed is not defined + fail_msg: A rejected decommission left _decdn_decommission_confirmed set. + - name: Confirm the real decommission non-interactively ansible.builtin.set_fact: decdn_decommission_confirm: "{{ ansible_play_hosts_all | sort | join(',') }}" - name: Decommission ansible.builtin.import_playbook: ../../playbooks/decommission.yml + +# Re-running must be safe: the units are gone, sponsord is not running (so the hold +# check passes), and every removal is already done. +- name: Decommission again + ansible.builtin.import_playbook: ../../playbooks/decommission.yml diff --git a/ansible/molecule/sponsord-onramp-lifecycle/verify.yml b/ansible/molecule/sponsord-onramp-lifecycle/verify.yml index e7f2a71..2760959 100644 --- a/ansible/molecule/sponsord-onramp-lifecycle/verify.yml +++ b/ansible/molecule/sponsord-onramp-lifecycle/verify.yml @@ -65,6 +65,8 @@ - /etc/systemd/system/sponsord.service - /etc/systemd/system/sponsord-onramp.service - /etc/systemd/system/alloy.service + - /etc/sponsord/.inputs.sha256 + - /etc/sponsord/.onramp-inputs.sha256 - /etc/sponsord/treasury-keystore.json - /etc/sponsord/treasury-password - /etc/sponsord/api-token @@ -75,11 +77,12 @@ - name: Require the units gone and every credential kept ansible.builtin.assert: that: - - _after.results[0:3] | map(attribute='stat.exists') | unique == [false] - - _after.results[3:] | map(attribute='stat.exists') | unique == [true] + - _after.results[0:5] | map(attribute='stat.exists') | unique == [false] + - _after.results[5:] | map(attribute='stat.exists') | unique == [true] fail_msg: >- - Decommission must remove the sponsord, onramp and Alloy units and keep the - treasury files, the API token, the Turnstile secret and the Caddyfile: + Decommission must remove the sponsord, onramp and Alloy units and both + restart-input records, and keep the treasury files, the API token, the + Turnstile secret and the Caddyfile: {{ _after.results | map(attribute='item') | zip(_after.results | map(attribute='stat.exists')) | list }} - name: Read the services' state diff --git a/ansible/molecule/sponsord-onramp/molecule.yml b/ansible/molecule/sponsord-onramp/molecule.yml index 32b20e4..adc8e56 100644 --- a/ansible/molecule/sponsord-onramp/molecule.yml +++ b/ansible/molecule/sponsord-onramp/molecule.yml @@ -19,8 +19,9 @@ # https chain through Caddy, the client address Caddy passes on. # # The proxy side effects (proxy "none", ACME, a Caddyfile the role did not write) -# run in ../sponsord-onramp-caddy on the same converge, so the two run side by side. -# Its inventory below must stay identical to this one (`make test-scripts` checks). +# run in ../sponsord-onramp-caddy on the same converge, so the two run side by side; +# so do ../sponsord-onramp-source and ../sponsord-onramp-lifecycle. Their +# inventories must stay identical to this one (`make test-scripts` checks). # # `baseline` is skipped (real-host-only — see ../default/molecule.yml), so the # tcp/80 + tcp/443 holes are covered by tests/firewall-holes instead. Release diff --git a/ansible/molecule/sponsord-onramp/side_effect.yml b/ansible/molecule/sponsord-onramp/side_effect.yml index 7cd785a..df9653e 100644 --- a/ansible/molecule/sponsord-onramp/side_effect.yml +++ b/ansible/molecule/sponsord-onramp/side_effect.yml @@ -13,8 +13,8 @@ # onramp; an unchanged re-run is a no-op; an emptied value never adopts the # role's own file; with the record discarded the host's file is accepted. # The proxy side effects (proxy "none", ACME, a foreign Caddyfile) run in -# ../sponsord-onramp-caddy, and the source install in ../sponsord-onramp-source, -# beside this scenario. +# ../sponsord-onramp-caddy, the source install in ../sponsord-onramp-source, and +# backup and decommission in ../sponsord-onramp-lifecycle, beside this scenario. # The host is left healthy on the role's own Caddyfile, `tls internal`, for verify. - name: Side effect — rotation, the fatal gate and the restart-input record hosts: all diff --git a/ansible/molecule/sponsord/files/sponsord-stub b/ansible/molecule/sponsord/files/sponsord-stub index 6ba7f54..2c597c8 100755 --- a/ansible/molecule/sponsord/files/sponsord-stub +++ b/ansible/molecule/sponsord/files/sponsord-stub @@ -41,6 +41,8 @@ UNHEALTHY_MARKER = "/etc/sponsord/stub-unhealthy" # it holds (the real daemon's sponsord_pool_topup_unconfirmed_since_unix), so the # role's restart guard can be exercised without a chain. HOLD_MARKER = "/etc/sponsord/stub-topup-hold" +# While this file exists, /metrics answers 503: the hold state cannot be read. +METRICS_DOWN_MARKER = "/etc/sponsord/stub-metrics-down" # Must match sponsord_rpc_url in the scenario's inventory, which carries shell # metacharacters and quotes on purpose. EXPECTED_RPC_URL = "https://rpc.example.invalid/?key=$HOME$(touch /tmp/sponsord-rpc-expanded)&q=o'brien \"x\"" @@ -108,6 +110,9 @@ class Handler(BaseHTTPRequestHandler): if self.path == "/healthz": body, ctype = b'{"ok":true}', "application/json" elif self.path == "/metrics": + if os.path.exists(METRICS_DOWN_MARKER): + self.send_error(503) + return try: with open(HOLD_MARKER, "rb") as f: held = int(f.read().strip() or b"0") @@ -115,6 +120,7 @@ class Handler(BaseHTTPRequestHandler): held = 0 body = ( b"sponsord_pool_topups_total 0\n" + b"sponsord_pool_keeper_failures_total 0\n" b"# TYPE sponsord_pool_topup_unconfirmed_since_unix gauge\n" + f"sponsord_pool_topup_unconfirmed_since_unix {held}\n".encode() ) diff --git a/ansible/molecule/sponsord/side_effect.yml b/ansible/molecule/sponsord/side_effect.yml index d8f3ac6..6f9b6a4 100644 --- a/ansible/molecule/sponsord/side_effect.yml +++ b/ansible/molecule/sponsord/side_effect.yml @@ -11,9 +11,11 @@ # recorded `host` and locked to 0600; a host edit restarts the daemon; an # inventory URL must not overwrite the host's file until confirmed. # 4. The top-up hold guard: while /metrics reports a held pool top-up, a converge -# that would restart the daemon fails before the restart, keeps the old PID -# and the old restart-inputs record, and the next converge after the hold -# clears applies the change; sponsord_restart_ignore_topup_hold overrides it. +# that would restart the daemon fails before the restart, keeping the old PID +# and the old restart-inputs record; reverting the change during the hold +# restarts nothing (the revert writes the recorded content back, which must +# not slip a restart past the guard); the override restarts during the hold; +# an unreadable /metrics refuses too; a plain converge restarts once it clears. # Release mode and treasury wallet generation run in ../sponsord-install, beside # this scenario. The first play must run on a fresh converge (it asserts no restart # yet), and it records the API token verify.yml compares against. @@ -353,6 +355,25 @@ {{ _sp_hold_before.status.MainPID }} -> {{ _sp_hold_refused.status.MainPID }}) or rewrote {{ _sp_inputs }}, so the next run would not retry the restart. + # The refused run left the 7200 env on disk. Reverting writes back exactly what + # the daemon is running on: nothing to restart, so no restart, and no refusal. + - name: Revert the change while the top-up is still held + ansible.builtin.include_role: + name: sponsord + + - name: Read the unit state after the revert + ansible.builtin.systemd_service: + name: sponsord + register: _sp_hold_reverted + + - name: Require the revert not to have restarted sponsord + ansible.builtin.assert: + that: + - _sp_hold_reverted.status.MainPID == _sp_hold_before.status.MainPID + fail_msg: >- + Reverting the refused change restarted sponsord during the hold (PID + {{ _sp_hold_before.status.MainPID }} -> {{ _sp_hold_reverted.status.MainPID }}). + - name: Converge the same change with the override ansible.builtin.include_role: name: sponsord @@ -376,6 +397,47 @@ path: /etc/sponsord/stub-topup-hold state: absent + # The override run left 7200 recorded; the inventory's 3600 is a pending restart. + - name: Make the stub's /metrics unreadable + ansible.builtin.copy: + dest: /etc/sponsord/stub-metrics-down + content: "" + mode: "0644" + + - name: Converge a change while the hold state cannot be read + block: + - name: Run the role against an unreadable /metrics + ansible.builtin.include_role: + name: sponsord + + - name: Fail — the guard restarted with the hold state unknown + ansible.builtin.fail: + msg: The role finished although /metrics answered 503. + rescue: + - name: Require the failure to come from the unreadable-state refusal + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Refuse to proceed while the hold state cannot be read' + fail_msg: >- + Expected the role to refuse at the unreadable-state check, but it failed + at "{{ ansible_failed_task.name }}". + always: + - name: Make the stub's /metrics readable again + ansible.builtin.file: + path: /etc/sponsord/stub-metrics-down + state: absent + + - name: Read the unit state after the unreadable-state refusal + ansible.builtin.systemd_service: + name: sponsord + register: _sp_hold_unknown + + - name: Require no restart while the hold state was unknown + ansible.builtin.assert: + that: + - _sp_hold_unknown.status.MainPID == _sp_hold_override.status.MainPID + fail_msg: sponsord restarted although its /metrics answered 503. + # Back on the inventory's TTL, which verify.yml checks. - name: Converge once the hold has cleared ansible.builtin.include_role: @@ -389,5 +451,5 @@ - name: Require the plain converge to restart once the hold cleared ansible.builtin.assert: that: - - _sp_hold_cleared.status.MainPID != _sp_hold_override.status.MainPID + - _sp_hold_cleared.status.MainPID != _sp_hold_unknown.status.MainPID fail_msg: The converge after the hold cleared did not restart sponsord. diff --git a/ansible/molecule/validation-sponsord/converge.yml b/ansible/molecule/validation-sponsord/converge.yml index 5e64700..17f9e31 100644 --- a/ansible/molecule/validation-sponsord/converge.yml +++ b/ansible/molecule/validation-sponsord/converge.yml @@ -242,6 +242,82 @@ decdn_rejected: "{{ decdn_rejected + ['sponsord-log-level'] }}" when: ansible_failed_task.name == "Validate sponsord_log_level" + # A newline would add a line to the EnvironmentFile after SPONSORD_BIND, and + # the later assignment wins: this one would put the API on every interface. + - name: "Case sponsord-log-level — a log level carrying a second env line" + block: + - name: Run sponsord with a log level holding a newline + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_log_level: "info\nSPONSORD_BIND=0.0.0.0:8090" + rescue: + - name: Record sponsord-log-level rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-log-level'] }}" + when: ansible_failed_task.name == "Validate sponsord_log_level" + + # --- sponsord tasks_from: backup (its settings are checked before any file) --- + - name: "Case sponsord-backup — a private key as a recipient" + block: + - name: Run the sponsord backup with a private key as its recipient + ansible.builtin.include_role: + name: sponsord + tasks_from: backup + vars: + # kics-scan ignore-line (a fake key under test, not a secret) + sponsord_backup_age_recipients: ["AGE-SECRET-KEY-1{{ 'Q' * 58 }}"] + rescue: + - name: Record sponsord-backup rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-backup'] }}" + when: ansible_failed_task.name == "Validate the backup settings" + + - name: "Case sponsord-backup — no recipients" + block: + - name: Run the sponsord backup with no recipients + ansible.builtin.include_role: + name: sponsord + tasks_from: backup + vars: + sponsord_backup_age_recipients: [] + rescue: + - name: Record sponsord-backup rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-backup'] }}" + when: ansible_failed_task.name == "Validate the backup settings" + + # The role makes the backup dir root 0700: /etc would lock the host out. + - name: "Case sponsord-backup-dir — a system directory" + block: + - name: Run the sponsord backup into /etc + ansible.builtin.include_role: + name: sponsord + tasks_from: backup + vars: + sponsord_backup_age_recipients: ["age1{{ 'q' * 58 }}"] + sponsord_backup_dir: /etc + rescue: + - name: Record sponsord-backup-dir rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-backup-dir'] }}" + when: ansible_failed_task.name == "Require the backup dir to stay clear of sponsord and system directories" + + - name: "Case sponsord-backup-dir — inside the credentials directory" + block: + - name: Run the sponsord backup into /etc/sponsord + ansible.builtin.include_role: + name: sponsord + tasks_from: backup + vars: + sponsord_backup_age_recipients: ["age1{{ 'q' * 58 }}"] + sponsord_backup_dir: /etc/sponsord/backups + rescue: + - name: Record sponsord-backup-dir rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-backup-dir'] }}" + when: ansible_failed_task.name == "Require the backup dir to stay clear of sponsord and system directories" + - name: "Case sponsord-rpc-newline — an RPC URL that injects a second variable" block: - name: Run sponsord with an RPC URL that injects a second variable @@ -878,6 +954,20 @@ decdn_rejected: "{{ decdn_rejected + ['onramp-limit'] }}" when: ansible_failed_task.name == "Validate the onramp's limits" + # A span filter: the onramp's Targets parser rejects it and then logs nothing. + - name: "Case onramp-log-level — a span filter" + block: + - name: Run sponsord_onramp with a span-filter log level + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_log_level: "[fund]=debug" + rescue: + - name: Record onramp-log-level rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-log-level'] }}" + when: ansible_failed_task.name == "Validate sponsord_onramp_log_level" + - name: "Case onramp-ip-header — a header name with a space" block: - name: Run sponsord_onramp with a header name with a space @@ -1173,7 +1263,8 @@ "sponsord-network-chain", "sponsord-pool-id", "sponsord-pool-id-empty", "sponsord-wallet-bool", "sponsord-wallet-cli", "sponsord-zero-address", "sponsord-bind", "sponsord-limit", "sponsord-limit", "sponsord-bool", "sponsord-bool", - "sponsord-log-level", "sponsord-rpc-newline", + "sponsord-log-level", "sponsord-log-level", "sponsord-backup", "sponsord-backup", + "sponsord-backup-dir", "sponsord-backup-dir", "onramp-log-level", "sponsord-rpc-newline", "sponsord-rpc-gate", "sponsord-treasury", "sponsord-treasury", "sponsord-token", "sponsord-dir-missing", "sponsord-dir-format", "sponsord-dir-cross", "sponsord-bind", "sponsord-overwrite-untracked", "sponsord-token", diff --git a/ansible/playbooks/decommission.yml b/ansible/playbooks/decommission.yml index e5d6366..2cf02af 100644 --- a/ansible/playbooks/decommission.yml +++ b/ansible/playbooks/decommission.yml @@ -1,9 +1,11 @@ --- -# Take a host out of service: stop and remove every service this repo deployed on -# it (decdn-node, sponsord-onramp and its Caddy, sponsord), then tear down the -# opt-in Grafana Alloy agent. Keeps the identities and keys (the node's keystore -# withdraws the bond; sponsord's treasury keystore owns the pool) and does NOT touch -# the chain; each role's last task prints what is left to do on-chain. +# Take a host out of service: stop every service this repo deployed on it and +# remove its unit (decdn-node, sponsord-onramp, sponsord; the onramp's Caddy is +# stopped and disabled, not removed), then tear down the opt-in Grafana Alloy agent. +# Keeps the identities and keys (the node's keystore withdraws the bond; sponsord's +# treasury keystore owns the pool) and does NOT touch the chain; each role's last +# task prints what is left to do. One typed confirmation, in the first play that +# reaches a host, names every service it covers there. # make decommission LIMIT= # LIMIT is required; asks for typed confirmation # Take a backup first (make backup). See docs/lifecycle.md. @@ -19,6 +21,8 @@ ansible.builtin.include_role: name: sponsord tasks_from: topup-hold + vars: + _sponsord_hold_action: stop - name: Decommission deCDN nodes hosts: decdn_nodes diff --git a/ansible/roles/decdn_node/README.md b/ansible/roles/decdn_node/README.md index 3bff3af..65c4a18 100644 --- a/ansible/roles/decdn_node/README.md +++ b/ansible/roles/decdn_node/README.md @@ -634,7 +634,8 @@ decdn node drain --wait # graceful shutdown. Without --wait it is fire-and-forg # Under this role's unit (Restart=always) systemd starts # a drained daemon again 5 s later: to take the node # down, use `systemctl stop decdn-node` (same graceful - # path, via SIGTERM) or `make decommission`. + # path, via SIGTERM) or `make decommission` + # (which also removes a co-located sponsord). # Read-only chain query, not admin RPC: lists active registered nodes and maps # node-ids/regions to operator addresses. Loads no keystore and spends nothing. diff --git a/ansible/roles/decdn_node/tasks/decommission.yml b/ansible/roles/decdn_node/tasks/decommission.yml index cf080d7..8ccd25f 100644 --- a/ansible/roles/decdn_node/tasks/decommission.yml +++ b/ansible/roles/decdn_node/tasks/decommission.yml @@ -44,14 +44,26 @@ - name: Ask for typed confirmation ansible.builtin.pause: prompt: >- - About to STOP decdn-node and REMOVE its systemd unit on: + About to STOP {{ _decom_services | join(', ') }}, REMOVE their systemd units + and tear down this repo's Grafana Alloy on: {{ ansible_play_hosts_all | sort | join(',') }}. - On-chain deregistration and unbonding are NOT done by this playbook. + This one confirmation covers all of them. Nothing on-chain is done (no + deregistration, unbonding or pool withdrawal). Type the host list above exactly to continue (times out after {{ decdn_decommission_prompt_seconds }} s) seconds: "{{ decdn_decommission_prompt_seconds | int }}" register: _decdn_decommission_typed run_once: true + vars: + # Every service this repo's playbooks/decommission.yml stops on these hosts, by + # their groups, so the one confirmation names all it covers (a later play does + # not ask again). This role's own service is always listed. + _decom_groups: "{{ ansible_play_hosts_all | map('extract', hostvars, 'group_names') | flatten | unique }}" + _decom_services: >- + {{ ((['decdn-node'] if 'decdn_nodes' in _decom_groups else []) + + (['sponsord-onramp and its Caddy'] if 'sponsord_onramp_hosts' in _decom_groups else []) + + (['sponsord'] if 'sponsord_hosts' in _decom_groups else []) + + ['decdn-node']) | unique }} when: - decdn_decommission_confirm is not defined - not (_decdn_decommission_confirmed | default(false) | bool) diff --git a/ansible/roles/sponsord/README.md b/ansible/roles/sponsord/README.md index 7cff58e..8d818eb 100644 --- a/ansible/roles/sponsord/README.md +++ b/ansible/roles/sponsord/README.md @@ -62,11 +62,14 @@ gate, installers, behind a TLS reverse proxy), is the keystore and confirmed on-chain that the treasury owns `sponsord_pool_id`. If `/healthz` does not answer 200 within the readiness window, the deploy fails. sponsord has no `config validate` command, so this is the config check. -- **Restarts on out-of-band changes.** The role records a hash of everything the - daemon starts from (credentials, both env files, the unit, the binary), once the - daemon is healthy. If any of them differs on the next run, it restarts the daemon: +- **Restarts exactly when an input changed.** The role records a hash of everything + the daemon starts from (credentials, both env files, the unit, the binary), once + the daemon is healthy. That comparison is the only thing that restarts it: a run + that changes an input, or finds one changed, restarts the daemon (through the + top-up hold guard, see Day 2): + - this run rendered a new setting, unit or binary; - you replaced a credential or edited a host-provisioned `secret.env`; - - an earlier run changed a file and then failed before its restart handler ran. + - an earlier run changed a file and then failed before restarting. ## Before the first deploy @@ -135,7 +138,7 @@ See [`defaults/main.yml`](defaults/main.yml) for the full list with comments. | `sponsord_secret_env_overwrite_host_file` | `false` | Confirm that `sponsord_rpc_url` may replace a `secret.env` the role did not write. | | `sponsord_log_level` | `""` | `RUST_LOG`: a level (`info`) or tracing directives. `""` logs errors only. | | `sponsord_stop_timeout_sec` | `120` | Cap on the graceful stop. The daemon waits for a top-up it already sent. | -| `sponsord_restart_ignore_topup_hold` | `false` | `true` restarts despite a held top-up (see Day 2). Only for a hold that can never mine. | +| `sponsord_restart_ignore_topup_hold` | `false` | `true` restarts or stops despite a held top-up or an unreadable `/metrics` (see Day 2). Only for a hold that can never mine; pass it as JSON. | | `sponsord_backup_age_recipients` | `decdn_backup_age_recipients` | `tasks_from: backup`: public keys the archive is encrypted to. | | `sponsord_backup_include_env` / `_dir` / `_fetch` / `_local_dir` | `false` / `/var/backups/sponsord` / `true` / `ansible/backups` | `tasks_from: backup`. | | `decdn_decommission_max_hosts` / `_prompt_seconds` | `1` / `300` | `tasks_from: decommission`, shared with the other roles. | @@ -169,7 +172,10 @@ export it, so watch the address yourself. next run retries. Wait for the hold to clear (upstream `docs/operator.md`, "Monitor": if the transaction was dropped, send 0-value transactions from the treasury to itself), then re-run. A running daemon whose `/metrics` does not answer - is refused too, since its hold state is unknown. + is refused too, since its hold state is unknown. The override is + `ANSIBLE_ARGS='-e {"sponsord_restart_ignore_topup_hold":true}'`. A restart the role + does not cause (a crash, a package upgrade restarting services) is not guarded: + the `SponsordTopupHeld` alert is how you learn of a hold. - **Rotate the API token:** write the new token to `/etc/sponsord/api-token`, then re-run the role, which restarts sponsord. The onramp reads the same file and restarts with the daemon (`PartOf=`). @@ -193,8 +199,9 @@ export it, so watch the address yourself. After a `source` install, the build user and its home stay behind (shared with the other roles on the host): `sudo userdel decdn-build && sudo rm -rf - /var/lib/decdn-build` once nothing on the host builds from source. Then delete `/etc/sponsord` (secrets, env files and the role's two `.sha256` - records) once the treasury files are safe elsewhere and the pool is drained. + /var/lib/decdn-build` once nothing on the host builds from source. Then delete `/etc/sponsord` (secrets, env files and the + `.secret.env.sha256` record) once the treasury files are safe elsewhere and the + pool is drained. ## Testing diff --git a/ansible/roles/sponsord/defaults/main.yml b/ansible/roles/sponsord/defaults/main.yml index 009a703..f90723e 100644 --- a/ansible/roles/sponsord/defaults/main.yml +++ b/ansible/roles/sponsord/defaults/main.yml @@ -106,7 +106,7 @@ sponsord_pool_watch_interval_secs: "" # [3600] how often the keeper checks # --- Logging ---------------------------------------------------------------------- # RUST_LOG for the daemon: a level (info) or tracing directives (sponsord=debug, # hyper=warn). "" omits it, and the daemon then logs errors only; info adds the -# listener and each top-up. No spaces or quotes. +# listener and each top-up. Letters, digits and _=,.:- only (no span filters). sponsord_log_level: "" # --- Secrets on the host -------------------------------------------------------- @@ -171,9 +171,11 @@ sponsord_stop_timeout_sec: 120 # (sponsord_pool_topup_unconfirmed_since_unix > 0 on /metrics). A restart forgets the # hold, and if the transaction is still pending the pool can be refilled twice # (upstream docs/operator.md, "Monitor"). So the role reads /metrics before any -# restart or stop it is about to cause, and fails instead while a hold is on; the -# next converge retries once it clears. true restarts anyway: only for a hold you -# have confirmed can never mine. +# restart it is about to cause (and decommission before it stops the daemon), and +# fails instead while a hold is on or /metrics does not answer; the next run retries +# once it clears. true skips both refusals: only for a hold you have confirmed can +# never mine. Pass it as JSON (-e '{"sponsord_restart_ignore_topup_hold":true}'): +# -e name=true is a string, which the boolean check refuses. sponsord_restart_ignore_topup_hold: false # --- Readiness -------------------------------------------------------------------- diff --git a/ansible/roles/sponsord/tasks/backup.yml b/ansible/roles/sponsord/tasks/backup.yml index 2cbeaa9..d080124 100644 --- a/ansible/roles/sponsord/tasks/backup.yml +++ b/ansible/roles/sponsord/tasks/backup.yml @@ -42,6 +42,11 @@ # The archive is written into sponsord_backup_dir, which the role makes root 0700, # so its shape is enforced before anything is touched (as decdn_node's # validate-lifecycle-paths.yml). +- name: Resolve the onramp's Turnstile secret path + ansible.builtin.set_fact: + _sponsord_backup_turnstile_file: >- + {{ sponsord_onramp_turnstile_secret_file | default(sponsord_etc ~ '/turnstile-secret') }} + - name: Constrain the backup paths (absolute, no traversal, no empty segments) ansible.builtin.assert: that: @@ -56,6 +61,7 @@ loop: - {name: sponsord_etc, value: "{{ sponsord_etc }}"} - {name: sponsord_backup_dir, value: "{{ sponsord_backup_dir }}"} + - {name: sponsord_onramp_turnstile_secret_file, value: "{{ _sponsord_backup_turnstile_file }}"} loop_control: label: "{{ item.name }}" @@ -73,17 +79,20 @@ outside {{ sponsord_etc }}, and not a system directory: the role makes it root-only (0700) and writes archives into it. The default is /var/backups/sponsord. -# The onramp keeps its Turnstile secret beside the daemon's files; archived when present. +# The onramp's Turnstile secret, at the onramp role's path when inventory moves it. +# Required on a host in sponsord_onramp_hosts (the "Require every path" check +# below fails without it); archived when present anywhere else. - name: Check for the onramp's Turnstile secret ansible.builtin.stat: - path: "{{ sponsord_etc }}/turnstile-secret" + path: "{{ _sponsord_backup_turnstile_file }}" register: _sponsord_backup_turnstile - name: Build the list of paths to archive ansible.builtin.set_fact: _sponsord_backup_paths: >- {{ [sponsord_treasury_keystore_file, sponsord_treasury_password_file, sponsord_api_token_file] - + ([sponsord_etc ~ '/turnstile-secret'] if _sponsord_backup_turnstile.stat.exists else []) + + ([_sponsord_backup_turnstile_file] + if _sponsord_backup_turnstile.stat.exists or 'sponsord_onramp_hosts' in group_names else []) + ([sponsord_secret_env_file] if sponsord_backup_include_env | bool else []) }} _sponsord_backup_file: >- {{ sponsord_backup_dir }}/{{ inventory_hostname }}-sponsord-{{ diff --git a/ansible/roles/sponsord/tasks/decommission.yml b/ansible/roles/sponsord/tasks/decommission.yml index 2454bea..5a5e30f 100644 --- a/ansible/roles/sponsord/tasks/decommission.yml +++ b/ansible/roles/sponsord/tasks/decommission.yml @@ -32,9 +32,12 @@ decdn_decommission_prompt_seconds must be a positive number of seconds. # Checked before the prompt, so nobody types a confirmation for a run that cannot -# proceed; the stop below follows within seconds. +# proceed, and again right before the stop: the prompt can wait minutes, and the +# keeper can start a hold meanwhile. - name: Refuse to stop a sponsord that holds a pool top-up ansible.builtin.include_tasks: topup-hold.yml + vars: + _sponsord_hold_action: stop # Typed confirmation (AGENTS.md hard rule 4), as decdn_node's. A host that already # confirmed in an earlier play of this run (a co-located node, or the onramp) is not @@ -42,13 +45,26 @@ - name: Ask for typed confirmation ansible.builtin.pause: prompt: >- - About to STOP sponsord and REMOVE its systemd unit on: {{ ansible_play_hosts_all | sort | join(',') }}. - The pool and the treasury's funds are NOT touched by this playbook. + About to STOP {{ _decom_services | join(', ') }}, REMOVE their systemd units + and tear down this repo's Grafana Alloy on: + {{ ansible_play_hosts_all | sort | join(',') }}. + This one confirmation covers all of them. Nothing on-chain is done: the pool + and the treasury's funds are NOT touched. Type the host list above exactly to continue (times out after {{ decdn_decommission_prompt_seconds }} s) seconds: "{{ decdn_decommission_prompt_seconds | int }}" register: _sponsord_decommission_typed run_once: true + vars: + # Every service this repo's playbooks/decommission.yml stops on these hosts, by + # their groups, so the one confirmation names all it covers (a later play does + # not ask again). This role's own service is always listed. + _decom_groups: "{{ ansible_play_hosts_all | map('extract', hostvars, 'group_names') | flatten | unique }}" + _decom_services: >- + {{ ((['decdn-node'] if 'decdn_nodes' in _decom_groups else []) + + (['sponsord-onramp and its Caddy'] if 'sponsord_onramp_hosts' in _decom_groups else []) + + (['sponsord'] if 'sponsord_hosts' in _decom_groups else []) + + ['sponsord']) | unique }} when: - decdn_decommission_confirm is not defined - not (_decdn_decommission_confirmed | default(false) | bool) @@ -67,6 +83,11 @@ ansible.builtin.set_fact: _decdn_decommission_confirmed: true +- name: Check again for a held pool top-up, right before the stop + ansible.builtin.include_tasks: topup-hold.yml + vars: + _sponsord_hold_action: stop + - name: Check for the sponsord unit ansible.builtin.stat: path: /etc/systemd/system/sponsord.service diff --git a/ansible/roles/sponsord/tasks/install.yml b/ansible/roles/sponsord/tasks/install.yml index b4f0a59..ea73692 100644 --- a/ansible/roles/sponsord/tasks/install.yml +++ b/ansible/roles/sponsord/tasks/install.yml @@ -177,7 +177,6 @@ owner: root group: root mode: "0755" - notify: Restart sponsord always: - name: Report the retained staging directory after a failure @@ -222,7 +221,6 @@ owner: root group: root mode: "0755" - notify: Restart sponsord # Runs every converge, so a binary that drifted from the pin is caught. With # sponsord_version empty (the usual source/manual case) it degrades to a liveness check. diff --git a/ansible/roles/sponsord/tasks/main.yml b/ansible/roles/sponsord/tasks/main.yml index 53abd4c..2740222 100644 --- a/ansible/roles/sponsord/tasks/main.yml +++ b/ansible/roles/sponsord/tasks/main.yml @@ -642,9 +642,9 @@ the file and set the matching sponsord_* variables in inventory instead. when: sponsord_rpc_url | length == 0 -# After every gate, so a deploy that stops at one has notified no restart for a unit -# that may not exist yet. A run that fails LATER loses its pending restart (Ansible -# runs no handlers for a failed host); the inputs record below catches that. +# After every gate, so a deploy that stops at one has replaced no binary for a unit +# that may not exist yet. Nothing here notifies a restart: the inputs record below +# decides that, so a run that fails before it leaves the restart to the next run. - name: Install the sponsord binary ansible.builtin.import_tasks: install.yml @@ -659,7 +659,6 @@ content: | SPONSORD_RPC_URL={{ sponsord_rpc_url | string | to_json(ensure_ascii=false) }} no_log: true - notify: Restart sponsord when: sponsord_rpc_url | length > 0 # Provenance only ("who wrote secret.env"), so it is written as soon as the file is @@ -730,7 +729,6 @@ owner: root group: root mode: "0644" - notify: Restart sponsord - name: Install the sponsord systemd unit ansible.builtin.template: @@ -739,7 +737,6 @@ owner: root group: root mode: "0644" - notify: Restart sponsord # --- Restart inputs ----------------------------------------------------------------- # Everything the daemon starts from: the credentials, both env files, the unit and @@ -747,7 +744,7 @@ # file), so whatever left the running daemon on stale inputs is caught on the next # converge — an operator rotating a credential or editing a host-provisioned # secret.env out of band, or an earlier run that changed a file and then failed -# before its restart handler ran. An ABSENT record counts only +# before restarting. An ABSENT record counts only # when the daemon is already running (below). - name: Hash the restart inputs ansible.builtin.stat: @@ -787,8 +784,12 @@ | map(attribute='checksum', default='') | join(' ') }} sponsord_inputs_recorded_sums: "{{ sponsord_inputs_recorded.content | default('') | b64decode | trim }}" -# Every notify source above (env files, unit, binary, secret.env) is also a hashed -# input, so this one test covers all of them; the hold guard below reads it too. +# The ONLY restart trigger. The tasks that write an input (secret.env, sponsord.env, +# the unit, the binary) notify nothing themselves: every one of them is hashed +# above, and a restart is owed exactly when the hashes differ from what the daemon +# last started healthy on. One predicate for the restart and the hold guard below, +# so no write (a revert onto the recorded content included) restarts unguarded. +# Under --check nothing is written, so no restart is reported. - name: Decide whether a sponsord restart is pending ansible.builtin.set_fact: _sponsord_restart_pending: >- @@ -796,6 +797,15 @@ or (sponsord_inputs_recorded_sums | length == 0 and sponsord_is_active.status.ActiveState | default('') == 'active') }} +# Before anything is queued: a refused run leaves no restart pending in the handler +# queue (so not even --force-handlers runs it), and writes no inputs record, so the +# restart is retried on the next run. +- name: Refuse a restart that would forget a held pool top-up + ansible.builtin.include_tasks: topup-hold.yml + vars: + _sponsord_hold_action: restart + when: _sponsord_restart_pending | bool + - name: Note restart inputs that changed since the daemon last started healthy ansible.builtin.debug: msg: >- @@ -808,12 +818,6 @@ notify: Restart sponsord when: _sponsord_restart_pending | bool -# Fails, before the handler runs and before the inputs record is written, while the -# daemon holds an unconfirmed pool top-up: the restart is retried on the next run. -- name: Refuse a restart that would forget a held pool top-up - ansible.builtin.include_tasks: topup-hold.yml - when: _sponsord_restart_pending | bool - - name: Apply pending sponsord changes ansible.builtin.meta: flush_handlers diff --git a/ansible/roles/sponsord/tasks/source.yml b/ansible/roles/sponsord/tasks/source.yml index cb4a450..4c40b53 100644 --- a/ansible/roles/sponsord/tasks/source.yml +++ b/ansible/roles/sponsord/tasks/source.yml @@ -423,7 +423,6 @@ executable: /usr/bin/python3 register: sponsord_source_install changed_when: sponsord_source_install.stdout | trim == 'changed' - notify: Restart sponsord # Several GB, and the next build starts from scratch anyway. Kept on # failure, for inspection; the next build replaces it. diff --git a/ansible/roles/sponsord/tasks/topup-hold.yml b/ansible/roles/sponsord/tasks/topup-hold.yml index 431d0d9..3da5bf6 100644 --- a/ansible/roles/sponsord/tasks/topup-hold.yml +++ b/ansible/roles/sponsord/tasks/topup-hold.yml @@ -5,12 +5,21 @@ # further top-up until that transaction mines or provably never can. The hold # lives only in the process: a restart forgets it, and if the transaction is still # pending somewhere the pool can be refilled twice (upstream docs/operator.md, -# "Monitor"). Included before the role flushes a pending restart (main.yml), and -# by the decommission entry points before they stop the daemon. +# "Monitor"). Included before the role queues a restart (main.yml), and by the +# decommission entry points before they stop the daemon. _sponsord_hold_action +# ("restart" or "stop") words the refusal. # -# A daemon that is not running holds nothing. A running one whose /metrics does -# not answer is refused: the hold state is unknown. A binary that predates the -# hold (no such gauge) passes, as there is nothing it could forget. +# Which daemon holds nothing: +# - one that is not running; +# - one whose process owns no TCP listener yet: upstream binds before it starts +# the keeper, so a daemon still starting (or crash-looping before the bind) +# has never topped up; +# - one whose /metrics carries sponsord's keeper series but no hold gauge: a +# binary that predates the hold. +# The probe goes to the address the running process listens on, not this run's +# sponsord_port, so a port change in this run cannot probe something else. A +# listening daemon whose /metrics does not answer, or answers without sponsord's +# keeper series, is refused: its hold state is unknown. - name: Read whether sponsord is running before touching it ansible.builtin.systemd_service: name: sponsord @@ -20,52 +29,85 @@ when: - not ansible_check_mode - _sponsord_hold_unit.status.ActiveState | default('') == 'active' + vars: + _action: "{{ _sponsord_hold_action | default('restart') }}" + # The JSON form: a key=value extra var is a string, which the role's boolean + # check refuses. + _override: >- + ANSIBLE_ARGS='-e {"sponsord_restart_ignore_topup_hold":true}' block: - # KICS flags plain HTTP here; the target is the loopback listener the role asserts. + - name: Read the listeners + ansible.builtin.command: + argv: [ss, -Hltnp] + register: _sponsord_hold_ss + changed_when: false + + # The local address (4th column) of each listener the unit's main process owns. + - name: Find the address the running sponsord listens on + ansible.builtin.set_fact: + _sponsord_hold_addrs: >- + {{ _sponsord_hold_ss.stdout_lines + | select('search', 'pid=' ~ _sponsord_hold_unit.status.MainPID ~ ',') + | map('split') | map(attribute=3) | list }} + + # KICS flags plain HTTP here; the target is the daemon's own loopback listener. - name: Read sponsord /metrics ansible.builtin.uri: - url: "http://{{ sponsord_bind_address }}:{{ sponsord_port }}/metrics" + url: "http://{{ _sponsord_hold_addrs[0] }}/metrics" return_content: true timeout: 5 register: _sponsord_hold_metrics + retries: 3 + delay: 2 + until: _sponsord_hold_metrics.status | default(-1) != -1 changed_when: false failed_when: false + when: _sponsord_hold_addrs | length > 0 - name: Parse the top-up hold gauge ansible.builtin.set_fact: _sponsord_hold_since: >- {{ (_sponsord_hold_metrics.content | default('') - | regex_search('(?m)^sponsord_pool_topup_unconfirmed_since_unix\s+([0-9.eE+]+)\s*$', '\1') + | regex_search('(?m)^sponsord_pool_topup_unconfirmed_since_unix(?:\{[^}]*\})?\s+([0-9.eE+]+)\s*$', '\1') | default([], true) | first | default('0')) | float | int }} + when: _sponsord_hold_addrs | length > 0 - name: Refuse to proceed while the hold state cannot be read ansible.builtin.assert: that: - _sponsord_hold_metrics.status | default(-1) == 200 + - _sponsord_hold_metrics.content | default('') is search('(?m)^sponsord_pool_keeper_failures_total\s') fail_msg: >- - sponsord is running but http://{{ sponsord_bind_address }}:{{ sponsord_port }}/metrics - did not answer 200 (status {{ _sponsord_hold_metrics.status | default('none') }}), - so whether it holds an unconfirmed pool top-up is unknown, and a restart or - stop could forget one. Find out why /metrics does not answer, or set - sponsord_restart_ignore_topup_hold=true once you have confirmed that no - top-up is pending (the treasury has no pending transaction). - when: not (sponsord_restart_ignore_topup_hold | bool) + sponsord is running and listening on {{ _sponsord_hold_addrs | join(', ') }}, + but its /metrics {{ 'did not answer 200 (status ' ~ (_sponsord_hold_metrics.status | default('none')) ~ ')' + if _sponsord_hold_metrics.status | default(-1) != 200 else 'carries no sponsord keeper series' }}, + so whether it holds an unconfirmed pool top-up is unknown, and this + {{ _action }} could forget one. sponsord was not + {{ 'restarted' if _action == 'restart' else 'stopped' }}. Find out why + /metrics does not answer, or, once you have confirmed the treasury has no + pending transaction, re-run with {{ _override }}. + when: + - _sponsord_hold_addrs | length > 0 + - not (sponsord_restart_ignore_topup_hold | bool) - name: Refuse to restart or stop sponsord while a pool top-up is held ansible.builtin.fail: msg: |- sponsord holds an unconfirmed pool top-up (since {{ _since }}). - A restart or stop now forgets the hold, and if that transaction is still - pending the pool can be refilled twice. Nothing was restarted or stopped, - and a deploy's changed files wait on disk for the next run. + A {{ _action }} now forgets the hold, and if that transaction is still + pending the pool can be refilled twice. sponsord was not {{ _done }}. The hold clears by itself once the transaction mines. If it was dropped, send 0-value transactions from the treasury to itself until the hold clears; the daemon's error log names the transaction and how many it takes: sudo journalctl -u sponsord -e | grep -i top Then re-run. Only if you have confirmed the transaction can never mine, - re-run with sponsord_restart_ignore_topup_hold=true. + re-run with {{ _override }}. vars: _since: "{{ '%Y-%m-%dT%H:%M:%SZ' | strftime(_sponsord_hold_since | int, utc=true) }}" + _done: >- + {{ 'restarted; the changed files wait on disk for the next run' if _action == 'restart' + else 'stopped (a service an earlier play of this run decommissioned stays stopped)' }} when: + - _sponsord_hold_addrs | length > 0 - _sponsord_hold_since | int > 0 - not (sponsord_restart_ignore_topup_hold | bool) diff --git a/ansible/roles/sponsord_onramp/README.md b/ansible/roles/sponsord_onramp/README.md index 7582e47..5761db4 100644 --- a/ansible/roles/sponsord_onramp/README.md +++ b/ansible/roles/sponsord_onramp/README.md @@ -138,6 +138,7 @@ See [`defaults/main.yml`](defaults/main.yml) for the full list with comments. | `sponsord_onramp_fund_rate_per_min`, `_poll_rate_per_min` | `""` | Per-address rate limits (upstream 10 and 120; `0` turns one off). | | `sponsord_onramp_bind_address` / `_port` | `127.0.0.1` / `8080` | IPv4 loopback only (asserted). | | `sponsord_onramp_readiness_retries` / `_delay` | `30` / `2` | `/healthz` window (about 60 s). | +| `sponsord_onramp_log_level` | `""` | `RUST_LOG`: a level or `target=level` pairs (`""` = info). | | `decdn_decommission_max_hosts` / `_prompt_seconds` | `1` / `300` | `tasks_from: decommission`, shared with the other roles. | ## Bringing your own proxy @@ -170,8 +171,9 @@ collector; its access logs are not shipped. `sponsord_onramp_min_cli_version` to make existing users re-run the installer. - **Certificate trouble:** `sudo journalctl -u caddy -e`. Check that the domain resolves to the host and that tcp/80 and tcp/443 reach it. -- **Logs:** `journalctl -u sponsord-onramp`. The onramp always logs at info: its - release is built without tracing's env-filter, so `RUST_LOG` does nothing. +- **Logs:** `journalctl -u sponsord-onramp`, at info by default. `sponsord_onramp_log_level` + sets `RUST_LOG`: a level or `target=level` pairs only. The release is built without + tracing's env-filter, so a span or field filter makes it log nothing at all. - **Removal:** `make decommission LIMIT=` (`tasks_from: decommission`, run before sponsord's) asks for the typed host list, stops the onramp and removes its unit, and stops and disables the role's Caddy (only when `/etc/caddy/Caddyfile` @@ -180,8 +182,8 @@ collector; its access logs are not shipped. ```bash sudo rm -f /usr/local/bin/sponsord-onramp /etc/sponsord/sponsord-onramp.env \ - /etc/sponsord/onramp-gate.html /etc/sponsord/.onramp-inputs.sha256 \ - /etc/sponsord/turnstile-secret + /etc/sponsord/onramp-gate.html /etc/sponsord/turnstile-secret \ + /etc/sponsord/.turnstile-secret.sha256 sudo rm -rf /usr/local/lib/sponsord-onramp sudo apt-get remove caddy ``` diff --git a/ansible/roles/sponsord_onramp/defaults/main.yml b/ansible/roles/sponsord_onramp/defaults/main.yml index fd568e8..740d248 100644 --- a/ansible/roles/sponsord_onramp/defaults/main.yml +++ b/ansible/roles/sponsord_onramp/defaults/main.yml @@ -151,6 +151,14 @@ sponsord_onramp_ttl_secs: "" # [the daemon's maximum] TTL per c sponsord_onramp_fund_rate_per_min: "" # [10] POST /v1/fund per address; 0 = off sponsord_onramp_poll_rate_per_min: "" # [120] GET /v1/capability per address; 0 = off +# --- Logging ---------------------------------------------------------------------- +# RUST_LOG for the onramp: a level (debug) or target=level pairs +# (sponsord_onramp=debug,info). "" omits it and the onramp logs at info. Its +# release is built without tracing's env-filter, so span or field filters are not +# understood: the onramp prints "Ignoring RUST_LOG" and then logs NOTHING. Letters, +# digits and _=,.:- only. +sponsord_onramp_log_level: "" + # --- Files on the host ---------------------------------------------------------- # Beside the daemon's files: the directory and the API token follow the sponsord # role's sponsord_etc / sponsord_api_token_file when inventory sets them (the diff --git a/ansible/roles/sponsord_onramp/tasks/decommission.yml b/ansible/roles/sponsord_onramp/tasks/decommission.yml index bbeb022..57ec9ec 100644 --- a/ansible/roles/sponsord_onramp/tasks/decommission.yml +++ b/ansible/roles/sponsord_onramp/tasks/decommission.yml @@ -35,13 +35,25 @@ - name: Ask for typed confirmation ansible.builtin.pause: prompt: >- - About to STOP sponsord-onramp (and the Caddy in front of it) and REMOVE its - systemd unit on: {{ ansible_play_hosts_all | sort | join(',') }}. + About to STOP {{ _decom_services | join(', ') }}, REMOVE their systemd units + and tear down this repo's Grafana Alloy on: + {{ ansible_play_hosts_all | sort | join(',') }}. + This one confirmation covers all of them. Nothing on-chain is done. Type the host list above exactly to continue (times out after {{ decdn_decommission_prompt_seconds }} s) seconds: "{{ decdn_decommission_prompt_seconds | int }}" register: _sponsord_onramp_decommission_typed run_once: true + vars: + # Every service this repo's playbooks/decommission.yml stops on these hosts, by + # their groups, so the one confirmation names all it covers (a later play does + # not ask again). This role's own service is always listed. + _decom_groups: "{{ ansible_play_hosts_all | map('extract', hostvars, 'group_names') | flatten | unique }}" + _decom_services: >- + {{ ((['decdn-node'] if 'decdn_nodes' in _decom_groups else []) + + (['sponsord-onramp and its Caddy'] if 'sponsord_onramp_hosts' in _decom_groups else []) + + (['sponsord'] if 'sponsord_hosts' in _decom_groups else []) + + ['sponsord-onramp and its Caddy']) | unique }} when: - decdn_decommission_confirm is not defined - not (_decdn_decommission_confirmed | default(false) | bool) diff --git a/ansible/roles/sponsord_onramp/tasks/main.yml b/ansible/roles/sponsord_onramp/tasks/main.yml index 64978d3..cfa9a5a 100644 --- a/ansible/roles/sponsord_onramp/tasks/main.yml +++ b/ansible/roles/sponsord_onramp/tasks/main.yml @@ -356,6 +356,17 @@ {{ sponsord_onramp_domain | default('your domain', true) }}. sponsord_onramp_brand_name may hold no control characters. +# RUST_LOG lands unquoted in an EnvironmentFile: no whitespace, quotes or control +# characters, and no span syntax the onramp's Targets filter would reject. +- name: Validate sponsord_onramp_log_level + ansible.builtin.assert: + that: + - sponsord_onramp_log_level | string is match('^[A-Za-z0-9_=,.:-]*\Z') + fail_msg: >- + sponsord_onramp_log_level must be "" (info), a level such as debug, or + target=level pairs such as sponsord_onramp=debug,info (letters, digits and + _=,.:- only), got "{{ sponsord_onramp_log_level }}". + - name: Validate the release pins the installers use ansible.builtin.assert: that: diff --git a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 index 5904d9b..89fb303 100644 --- a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 +++ b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 @@ -31,6 +31,7 @@ ONRAMP_GATE_TEMPLATE={{ sponsord_onramp_gate_template_file }} ('ONRAMP_TTL_SECS', sponsord_onramp_ttl_secs), ('ONRAMP_FUND_RATE_PER_MIN', sponsord_onramp_fund_rate_per_min), ('ONRAMP_POLL_RATE_PER_MIN', sponsord_onramp_poll_rate_per_min), + ('RUST_LOG', sponsord_onramp_log_level), ] if value | string | length > 0 %} {{ name }}={{ value }} {% endfor %} diff --git a/charts/decdn-node/files/monitoring/README.md b/charts/decdn-node/files/monitoring/README.md index 2a0d29d..b39c5b4 100644 --- a/charts/decdn-node/files/monitoring/README.md +++ b/charts/decdn-node/files/monitoring/README.md @@ -52,7 +52,7 @@ which runs on the Ansible and Compose paths only. The chart never renders it: it | File | What it is | |------|------------| | `sponsord/dashboard-sponsord.json` | `uid: decdn-sponsord`: pool balance, the top-up hold, keeper reads and top-ups, capabilities issued, errors by code, the daemon's and the onramp's logs. | -| `sponsord/prometheus-alerts.yml` | Rule group `sponsord`: down, keeper failing, stale pool read, a held top-up (and one held over 2 h), an empty pool, sponsord's own request errors. `runbook_url` points at upstream's `docs/operator.md`, "Monitor". | +| `sponsord/prometheus-alerts.yml` | Rule group `sponsord`: down, keeper failing, a stale pool read (or none since start), a held top-up (and one held over 2 h), an empty pool, sponsord's own request errors. `runbook_url` points at upstream's `docs/operator.md`, "Monitor". | They select `job="sponsord"`, which the `grafana_alloy` role stamps on sponsord's `/metrics` (`grafana_alloy_sponsord_job`); log panels select `unit="sponsord.service"` diff --git a/charts/decdn-node/files/monitoring/sponsord/dashboard-sponsord.json b/charts/decdn-node/files/monitoring/sponsord/dashboard-sponsord.json index d274c36..f00c1d7 100644 --- a/charts/decdn-node/files/monitoring/sponsord/dashboard-sponsord.json +++ b/charts/decdn-node/files/monitoring/sponsord/dashboard-sponsord.json @@ -197,7 +197,7 @@ "uid": "${DS_PROMETHEUS}" }, "title": "Top-up hold", - "description": "sponsord_pool_topup_unconfirmed_since_unix: non-zero while a top-up came back unconfirmed and further top-ups are held. DO NOT restart sponsord while HELD: a restart forgets the hold and the pool can be refilled twice (the deploy tooling refuses). See upstream docs/operator.md, Monitor.", + "description": "sponsord_pool_topup_unconfirmed_since_unix: non-zero while a top-up came back unconfirmed and further top-ups are held. DO NOT restart sponsord while HELD: a restart forgets the hold and the pool can be refilled twice (the Ansible role refuses; on Compose, check by hand). See upstream docs/operator.md, Monitor.", "type": "stat", "targets": [ { @@ -242,7 +242,7 @@ "type": "range", "options": { "from": 1, - "to": null, + "to": 1000000000000.0, "result": { "text": "HELD — do not restart" } @@ -281,7 +281,7 @@ "uid": "${DS_PROMETHEUS}" }, "title": "Since last balance read", - "description": "time() - sponsord_pool_last_check_unix. The keeper reads the pool at start and every SPONSORD_POOL_WATCH_INTERVAL_SECS (1h by default); SponsordPoolCheckStale fires past 3h.", + "description": "time() - sponsord_pool_last_check_unix. The keeper reads the pool at start and every SPONSORD_POOL_WATCH_INTERVAL_SECS (1h by default); SponsordPoolCheckStale fires past 3h. Empty while no read has succeeded since start (SponsordPoolNeverChecked).", "type": "stat", "targets": [ { @@ -289,7 +289,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "time() - sponsord_pool_last_check_unix{job=\"sponsord\", instance=~\"$instance\"}", + "expr": "time() - (sponsord_pool_last_check_unix{job=\"sponsord\", instance=~\"$instance\"} > 0)", "refId": "A", "legendFormat": "{{instance}}" } @@ -725,7 +725,7 @@ "uid": "${DS_LOKI}" }, "title": "Onramp log", - "description": "sponsord-onramp's journal (unit sponsord-onramp.service), when the onramp runs on this host.", + "description": "sponsord-onramp's journal (unit sponsord-onramp.service), when the onramp runs on this host. Info by default (sponsord_onramp_log_level).", "type": "logs", "targets": [ { diff --git a/charts/decdn-node/files/monitoring/sponsord/prometheus-alerts.yml b/charts/decdn-node/files/monitoring/sponsord/prometheus-alerts.yml index 6fa067d..978a7fc 100644 --- a/charts/decdn-node/files/monitoring/sponsord/prometheus-alerts.yml +++ b/charts/decdn-node/files/monitoring/sponsord/prometheus-alerts.yml @@ -13,9 +13,9 @@ # sponsord runs on the Ansible and Compose paths only. The decdn-node chart does # NOT render this file (its PrometheusRule reads ../prometheus-alerts.yml). # -# Validate locally: -# promtool check rules sponsord/prometheus-alerts.yml -# promtool test rules ../../tests/sponsord-alerts_test.yml +# Validate locally with `make lint-helm`: it runs promtool check rules on this +# file, and promtool test rules on ../../../tests/sponsord-alerts_test.yml against +# a copy with the annotations stripped (see that file). # # Load into Grafana Cloud with mimirtool, or into Prometheus via rule_files (see # ../README.md). @@ -61,10 +61,13 @@ groups: # The keeper reads the pool at start, then every # SPONSORD_POOL_WATCH_INTERVAL_SECS (3600 by default). Three missed - # periods; raise it if you lengthened the interval. A gauge of 0 (never - # read since start) also fires, after the same wait. + # periods; raise it if you lengthened the interval. A gauge of 0 (no read + # has succeeded since start) is SponsordPoolNeverChecked below: time() - 0 + # would fire this one after just its `for`. - alert: SponsordPoolCheckStale - expr: time() - sponsord_pool_last_check_unix > 3 * 3600 + expr: >- + sponsord_pool_last_check_unix > 0 + and time() - sponsord_pool_last_check_unix > 3 * 3600 for: 15m labels: component: sponsor @@ -78,6 +81,22 @@ groups: and no top-up is being decided. Check the RPC endpoint and the keeper's errors: journalctl -u sponsord -e. + - alert: SponsordPoolNeverChecked + expr: sponsord_pool_last_check_unix == 0 + for: 3h + labels: + component: sponsor + severity: warning + annotations: + runbook_url: "https://github.com/decdn/sponsord/blob/main/docs/operator.md#5-monitor" + summary: "sponsord has not read its pool balance since it started on {{ $labels.instance }}" + description: | + sponsord_pool_last_check_unix is still 0 three hours after + the series appeared: the keeper has not once read the pool + since sponsord started, so no top-up is being decided. Check + the RPC endpoint and the keeper's errors: + journalctl -u sponsord -e. + - alert: SponsordTopupHeld expr: sponsord_pool_topup_unconfirmed_since_unix > 0 for: 15m @@ -91,8 +110,9 @@ groups: The keeper broadcast a top-up and could not read its receipt, so it sends no further top-up until that transaction mines or provably never can. DO NOT restart sponsord: a restart forgets - the hold, and the pool can be refilled twice. The deploy - tooling refuses restarts while this holds. It clears by itself + the hold, and the pool can be refilled twice. The Ansible role + refuses restarts while this holds; on Compose, and against a + crash or package-upgrade restart, nothing does. It clears by itself once the transaction mines. If it was dropped, send 0-value transactions from the treasury to itself until it clears; the error log line names the transaction and how many it takes. @@ -136,9 +156,16 @@ groups: # chain_unavailable (503): a signer's on-chain registration could not be # read; internal (500): a server-side failure. Both are sponsord's own - # faults, not a caller's. + # faults, not a caller's. sponsord exports a code's series only from its + # first error, already at 1 or more, which increase() cannot see: the + # second operand catches a series that appeared within the window. - alert: SponsordRequestFailures - expr: increase(sponsord_request_errors_total{code=~"chain_unavailable|internal"}[15m]) > 0 + expr: >- + increase(sponsord_request_errors_total{code=~"chain_unavailable|internal"}[15m]) > 0 + or ( + sponsord_request_errors_total{code=~"chain_unavailable|internal"} + unless sponsord_request_errors_total{code=~"chain_unavailable|internal"} offset 15m + ) for: 0m labels: component: sponsor diff --git a/charts/decdn-node/tests/sponsord-alerts_test.yml b/charts/decdn-node/tests/sponsord-alerts_test.yml index bf38f1d..daeaa60 100644 --- a/charts/decdn-node/tests/sponsord-alerts_test.yml +++ b/charts/decdn-node/tests/sponsord-alerts_test.yml @@ -51,14 +51,18 @@ tests: exp_alerts: - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: critical} - # Last check at t=60s and never again: stale once 3h have passed, plus for: 15m. - # A keeper that checks every hour never fires. + # a: last check at t=60s and never again, stale once 3h have passed, plus + # for: 15m. b: a last check that keeps up with the clock, never stale. + # c: no check has ever succeeded (0): not "stale" (time() - 0 would fire after + # just the for), but SponsordPoolNeverChecked after its 3h. - interval: 5m input_series: - series: 'sponsord_pool_last_check_unix{job="sponsord", instance="a"}' values: '60x60' - series: 'sponsord_pool_last_check_unix{job="sponsord", instance="b"}' values: '0+300x60' + - series: 'sponsord_pool_last_check_unix{job="sponsord", instance="c"}' + values: '0x60' alert_rule_test: - eval_time: 3h alertname: SponsordPoolCheckStale @@ -67,6 +71,13 @@ tests: alertname: SponsordPoolCheckStale exp_alerts: - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: warning} + - eval_time: 2h + alertname: SponsordPoolNeverChecked + exp_alerts: [] + - eval_time: 3h30m + alertname: SponsordPoolNeverChecked + exp_alerts: + - exp_labels: {job: sponsord, instance: c, component: sponsor, severity: warning} # An empty pool fires only once the keeper has read it. - interval: 1m @@ -85,15 +96,21 @@ tests: exp_alerts: - exp_labels: {job: sponsord, instance: a, component: sponsor, severity: critical} - # A keeper failure fires at once; the request-error rule ignores caller faults. + # A keeper failure fires at once (that counter is exported from start, at 0). + # The request-error series are written as sponsord exports them: absent until a + # code's first error, then already non-zero. Both of sponsord's own codes fire + # on that first appearance, a caller fault (bad_request) never does, and the + # alert resolves once a code stops rising for 15m. - interval: 1m input_series: - series: 'sponsord_pool_keeper_failures_total{job="sponsord", instance="a"}' values: '0 0 1 1 1' - series: 'sponsord_request_errors_total{job="sponsord", instance="a", code="chain_unavailable"}' - values: '0 0 2 2 2' + values: '_ _ 2x40' + - series: 'sponsord_request_errors_total{job="sponsord", instance="a", code="internal"}' + values: '_ _ _ 1x40' - series: 'sponsord_request_errors_total{job="sponsord", instance="a", code="bad_request"}' - values: '0 0 9 9 9' + values: '_ _ 9x40' alert_rule_test: - eval_time: 3m alertname: SponsordKeeperFailing @@ -103,3 +120,7 @@ tests: alertname: SponsordRequestFailures exp_alerts: - exp_labels: {job: sponsord, instance: a, code: chain_unavailable, component: sponsor, severity: warning} + - exp_labels: {job: sponsord, instance: a, code: internal, component: sponsor, severity: warning} + - eval_time: 30m + alertname: SponsordRequestFailures + exp_alerts: [] diff --git a/compose/README.md b/compose/README.md index dc499ec..81e902c 100644 --- a/compose/README.md +++ b/compose/README.md @@ -248,11 +248,13 @@ the proxy. `ONRAMP_GATE_TEMPLATE` at it in `sponsord-onramp.env`: ```bash - sudo install -m 0644 -o root -g root gate.html /etc/sponsord/onramp-gate/gate.html + sudo install -D -m 0644 -o root -g root gate.html /etc/sponsord/onramp-gate/gate.html # in sponsord-onramp.env: ONRAMP_GATE_TEMPLATE=/etc/sponsord/onramp-gate/gate.html ``` - A path the onramp cannot read stops it at start, with the path in its log. + The onramp serves whatever file `ONRAMP_GATE_TEMPLATE` names, to anyone, so it + must point inside `/etc/sponsord/onramp-gate/`, never at `/run/secrets/`. A path + it cannot read stops it at start, with the path in its log. 6. **TLS in front of the onramp.** Either: - **Caddy** (`caddy` profile): point the domain's DNS at this host and open @@ -296,10 +298,11 @@ that recreates it, a treasury rotation), check that it holds no unconfirmed pool top-up: ```bash -curl -s 127.0.0.1:8090/metrics | grep '^sponsord_pool_topup_unconfirmed_since_unix' +curl -fsS 127.0.0.1:8090/metrics | grep '^sponsord_pool_topup_unconfirmed_since_unix' ``` -Anything but `0` is a held top-up: the keeper broadcast one and could not read its +`… 0` means no hold. An error or no output means the state is unknown: do not +restart until you know. Anything else is a held top-up: the keeper broadcast one and could not read its receipt, so it sends no other until that transaction mines or provably never can. A restart forgets the hold, and the pool can then be refilled twice. Wait for it to clear; if the transaction was dropped, send 0-value transactions from the treasury to @@ -327,16 +330,23 @@ yours. The `SponsordTopupHeld` alert (below) flags it. wallet must own `SPONSORD_POOL_ID`, or sponsord exits at start. - **Health:** no container healthchecks, as for the node. Probe the two `/healthz` endpoints above from the host. sponsord also serves `/metrics` on the same port, and - logs errors only unless `RUST_LOG=info` is set in `sponsord.env`. The onramp ignores - `RUST_LOG` and always logs at info. + logs errors only unless `RUST_LOG=info` is set in `sponsord.env`. The onramp logs + at info; its `RUST_LOG` takes a level or `target=level` pairs only (a span or + field filter makes it log nothing). - **Dashboard and alerts:** a sponsord dashboard and alert rules (pool balance, keeper failures, a held top-up) are in [`charts/decdn-node/files/monitoring/sponsord/`](../charts/decdn-node/files/monitoring/README.md#sponsord). - **Backup:** the treasury keystore and password are the only copy of the key that - owns the pool and its funds. Copy them off the host encrypted, e.g. - `sudo tar -C / -czf - etc/sponsord/treasury-keystore.json etc/sponsord/treasury-password etc/sponsord/api-token | age -r > sponsord.tar.age` - (the Ansible path's `make backup` does the same, see - [`docs/lifecycle.md`](../docs/lifecycle.md)). + owns the pool and its funds. Copy them off the host encrypted, with the same + files the Ansible path's `make backup` takes + ([`docs/lifecycle.md`](../docs/lifecycle.md)): + + ```bash + set -o pipefail + sudo tar -C / --numeric-owner -czf - etc/sponsord/treasury-keystore.json \ + etc/sponsord/treasury-password etc/sponsord/api-token etc/sponsord/turnstile-secret \ + | age -r age1… -o sponsord.tar.age + ``` ## Before a release: a local image diff --git a/compose/compose.yaml b/compose/compose.yaml index 7c3a424..59dc163 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -166,6 +166,8 @@ services: # the HTML in this directory and set ONRAMP_GATE_TEMPLATE to its path in # sponsord-onramp.env. A fixed directory rather than a configurable file, so # no setting can mount a credential into a server that publishes the page. + # ONRAMP_GATE_TEMPLATE itself is free-form, and the onramp serves whatever it + # names: it must point inside this directory, never at /run/secrets. # Created empty when absent: unused, the built-in page is served. - type: bind source: /etc/sponsord/onramp-gate diff --git a/compose/sponsord-onramp.env.example b/compose/sponsord-onramp.env.example index 942c942..9365cb6 100644 --- a/compose/sponsord-onramp.env.example +++ b/compose/sponsord-onramp.env.example @@ -43,12 +43,14 @@ ONRAMP_CLI_SUMS_SHA256= # ONRAMP_TTL_SECS= # ONRAMP_BRAND_NAME=deCDN # Your own gate page instead of the built-in one (placeholders: upstream -# docs/operator.md). It must live in /etc/sponsord/onramp-gate/, the only -# directory compose.yaml mounts for it. +# docs/operator.md). Put it in /etc/sponsord/onramp-gate/, the directory +# compose.yaml mounts for it, and point here INSIDE that directory: the onramp +# serves whatever file this names to anyone, so never a path under /run/secrets. # ONRAMP_GATE_TEMPLATE=/etc/sponsord/onramp-gate/gate.html # POST /v1/fund and GET /v1/capability per address per minute; 0 turns a limit off # ONRAMP_FUND_RATE_PER_MIN=10 # ONRAMP_POLL_RATE_PER_MIN=120 # ONRAMP_RELEASES_BASE=https://github.com/decdn -# No RUST_LOG: the onramp release is built without tracing's env-filter, so it -# ignores it and always logs at info. +# Log level, info by default: a level or target=level pairs only. The release is +# built without tracing's env-filter, so a span or field filter makes it log nothing. +# RUST_LOG=info diff --git a/docs/lifecycle.md b/docs/lifecycle.md index caa2642..c1a0fcf 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -91,7 +91,8 @@ identity on the network and on-chain. **Stop the old host first.** `make decommission LIMIT=old-host` (it removes the node's service and this repo's Alloy agent and keeps the data; ignore the on-chain exit steps it prints, since the identity is moving, not leaving) or run `sudo systemctl disable --now decdn-node` - on it. + on it. On a host that also runs sponsord, `make decommission` takes sponsord and + its onramp down too: to move only the node, use the `systemctl` command. 2. **Prepare the new host** with a normal deploy. Do *not* set `decdn_node_generate_keystore: true` for it. The deploy creates the `decdn` user @@ -139,7 +140,10 @@ make decommission LIMIT= # LIMIT is required; you type the host name to It stops `decdn-node` with `systemctl` (SIGTERM, which is the daemon's graceful drain path), disables it and removes the unit, and tears down the Grafana Alloy agent this -repo installed, if any. `-e decdn_decommission_purge_cache=true` also deletes the +repo installed, if any. On a host that also runs sponsord it removes sponsord and its +onramp as well, after checking for a held top-up +([sponsord](#sponsord-the-onboarding-sponsor)); the one confirmation prompt lists every +service it covers. `-e decdn_decommission_purge_cache=true` also deletes the cache. It refuses to run on more than one host unless you raise `decdn_decommission_max_hosts`, and an unanswered confirmation prompt fails after `decdn_decommission_prompt_seconds` (300). Both entry points refuse a @@ -238,18 +242,36 @@ refilled twice. So the role reads `/metrics` before any restart a deploy would c and `make decommission` before it stops sponsord, and both **fail** while a hold is on. Wait for it to clear (send 0-value transactions from the treasury to itself if the transaction was dropped; the daemon's error log line says how many), then re-run. -`-e sponsord_restart_ignore_topup_hold=true` overrides the deploy's check, only for a -hold you have confirmed can never mine. +`ANSIBLE_ARGS='-e {"sponsord_restart_ignore_topup_hold":true}'` overrides both checks +(and the refusal when `/metrics` does not answer), only for a hold you have confirmed +can never mine. Use the JSON form: `-e name=true` passes the string `"true"`, which +the role's boolean check refuses. Nothing guards a restart this repo does not cause +(a crash, or a package upgrade restarting services); the `SponsordTopupHeld` alert +is how you learn of a hold. **Migration.** Never run two sponsord daemons on one treasury: both would top up the -pool. Decommission the old host first (`make decommission LIMIT=old-host`), stream -the archive into the new host as for a node (step 3 above, without the `chown`: the -files stay `root` `0600`), then deploy it. +pool. + +1. Back up and decommission the old host: `make backup LIMIT=old-host`, then + `make decommission LIMIT=old-host` (on a host that also runs a node, that takes the + node down too). Take the old host out of `sponsord_hosts` and + `sponsord_onramp_hosts`: decommission keeps `/etc/sponsord`, so a later deploy + would start a second daemon on the same treasury. +2. Restore the fetched archive onto the new host. The files stay `root` `0600`: + + ```bash + age -d -i ~/.config/decdn/backup-age.key ansible/backups/old-host/.tar.age \ + | ssh new-host 'sudo tar -xzf - -C / --no-same-owner' + ``` + +3. Deploy the new host. **Decommission.** On a host in `sponsord_onramp_hosts`, `make decommission` stops and removes the onramp's unit and stops and disables the role's Caddy (only when `/etc/caddy/Caddyfile` carries the role's marker). On a host in `sponsord_hosts`, it -then stops and removes sponsord's unit. A co-located host is asked to confirm once. +then checks for a held top-up once more and stops and removes sponsord's unit. One +typed confirmation per run, in the first play that reaches the host, names every +service it covers there. It keeps `/etc/sponsord` (the treasury owns the pool), the binaries and the caddy package, and does not touch the chain: the pool and its USDC stay with the treasury wallet until you withdraw them with the `decdn` CLI. tcp/80 and tcp/443 stay open diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index 3ae109f..9b8c0ba 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -407,6 +407,21 @@ cmp -s "$repo/ansible/roles/decdn_node/files/install-build-output.py" \ || fail "decdn_node and sponsord files/install-build-output.py differ" pass "source-build and decommission defaults, and the installer, identical across roles" +# decommission.yml must check every sponsord host for a held top-up before any play +# stops a service: a refusal after the node and onramp plays would leave the host +# half decommissioned. The role checks again right before its own stop. +if command -v yq >/dev/null; then + pb="$repo/ansible/playbooks/decommission.yml" + [[ "$(yq '.[0].hosts' "$pb")" == sponsord_hosts \ + && "$(yq '.[0].tasks[0]["ansible.builtin.include_role"].tasks_from' "$pb")" == topup-hold ]] \ + || fail "playbooks/decommission.yml: the first play must be sponsord's top-up hold check" + pass "decommission.yml checks for a held top-up before it stops anything" +elif [[ -n ${CI:-} ]]; then + fail "yq is not on PATH in CI; the decommission play-order check would be skipped" +else + skipped+=("decommission play-order check (needs yq)") +fi + # --- baseline firewall holes per host shape, and playbook guards --------------------- if command -v ansible >/dev/null; then expect 0 "firewall holes resolve per host shape, co-located included" \ From d8f6e0ec4797ef747a4bf489a60257b60b33c149 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 7 Oct 2026 22:50:20 +0300 Subject: [PATCH 6/6] fix: address the PR review on decommission, the gate page, backup and logging - decommission.yml: the first play enforces decdn_decommission_max_hosts against every host the run touches (each role checks only its own play, so a node-only and a sponsord-only host both passed), then checks for a held top-up. - compose: the onramp starts through a pinned /bin/sh check that refuses an ONRAMP_GATE_TEMPLATE resolving outside /etc/sponsord/onramp-gate/; upstream would serve any file, and /run/secrets holds the daemon token and the Turnstile secret. invariants.jq pins the script byte for byte (no flags). - backup: every archived credential must be a regular file; tar stores a symlink as the link, not the key. - log levels: each comma-separated part must be a level or target=level; a misspelled level or a bare non-level word silences the onramp (and reads as a target for sponsord). - render-test.sh: the local promtool test step honours PROMTOOL. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 +++ ansible/galaxy/CHANGELOG.md | 12 ++++--- .../sponsord-onramp-lifecycle/side_effect.yml | 30 +++++++++++++++++ .../molecule/validation-sponsord/converge.yml | 33 +++++++++++++++++-- ansible/playbooks/decommission.yml | 27 ++++++++++++--- ansible/roles/sponsord/defaults/main.yml | 3 +- ansible/roles/sponsord/tasks/backup.yml | 15 ++++++--- ansible/roles/sponsord/tasks/main.yml | 19 +++++++---- .../roles/sponsord_onramp/defaults/main.yml | 5 +-- ansible/roles/sponsord_onramp/tasks/main.yml | 19 +++++++---- ansible/tests/playbook-guards/inventory.yml | 5 ++- charts/decdn-node/tests/render-test.sh | 7 ++-- compose/README.md | 7 ++-- compose/compose.yaml | 24 ++++++++++++-- compose/sponsord-onramp.env.example | 5 +-- compose/tests/invariants.jq | 14 ++++++-- tests/scripts-test.sh | 19 ++++++++--- 17 files changed, 200 insertions(+), 49 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1124754..f92d6c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -281,6 +281,11 @@ and its rules have promtool unit tests (`charts/decdn-node/tests/sponsord-alerts read-only one by one into `/run/secrets/`. Listeners and secret paths are set in `environment:`, which beats env files, because the release images default to `0.0.0.0`. + - **Onramp entrypoint:** upstream serves whatever `ONRAMP_GATE_TEMPLATE` names, and + the container also mounts the daemon token and the Turnstile secret, so the + onramp starts through a `/bin/sh` check that the path resolves into + `/etc/sponsord/onramp-gate/`, then a flagless `exec`. `invariants.jq` pins that + script byte for byte (`onramp_entrypoint`): change both together. - **Caddy:** the official image by digest, non-root, keeping only `NET_BIND_SERVICE`. `compose/Caddyfile` is a hand-kept copy of the role's `Caddyfile.j2`: change one, change both. diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index a5ef0a0..34f9532 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -195,12 +195,14 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (passed as JSON) overrides it. - `sponsord`, `sponsord_onramp`: `sponsord_log_level` / `sponsord_onramp_log_level`, rendered as `RUST_LOG` (`""` keeps the daemon's errors-only and the onramp's info - default). The onramp is built without tracing's env-filter, so it takes a level or - `target=level` pairs only; the role refuses anything else. + default). Each comma-separated part must be a level or `target=level`: tracing + reads any other bare word as a target name, and the onramp (built without + tracing's env-filter) logs nothing at all on a value it cannot parse. - `sponsord`: `tasks_from: backup` (`sponsord_backup_*`): the treasury keystore and password, the API token and the onramp's Turnstile secret (`secret.env` opt-in), taken hot and encrypted on the host to `sponsord_backup_age_recipients` (default - `decdn_backup_age_recipients`), then fetched. + `decdn_backup_age_recipients`), then fetched. Each must be a regular file: tar + would store a symlink as the link, not the key. - `sponsord`, `sponsord_onramp`: `tasks_from: decommission`. The onramp's stops it, removes its unit, and stops and disables the role's Caddy (marker-gated). sponsord's refuses while a top-up is held, then stops sponsord and removes its @@ -232,7 +234,9 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) `sponsord_hosts` (and `sponsord_onramp_hosts`): a backup of sponsord's credentials; a decommission that first checks every sponsord host for a held top-up, then removes the node, the onramp and sponsord, and tears down Alloy on - either kind of host. + either kind of host. Its first play enforces `decdn_decommission_max_hosts` + against every host the run touches: each role checks only its own play, so one + node-only and one sponsord-only host would otherwise both pass. - **Breaking:** `decdn_node_install_method`, `sponsord_install_method` and `sponsord_onramp_install_method` now default to `release` (was `manual`). An diff --git a/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml b/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml index 619547c..f54a4c5 100644 --- a/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml +++ b/ansible/molecule/sponsord-onramp-lifecycle/side_effect.yml @@ -37,6 +37,36 @@ hosts: all become: true tasks: + # tar would store the link, not the token: the backup must refuse, not succeed. + - name: Back up with the API token replaced by a symlink + block: + - name: Move the API token aside and symlink it back + ansible.builtin.shell: + cmd: mv /etc/sponsord/api-token /etc/sponsord/api-token.real && ln -s api-token.real /etc/sponsord/api-token + changed_when: true + + - name: Run the sponsord backup against a symlinked credential + ansible.builtin.include_role: + name: sponsord + tasks_from: backup + + - name: Fail if the backup archived a symlink + ansible.builtin.fail: + msg: The backup succeeded with a symlinked API token. + rescue: + - name: Require the regular-file check to be what failed + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Require every path to archive, as a regular file' + fail_msg: "Failed at '{{ ansible_failed_task.name }}', not the regular-file check." + always: + - name: Put the API token back + ansible.builtin.shell: + cmd: >- + if [ -L /etc/sponsord/api-token ]; then + rm /etc/sponsord/api-token && mv /etc/sponsord/api-token.real /etc/sponsord/api-token; fi + changed_when: true + - name: Make the stub report a held top-up ansible.builtin.copy: dest: /etc/sponsord/stub-topup-hold diff --git a/ansible/molecule/validation-sponsord/converge.yml b/ansible/molecule/validation-sponsord/converge.yml index 17f9e31..a807430 100644 --- a/ansible/molecule/validation-sponsord/converge.yml +++ b/ansible/molecule/validation-sponsord/converge.yml @@ -242,6 +242,20 @@ decdn_rejected: "{{ decdn_rejected + ['sponsord-log-level'] }}" when: ansible_failed_task.name == "Validate sponsord_log_level" + # A bare word that is not a level reads as a target name: logs nothing. + - name: "Case sponsord-log-level — a bare word that is not a level" + block: + - name: Run sponsord with a bare non-level word + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_log_level: "verbose" + rescue: + - name: Record sponsord-log-level rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-log-level'] }}" + when: ansible_failed_task.name == "Validate sponsord_log_level" + # A newline would add a line to the EnvironmentFile after SPONSORD_BIND, and # the later assignment wins: this one would put the API on every interface. - name: "Case sponsord-log-level — a log level carrying a second env line" @@ -955,6 +969,20 @@ when: ansible_failed_task.name == "Validate the onramp's limits" # A span filter: the onramp's Targets parser rejects it and then logs nothing. + # A misspelled level: the Targets parser rejects the whole value and logs nothing. + - name: "Case onramp-log-level — an unknown level" + block: + - name: Run sponsord_onramp with an unknown level + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_log_level: "sponsord_onramp=verbose" + rescue: + - name: Record onramp-log-level rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-log-level'] }}" + when: ansible_failed_task.name == "Validate sponsord_onramp_log_level" + - name: "Case onramp-log-level — a span filter" block: - name: Run sponsord_onramp with a span-filter log level @@ -1263,8 +1291,9 @@ "sponsord-network-chain", "sponsord-pool-id", "sponsord-pool-id-empty", "sponsord-wallet-bool", "sponsord-wallet-cli", "sponsord-zero-address", "sponsord-bind", "sponsord-limit", "sponsord-limit", "sponsord-bool", "sponsord-bool", - "sponsord-log-level", "sponsord-log-level", "sponsord-backup", "sponsord-backup", - "sponsord-backup-dir", "sponsord-backup-dir", "onramp-log-level", "sponsord-rpc-newline", + "sponsord-log-level", "sponsord-log-level", "sponsord-log-level", "sponsord-backup", + "sponsord-backup", "sponsord-backup-dir", "sponsord-backup-dir", "onramp-log-level", + "onramp-log-level", "sponsord-rpc-newline", "sponsord-rpc-gate", "sponsord-treasury", "sponsord-treasury", "sponsord-token", "sponsord-dir-missing", "sponsord-dir-format", "sponsord-dir-cross", "sponsord-bind", "sponsord-overwrite-untracked", "sponsord-token", diff --git a/ansible/playbooks/decommission.yml b/ansible/playbooks/decommission.yml index 2cf02af..83dc802 100644 --- a/ansible/playbooks/decommission.yml +++ b/ansible/playbooks/decommission.yml @@ -9,20 +9,37 @@ # make decommission LIMIT= # LIMIT is required; asks for typed confirmation # Take a backup first (make backup). See docs/lifecycle.md. -# Before anything stops: sponsord must not be stopped while it holds an unconfirmed -# pool top-up (a stop forgets the hold, and the pool could be refilled twice). The -# sponsord play below checks again right before its stop. -- name: Check sponsord hosts for a held pool top-up - hosts: sponsord_hosts +# Before anything stops, over every host this run decommissions: +# - the host cap, against all of them: each role checks only its own play, so a +# LIMIT naming one node-only host and one sponsord-only host would pass both +# plays' checks and decommission two; +# - sponsord must not be stopped while it holds an unconfirmed pool top-up (a stop +# forgets the hold, and the pool could be refilled twice). The sponsord role +# checks again right before its stop. +- name: Check the hosts before anything stops + hosts: decdn_nodes:sponsord_onramp_hosts:sponsord_hosts become: true gather_facts: false tasks: + - name: Refuse to decommission more hosts than allowed at once + ansible.builtin.assert: + that: + - decdn_decommission_max_hosts | default(1) | int > 0 + - ansible_play_hosts_all | length <= decdn_decommission_max_hosts | default(1) | int + fail_msg: >- + This run decommissions {{ ansible_play_hosts_all | length }} hosts + ({{ ansible_play_hosts_all | sort | join(', ') }}), more than + decdn_decommission_max_hosts ({{ decdn_decommission_max_hosts | default(1) }}). + Scope it with LIMIT, or raise decdn_decommission_max_hosts deliberately. + quiet: true + - name: Refuse while sponsord holds a pool top-up ansible.builtin.include_role: name: sponsord tasks_from: topup-hold vars: _sponsord_hold_action: stop + when: "'sponsord_hosts' in group_names" - name: Decommission deCDN nodes hosts: decdn_nodes diff --git a/ansible/roles/sponsord/defaults/main.yml b/ansible/roles/sponsord/defaults/main.yml index f90723e..719bd6d 100644 --- a/ansible/roles/sponsord/defaults/main.yml +++ b/ansible/roles/sponsord/defaults/main.yml @@ -106,7 +106,8 @@ sponsord_pool_watch_interval_secs: "" # [3600] how often the keeper checks # --- Logging ---------------------------------------------------------------------- # RUST_LOG for the daemon: a level (info) or tracing directives (sponsord=debug, # hyper=warn). "" omits it, and the daemon then logs errors only; info adds the -# listener and each top-up. Letters, digits and _=,.:- only (no span filters). +# listener and each top-up. Each comma-separated part must be a level or +# target=level (no span or field filters). sponsord_log_level: "" # --- Secrets on the host -------------------------------------------------------- diff --git a/ansible/roles/sponsord/tasks/backup.yml b/ansible/roles/sponsord/tasks/backup.yml index d080124..2598ec5 100644 --- a/ansible/roles/sponsord/tasks/backup.yml +++ b/ansible/roles/sponsord/tasks/backup.yml @@ -104,14 +104,21 @@ register: _sponsord_backup_stat loop: "{{ _sponsord_backup_paths }}" -- name: Require every path to archive +# Regular files only: tar stores a symlink as the link, not the key it points to, +# so a symlinked keystore would back up as a dangling link and restore nothing. +- name: Require every path to archive, as a regular file ansible.builtin.assert: that: - _sponsord_backup_stat.results | rejectattr('stat.exists') | list | length == 0 + - _sponsord_backup_stat.results | selectattr('stat.exists') | rejectattr('stat.isreg') | list | length == 0 fail_msg: >- - Missing on the host: {{ _sponsord_backup_stat.results | rejectattr('stat.exists') - | map(attribute='item') | join(', ') }}. There is no treasury to back up yet - (deploy sponsord first), or a path variable does not match this host. + {% set missing = _sponsord_backup_stat.results | rejectattr('stat.exists') | map(attribute='item') | list %} + {% set odd = _sponsord_backup_stat.results | selectattr('stat.exists') | rejectattr('stat.isreg') + | map(attribute='item') | list %} + {{ ('Missing on the host: ' ~ missing | join(', ') ~ '. There is no treasury to back up yet + (deploy sponsord first), or a path variable does not match this host. ') if missing else '' }} + {{- ('Not a regular file (a symlink or directory): ' ~ odd | join(', ') ~ '. A backup of + it would not hold the credential; replace it with the file itself.') if odd else '' }} - name: Install age ansible.builtin.apt: diff --git a/ansible/roles/sponsord/tasks/main.yml b/ansible/roles/sponsord/tasks/main.yml index 2740222..6f8c6e0 100644 --- a/ansible/roles/sponsord/tasks/main.yml +++ b/ansible/roles/sponsord/tasks/main.yml @@ -319,16 +319,23 @@ sponsord_restart_ignore_topup_hold must be booleans (true/false), not quoted strings. -# RUST_LOG is a level or comma-separated tracing directives. It lands unquoted in an -# EnvironmentFile, so whitespace, quotes and control characters are refused. +# RUST_LOG lands unquoted in an EnvironmentFile, and tracing reads a bare word that +# is not a level as a target name, so a typo like "verbose" would silence the log. +# Each comma-separated part must be a level or target=level; nothing else (no +# whitespace, quotes, span or field filters) gets through. - name: Validate sponsord_log_level + vars: + # One tracing directive: a level, or target=level. + _directive: '(?i)^(?:[a-z_][a-z0-9_:]*=)?(?:trace|debug|info|warn|error|off|[0-5])\Z' ansible.builtin.assert: that: - - sponsord_log_level | string is match('^[A-Za-z0-9_=,.:-]*\Z') + - >- + (sponsord_log_level | string) == '' + or (sponsord_log_level | string).split(',') | reject('match', _directive) | list | length == 0 fail_msg: >- - sponsord_log_level must be "" (errors only), a level such as info, or tracing - directives such as sponsord=debug,hyper=warn (letters, digits and _=,.:- - only), got "{{ sponsord_log_level }}". + sponsord_log_level must be "" (errors only), or comma-separated parts that are + each a level (trace, debug, info, warn, error, off) or target=level, such as + info or sponsord=debug,hyper=warn; got "{{ sponsord_log_level }}". # --- RPC URL: inventory or host-provisioned secret.env ------------------------------ # The same two authoring paths, provenance record and guards as decdn_node's diff --git a/ansible/roles/sponsord_onramp/defaults/main.yml b/ansible/roles/sponsord_onramp/defaults/main.yml index 740d248..1e3e663 100644 --- a/ansible/roles/sponsord_onramp/defaults/main.yml +++ b/ansible/roles/sponsord_onramp/defaults/main.yml @@ -155,8 +155,9 @@ sponsord_onramp_poll_rate_per_min: "" # [120] GET /v1/capability per add # RUST_LOG for the onramp: a level (debug) or target=level pairs # (sponsord_onramp=debug,info). "" omits it and the onramp logs at info. Its # release is built without tracing's env-filter, so span or field filters are not -# understood: the onramp prints "Ignoring RUST_LOG" and then logs NOTHING. Letters, -# digits and _=,.:- only. +# understood: the onramp prints "Ignoring RUST_LOG" and then logs NOTHING, and a +# bare word that is not a level is read as a target. The role accepts only parts +# that are each a level or target=level. sponsord_onramp_log_level: "" # --- Files on the host ---------------------------------------------------------- diff --git a/ansible/roles/sponsord_onramp/tasks/main.yml b/ansible/roles/sponsord_onramp/tasks/main.yml index cfa9a5a..86b2620 100644 --- a/ansible/roles/sponsord_onramp/tasks/main.yml +++ b/ansible/roles/sponsord_onramp/tasks/main.yml @@ -356,16 +356,23 @@ {{ sponsord_onramp_domain | default('your domain', true) }}. sponsord_onramp_brand_name may hold no control characters. -# RUST_LOG lands unquoted in an EnvironmentFile: no whitespace, quotes or control -# characters, and no span syntax the onramp's Targets filter would reject. +# RUST_LOG lands unquoted in an EnvironmentFile, and the onramp's Targets filter +# reads a bare word that is not a level as a target, and rejects a bad level +# outright; either way the onramp then logs nothing. Each comma-separated part +# must be a level or target=level (the sponsord role's sponsord_log_level rule). - name: Validate sponsord_onramp_log_level + vars: + # One tracing directive: a level, or target=level. + _directive: '(?i)^(?:[a-z_][a-z0-9_:]*=)?(?:trace|debug|info|warn|error|off|[0-5])\Z' ansible.builtin.assert: that: - - sponsord_onramp_log_level | string is match('^[A-Za-z0-9_=,.:-]*\Z') + - >- + (sponsord_onramp_log_level | string) == '' + or (sponsord_onramp_log_level | string).split(',') | reject('match', _directive) | list | length == 0 fail_msg: >- - sponsord_onramp_log_level must be "" (info), a level such as debug, or - target=level pairs such as sponsord_onramp=debug,info (letters, digits and - _=,.:- only), got "{{ sponsord_onramp_log_level }}". + sponsord_onramp_log_level must be "" (info), or comma-separated parts that are + each a level (trace, debug, info, warn, error, off) or target=level, such as + debug or sponsord_onramp=debug,info; got "{{ sponsord_onramp_log_level }}". - name: Validate the release pins the installers use ansible.builtin.assert: diff --git a/ansible/tests/playbook-guards/inventory.yml b/ansible/tests/playbook-guards/inventory.yml index fbec66c..024f660 100644 --- a/ansible/tests/playbook-guards/inventory.yml +++ b/ansible/tests/playbook-guards/inventory.yml @@ -1,10 +1,13 @@ --- # Fixture for tests/scripts-test.sh's playbook-guard cases. Nothing here is ever -# contacted: the cases run only the guard play, which gathers no facts. +# contacted: the cases run only guard tasks, which gather no facts. all: vars: ansible_connection: local children: + decdn_nodes: + hosts: + node-only: {} sponsord_hosts: hosts: both: {} diff --git a/charts/decdn-node/tests/render-test.sh b/charts/decdn-node/tests/render-test.sh index 459554c..f74c70f 100755 --- a/charts/decdn-node/tests/render-test.sh +++ b/charts/decdn-node/tests/render-test.sh @@ -13,8 +13,9 @@ # to "" to skip, loudly. # PROMTOOL_IMAGE promtool's container image, for `promtool test rules`, which # reads files rather than stdin: the test directory is mounted into -# it (the Makefile passes the same pinned image). Unset: the local -# `promtool`. Skipped with PROMTOOL. +# it (the Makefile passes the same pinned image). Unset: PROMTOOL is +# run in that directory instead, so it must then be a local binary. +# Skipped with PROMTOOL. # DECDN_CLI path to a real `decdn` binary: also run `decdn config validate` # on every positive render (catches value/type errors the key check # can't). Skipped, loudly, when unset. @@ -162,7 +163,7 @@ if [ -n "$promtool" ]; then docker run --rm -v "$work/rules-test:/w:ro" -w /w --entrypoint promtool "$PROMTOOL_IMAGE" \ test rules sponsord-alerts_test.yml > "$work/promtool.out" 2>&1 else - (cd "$work/rules-test" && promtool test rules sponsord-alerts_test.yml) > "$work/promtool.out" 2>&1 + (cd "$work/rules-test" && $promtool test rules sponsord-alerts_test.yml) > "$work/promtool.out" 2>&1 fi || { cat "$work/promtool.out" >&2; fail "promtool test rules (tests/sponsord-alerts_test.yml)"; } pass "promtool test rules: sponsord alerts" else diff --git a/compose/README.md b/compose/README.md index 81e902c..ca0fdb1 100644 --- a/compose/README.md +++ b/compose/README.md @@ -252,9 +252,10 @@ the proxy. # in sponsord-onramp.env: ONRAMP_GATE_TEMPLATE=/etc/sponsord/onramp-gate/gate.html ``` - The onramp serves whatever file `ONRAMP_GATE_TEMPLATE` names, to anyone, so it - must point inside `/etc/sponsord/onramp-gate/`, never at `/run/secrets/`. A path - it cannot read stops it at start, with the path in its log. + The onramp serves whatever file `ONRAMP_GATE_TEMPLATE` names, to anyone, and its + container also holds the API token and the Turnstile secret. So `compose.yaml` + starts it through a check: unless the path resolves (symlinks followed) to a file + in `/etc/sponsord/onramp-gate/`, the onramp exits at start and its log says why. 6. **TLS in front of the onramp.** Either: - **Caddy** (`caddy` profile): point the domain's DNS at this host and open diff --git a/compose/compose.yaml b/compose/compose.yaml index 59dc163..80a15e1 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -165,15 +165,33 @@ services: # A custom gate page (upstream docs/operator.md, "Your own gate page"): put # the HTML in this directory and set ONRAMP_GATE_TEMPLATE to its path in # sponsord-onramp.env. A fixed directory rather than a configurable file, so - # no setting can mount a credential into a server that publishes the page. - # ONRAMP_GATE_TEMPLATE itself is free-form, and the onramp serves whatever it - # names: it must point inside this directory, never at /run/secrets. + # no setting can mount a credential into a server that publishes the page; + # the entrypoint above refuses a template path outside it. # Created empty when absent: unused, the built-in page is served. - type: bind source: /etc/sponsord/onramp-gate target: /etc/sponsord/onramp-gate read_only: true bind: {create_host_path: true} + # Upstream reads ONRAMP_GATE_TEMPLATE with no path check and serves that file to + # anyone, while /run/secrets in this same container holds the daemon's token and + # the Turnstile secret. So the onramp starts only if the template is unset or + # resolves (symlinks followed) to a file in /etc/sponsord/onramp-gate/. No flags: + # the binary gets none (a flag would beat the loopback environment), and + # tests/invariants.jq pins this script exactly. + entrypoint: + - /bin/sh + - -c + - | + t="$${ONRAMP_GATE_TEMPLATE-}" + if [ -n "$$t" ]; then + r="$$(realpath -e -- "$$t")" || { echo "ONRAMP_GATE_TEMPLATE $$t is not readable" >&2; exit 1; } + case "$$r" in + /etc/sponsord/onramp-gate/*) ;; + *) echo "ONRAMP_GATE_TEMPLATE must be a file in /etc/sponsord/onramp-gate/ (it resolves to $$r)" >&2; exit 1 ;; + esac + fi + exec sponsord-onramp # It reads the daemon's limits only at start, so `docker compose restart # sponsord` restarts it too, like the unit's PartOf= (a stop does not # propagate). It exits while the daemon is unreachable; the restart policy diff --git a/compose/sponsord-onramp.env.example b/compose/sponsord-onramp.env.example index 9365cb6..079bbbc 100644 --- a/compose/sponsord-onramp.env.example +++ b/compose/sponsord-onramp.env.example @@ -44,8 +44,9 @@ ONRAMP_CLI_SUMS_SHA256= # ONRAMP_BRAND_NAME=deCDN # Your own gate page instead of the built-in one (placeholders: upstream # docs/operator.md). Put it in /etc/sponsord/onramp-gate/, the directory -# compose.yaml mounts for it, and point here INSIDE that directory: the onramp -# serves whatever file this names to anyone, so never a path under /run/secrets. +# compose.yaml mounts for it, and point here inside that directory: the onramp +# serves this file to anyone, so compose.yaml refuses to start it on a path that +# resolves anywhere else (/run/secrets holds its credentials). # ONRAMP_GATE_TEMPLATE=/etc/sponsord/onramp-gate/gate.html # POST /v1/fund and GET /v1/capability per address per minute; 0 turns a limit off # ONRAMP_FUND_RATE_PER_MIN=10 diff --git a/compose/tests/invariants.jq b/compose/tests/invariants.jq index d7e817a..f634220 100644 --- a/compose/tests/invariants.jq +++ b/compose/tests/invariants.jq @@ -51,6 +51,11 @@ def inline_secrets: { "sponsord-onramp": ["ONRAMP_DAEMON_TOKEN", "ONRAMP_TURNSTILE_SECRET"] }; +# The onramp's entrypoint, as `config` renders it ($$ is Compose's escape for $): +# refuse an ONRAMP_GATE_TEMPLATE outside /etc/sponsord/onramp-gate/, then exec the +# binary with no arguments. A change to compose.yaml's script must be made here too. +def onramp_entrypoint: ["/bin/sh", "-c", "t=\"$${ONRAMP_GATE_TEMPLATE-}\"\nif [ -n \"$$t\" ]; then\n r=\"$$(realpath -e -- \"$$t\")\" || { echo \"ONRAMP_GATE_TEMPLATE $$t is not readable\" >&2; exit 1; }\n case \"$$r\" in\n /etc/sponsord/onramp-gate/*) ;;\n *) echo \"ONRAMP_GATE_TEMPLATE must be a file in /etc/sponsord/onramp-gate/ (it resolves to $$r)\" >&2; exit 1 ;;\n esac\nfi\nexec sponsord-onramp\n"]; + .services as $all | (["caddy", "decdn-node", "sponsord", "sponsord-onramp"] as $want | check("compose.yaml"; "services are exactly \($want | join(", "))"; ($all | keys) == $want)), @@ -81,10 +86,13 @@ def inline_secrets: { check("sponsord"; "stop_grace_period is 120s"; $all.sponsord.stop_grace_period == "2m0s"), check("sponsord-onramp"; "stop_grace_period is 30s"; $all["sponsord-onramp"].stop_grace_period == "30s"), - # The sponsord daemons run the image's entrypoint with no flags: a flag beats the + # The sponsord daemons run the image's binary with no flags: a flag beats the # environment, so `--bind 0.0.0.0:…` would get past the loopback checks below. - (["sponsord", "sponsord-onramp"][] as $n - | check($n; "no command or entrypoint override"; $all[$n].command == null and $all[$n].entrypoint == null)), + # sponsord runs the image's entrypoint; the onramp runs exactly the gate-page + # path check in compose.yaml, which ends in a flagless exec of the binary. + check("sponsord"; "no command or entrypoint override"; $all.sponsord.command == null and $all.sponsord.entrypoint == null), + check("sponsord-onramp"; "entrypoint is exactly the gate-page check, with no command"; + $all["sponsord-onramp"].command == null and $all["sponsord-onramp"].entrypoint == onramp_entrypoint), # The sponsor's listeners stay on loopback whatever an env file says (the release # images default to 0.0.0.0); the rendered environment has env_file merged in. diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index 9b8c0ba..3bf15b4 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -272,6 +272,8 @@ variant "onramp by tag" "sponsord-onramp: image is pinned" variant "onramp holds keystore" "sponsord-onramp: mounts are exactly" "${onr} s#^(\s*)volumes:\$#\1volumes:\n\1 - /etc/sponsord/treasury-keystore.json:/run/secrets/k:ro#" variant "writable turnstile mount" "sponsord-onramp: mounts are exactly" "${onr} {/target: \/run\/secrets\/turnstile-secret$/{n;s/read_only: true/read_only: false/}}" variant "writable gate-page mount" "sponsord-onramp: mounts are exactly" "${onr} {/target: \/etc\/sponsord\/onramp-gate$/{n;s/read_only: true/read_only: false/}}" +variant "onramp entrypoint flag" "sponsord-onramp: entrypoint is exactly" "${onr} s#^(\s*)exec sponsord-onramp\$#\1exec sponsord-onramp --bind 0.0.0.0:8080#" +variant "gate check dropped" "sponsord-onramp: entrypoint is exactly" "${onr} s#^(\s*)/etc/sponsord/onramp-gate/\*\) ;;\$#\1*) ;;#" variant "gate page from a secret" "sponsord-onramp: mounts are exactly" "${onr} s#source: /etc/sponsord/onramp-gate\$#source: /etc/sponsord/treasury-keystore.json#" variant "inline Turnstile secret" "sponsord-onramp: no inline ONRAMP_TURNSTILE_SECRET" "${onr} s/^(\s*)ONRAMP_BIND: (.*)$/&\n\1ONRAMP_TURNSTILE_SECRET: x/" variant "onramp on sponsord hosts" "sponsord-onramp: profiles are [onramp]" "${onr} s/^(\s*)profiles: \[onramp\]$/\1profiles: [onramp, sponsord]/" @@ -412,10 +414,11 @@ pass "source-build and decommission defaults, and the installer, identical acros # half decommissioned. The role checks again right before its own stop. if command -v yq >/dev/null; then pb="$repo/ansible/playbooks/decommission.yml" - [[ "$(yq '.[0].hosts' "$pb")" == sponsord_hosts \ - && "$(yq '.[0].tasks[0]["ansible.builtin.include_role"].tasks_from' "$pb")" == topup-hold ]] \ - || fail "playbooks/decommission.yml: the first play must be sponsord's top-up hold check" - pass "decommission.yml checks for a held top-up before it stops anything" + [[ "$(yq '.[0].hosts' "$pb")" == decdn_nodes:sponsord_onramp_hosts:sponsord_hosts \ + && "$(yq '.[0].tasks[0]["ansible.builtin.assert"] | has("that")' "$pb")" == true \ + && "$(yq '.[0].tasks[1]["ansible.builtin.include_role"].tasks_from' "$pb")" == topup-hold ]] \ + || fail "playbooks/decommission.yml: the first play must check the host cap, then sponsord's top-up hold" + pass "decommission.yml checks the host cap and a held top-up before it stops anything" elif [[ -n ${CI:-} ]]; then fail "yq is not on PATH in CI; the decommission play-order check would be skipped" else @@ -443,6 +446,14 @@ if command -v ansible >/dev/null; then expect 2 "sponsord.yml refuses an onramp host outside sponsord_hosts" guard onramp-only grep -q "is in sponsord_onramp_hosts but not in sponsord_hosts" "$work/out" \ || { cat "$work/out" >&2; fail "the onramp-only refusal did not come from the placement check"; } + # decommission.yml's host cap covers the whole run: a node-only host and a + # sponsord host pass each role's own per-play cap but not this one. Only the + # cap task is lifted (the hold check needs the roles). + yq '[.[0] | .tasks = [.tasks[0]]]' "$repo/ansible/playbooks/decommission.yml" > "$work/guard.yml" + expect 0 "decommission.yml accepts one host" guard node-only + expect 2 "decommission.yml refuses a node host plus a sponsord host" guard node-only,both + grep -q "This run decommissions 2 hosts" "$work/out" \ + || { cat "$work/out" >&2; fail "the two-host refusal did not come from the run-wide host cap"; } elif [[ -n ${CI:-} ]]; then fail "yq is not on PATH in CI; the playbook-guard cases would be skipped" else